```
import jax.numpy as jnp
from jax import grad, jit, vmap
from jax import jacfwd, jacrev
from jax import random
import numpy as np
import sympy as sym
import matplotlib.pyplot as plt

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
from celluloid import Camera
import time

class Visualization:
    def __init__(self, X_opt, cost, lb, ub):
        self.X_opt = X_opt
        self.cost = cost
        self.lb = lb
        self.ub = ub

        self.f = plt.figure(figsize=(12, 8))
        plt.rc('axes', linewidth=2)

        x = np.arange(lb, ub, 0.1)
        self.X, self.Y = np.meshgrid(x, x)
        M = {0: self.X, 1: self.Y}
        self.F = self.cost(M)

    def plot_optimization(self):
        plt.plot([self.X_opt[i][0] for i in range(len(self.X_opt))],
                 [self.X_opt[i][1] for i in range(len(self.X_opt))], '-*', alpha=0.5, color='red', markersize=15, lw=2.0)
        plt.contour(self.X, self.Y, self.F)
        plt.xlabel('$x_1$', size=25)
        plt.ylabel('$x_2$', size=25)
        plt.show()
        # f.savefig("steepest_descent_armijo.pdf", bbox_inches='tight')

    def animate_optimization_2D(self):
        camera = Camera(plt.figure(figsize=(10, 8)))

        # plot contour of the function
        # plt.contour(X, Y, F)
        # policy
        x1 = [X_opt[i][0] for i in range(len(X_opt))]
        x2 = [X_opt[i][1] for i in range(len(X_opt))]

        for i in range(len(x1)):
            plt.contour(X, Y, F)
            x1data, x2data = x1[:i+1], x2[:i+1]
            plt.plot(x1data, x2data, '-', color='red', markersize=10, lw=2.0)
            plt.plot(x1data[-1], x2data[-1], 'o', color='red', markersize=10, lw=2.0)
            plt.xlabel('$x_1$', size=25)
            plt.ylabel('$x_2$', size=25)
            camera.snap()

        anim = camera.animate(blit=True)
        anim.save('optimization_animation.gif', writer='imagemagick')
        plt.show()

    def animate_optimization_3D(self):
        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')

        camera = Camera(plt.figure(figsize=(10, 8)))

        x1 = [X_opt[i][0] for i in range(len(X_opt))]
        x2 = [X_opt[i][1] for i in range(len(X_opt))]

        for i in range(len(x1)):
            plt.contour(X, Y, F)
            x1data, x2data = x1[:i+1], x2[:i+1]
            plt.plot(x1data, x2data, '-', color='red', markersize=10, lw=2.0)
            plt.plot(x1data[-1], x2data[-1], 'o', color='red', markersize=10, lw=2.0)
            plt.xlabel('$x_1$', size=25)
            plt.ylabel('$x_2$', size=25)
            camera.snap()

        anim = camera.animate(blit=True)
        anim.save('optimization_animation.gif', writer='imagemagick')
        plt.show()

class UnconstrainedOptimization:
    def __init__(self, cost, direction_type, stepsize_type, x0,
                 termination_params, algo_params=None):
        self.cost = cost
        self.x0 = jnp.array(x0)
        self.direction_type = direction_type
        self.stepsize_type = stepsize_type
        self.algo_params = algo_params
        self.termination_params = termination_params

    def armijo_condition(self, alpha, sigma, x_current, x_next, direction):
        LHS = self.cost(x_current) - self.cost(x_next)
        RHS = - sigma * alpha * jnp.matmul(grad(self.cost)(x_current), direction)
        condition = True if (LHS >= RHS) else False
        return condition

    def get_step_size(self, x_current, direction):

        if self.stepsize_type == 'minimization_rule':
            # create symbolic step size: alpha
            alpha = sym.Symbol('alpha')
            # convert jax device array to numpy list
            x_current = x_current.tolist()
            direction = direction.tolist()
            sym_vec = []
            for d in range(len(direction)):
                mm = x_current[d] + alpha * direction[d]
                sym_vec.append(mm)
            sym_cost = self.cost(sym_vec)
            sym_derivative = sym.diff(sym_cost, alpha)
            alpha_sym = sym.solveset(sym_derivative, alpha)
            alpha_sym = list(alpha_sym)
            print(alpha_sym)

            # only keep the real number by checking data type
            i = 0
            while True:
                condition = type(alpha_sym[i] ** 2) is sym.numbers.Float
                if condition is True:
                    val = alpha_sym[i]
                    break
                else:
                    i += 1

            alpha = float(val)

        elif self.stepsize_type == 'constant':
            alpha = self.algo_params[2]

        elif self.stepsize_type == 'armijo_rule':
            beta, s, sigma = self.algo_params
            m = 0
            condition = False
            while True:
                alpha = pow(beta, m) * s
                x_next = x_current + alpha * direction
                condition = self.armijo_condition(alpha, sigma, x_current, x_next, direction)

                if condition is True:
                    break
                m += 1
        else:
            raise Exception(
                'Unknown step size type, please input one of the following: minimization_tule, constant, armijo rule')

        return alpha

    def get_gradient_direction(self, x_val):
        # get derivative of the cost function
        derivative = grad(self.cost)(x_val)

        if self.direction_type == 'steepest_descent':
            # D in case of steepest descent is identity matrix
            D = jnp.identity(len(x_val))
            direction = - jnp.matmul(D, derivative)

        elif self.direction_type == 'newton_method':
            hessian = jacfwd(jacrev(self.cost))(x_val)
            D = jnp.linalg.inv(hessian)
            direction = - jnp.matmul(D, derivative)

        else:
            raise Exception(
                'Unknown gradient based method, please input one of the following: '
                'steepest_descent, newton_method')

        return direction

    def update(self, x_current, X_opt, error_val, error_cache):

        # update states
        X_opt.append(np.asarray(x_current, dtype=float))
        direction = self.get_gradient_direction(x_current)
        step_size = self.get_step_size(x_current, direction)
        x_next = x_current + step_size * direction
        error_val = x_current - x_next
        error_norm = jnp.linalg.norm(error_val)
        error_cache.append(error_norm)

        return x_next, X_opt, error_norm, error_cache

    def run_algorithm(self):
        """
        Runs the algorithm provided the user with starting and termination conditions,
        If the termination condition is "convergence" then the algorithm runs until convergence
        is achieved upto the desired value, the other termination option is to run the algorithm for
        a fixed number of timesteps.
        :return: X_opt = a matrix with co-ordinates of all the states obtained from running the algorithm
        """

        # get termination conditions from algo_params
        termination_type, termination_condition = self.termination_params

        x_current = self.x0
        X_opt = []
        error_norm = np.inf
        error_cache = []
        fig = plt.figure()

        if termination_type == "convergence":
            while error_norm > termination_condition:
                x_next, X_opt, error_norm, error_cache = self.update(
                    x_current, X_opt, error_norm, error_cache)
                # update state vector
                x_current = x_next

        elif termination_type == "fixed_steps":
            for t in range(termination_condition):
                x_next, X_opt, error_norm, error_cache = self.update(
                    x_current, X_opt, error_norm, error_cache)
                # update state vector
                x_current = x_next

        return X_opt, error_cache
```

```

cost = lambda x: 3*pow(x[0], 2) + pow(x[1], 4)

x0 = [-2.0, -2.0]
algo_params = [0.5, 1, 0.1]
opt = UnconstrainedOptimization(cost, 'steepest_descent', 'armijo_rule', x0,
                                ['fixed_steps', 25], algo_params)
X_opt, error_cache = opt.run_algorithm()
viz = Visualization(X_opt, cost, 1.5, -1.5)
viz.animate_optimization()

```

```
cost = lambda x: 3*pow(x[0], 2) + pow(x[1], 4)

x = np.arange(-2.0, +2.0, 0.1)
X, Y = np.meshgrid(x, x)
M = {0: X, 1: Y}
F = cost(M)

fig = plt.figure(figsize=(10, 8))
ax = plt.axes(projection='3d')
ax.view_init(45, 30)
camera = Camera(fig)

# plot contour of the function
# plt.contour(X, Y, F)
# policy
x1 = [X_opt[i][0] for i in range(len(X_opt))]
x2 = [X_opt[i][1] for i in range(len(X_opt))]
f_opt = [cost(X_opt[i]) for i in range(len(X_opt))]

ax.scatter3D(x1data[0], x2data[0], f_opt[0], 'o', color='black', lw=2.0)
for i in range(len(x1)):
    x1data, x2data = x1[:i+1], x2[:i+1]
    ax.plot3D(x1data, x2data, f_opt[:i+1], '-', color='black', lw=2.0)
    ax.scatter3D(x1data[-1], x2data[-1], f_opt[-1], 'o', color='black', lw=2.0)
    ax.plot_surface(X, Y, F, rstride=1, cstride=1,
            cmap='viridis', edgecolor='none')

    camera.snap()

anim = camera.animate(blit=True)
anim.save('optimization_animation.gif', writer='imagemagick')
plt.show()
```

```

```

```

```

```

```

```

```

.ipynb_checkpoints/Test code notebook 01 -checkpoint.ipynb


# Asteroid detection
Mélina VANEL
19/02/2021

The purpose of this notebook is to estimate how long ( in days ) an asteroid would be visible by a telescope given some parameters like minimum magnitude detectable which depends on the integration time, the detector size, the FOV...

The final goal is to study the possibilty to find a strategy to be able to detected asteroids above 100m diameter with a 99.9% probability. That is why in this study we will often compute quantities for 100m asteroids because it represents the worst case ( less bright as they are the smallest asteroids we want to detect ).

```python
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import colorbar, pcolor, show
from matplotlib import ticker,cm
import pandas as pd
import matplotlib as mpl
from scipy.integrate import quad
```

```python
###################
# Constants
###################

h = 6.62607004e-34              # Planck Constant

c = 299792458.0                 # Speed of light

G = 6.67*10**(-11)              # Gravitational constant

M = 1.989*10**(30)              # Solar mass
```

#### Diameter, absolute magnitude and apparent magnitude

First of all, it is important to find the link between the diameter and the Absolute Magnitude which is :
\begin{align} D=\frac{1329km*10^{\frac{-H}{5}}}{\sqrt{p_v}}\end{align}

With $D$ the diameter in km, $H$ the absolute magnitude and $p_v$ the geometric albedo.

**Source** : *Physical properties of near-Earth asteroids from thermal infrared observations and thermal modeling
/ Marco Delbo*

The absolute magnitude is measured by the telescope but the geometric albedo $p_v$ cannot be measured, that is why we decided to take $p_v$ between 0.03 and 0.15 as its distribution among C type asteroids ( which are the most abundant ) is :

```python
from IPython.display import Image
Image("pv.png",width=300)
```

```python
def D_to_abs_mag(D,p_v):
    '''
    Parameters
    ----------
    D : Diameter (km)
    p_v : Geometric Albedo ( from 0.03 to 0.15 for NEO )

    Returns
    -------
    H : Absolute Magnitude
    '''
    return -5*np.log10(np.sqrt(p_v)*D/1329)

def abs_mag_to_D(H,pv):
    '''
    Parameters
    ----------
    H : Absolute Magnitude
    pv : Geometric Albedo ( from 0.03 to 0.15 for NEO )

    Returns
    -------
    H : Diameter in meter (m)
    '''
    return 1329*10**(-H/5)/(np.sqrt(pv))*10**3
```

Given two distances :
- distance to the Sun $d_{sun}$ in UA
- distance to the observer $d_{obs}$ in UA

it is then possible to compute the apparent magnitude m from absolute magnitude H :

\begin{align} m = H + 5log_{10}\left ( \frac{d_{sun}*d_{obs}}{1UA^2}\right ) -2.5log_{10}(q)\end{align}

With q the phase angle, that we will assume to be $0.290+0.684*0.15$

**Source :** *https://en.wikipedia.org/wiki/Absolute_magnitude*

```python
def abs_mag_to_mag(H,d_sun_ast,d_obs_ast):
    '''
    Parameters
    ----------
    H : Absolute Magnitude
    d_sun_ast : distance bewteen Sun and NEO ( /!\ UA )
    d_obs_ast : distance bewteen Satellite and NEO ( /!\ UA )

    Returns
    -------
    m : Apparent Magnitude
    '''
    # Distance in UA
    q=0.290+0.684*0.15
    return H + 5*np.log10(d_sun_ast*d_obs_ast) - 2.5*np.log10(q)
```

```python
D_SUN = np.linspace(0.00001,6,1000)     # in UA
D_OBS = np.linspace(0.00001,5,1000)     # in UA
X, Y = np.meshgrid(D_SUN, D_OBS)

mag_abs=D_to_abs_mag(0.1,0.03)          # absolute magnitude of a 100m diameter asteroid with pv = 0.03
mag_app = abs_mag_to_mag(mag_abs,X,Y)

fig, ax = plt.subplots()
plt.pcolor(X, Y, mag_app)               # from magnitude 18 because if the asteroid is very close to the Sun and the observer
                                        # the magnitude will be very low and it won't be easy to put very low and high magnitude
                                        # on the same graph that is why there is green in the back left corner
level = [18,20,21,22,23,24,25,26,27,28,29,30,31,32,35,40]
cs = ax.contourf(X, Y, mag_app,level)

cbar = fig.colorbar(cs)
cbar.ax.set_ylabel('mag')
plt.xlabel('Distance to observer (UA)')
plt.ylabel('Distance to Sun (UA)')
plt.title('Magnitude for 100m diameter and 0.03 geometric albedo')
plt.show()
```

In order to be able to determine if an asteroid is detected or not it is then necessary to compute the flux of a given source with a certain magnitude m. We assume that a source of magnitude 0 has a flux of $3.6*10^6$ ph/s/cm². Moreover, we consider that the background has a magnitude of 22mag/arcsec² which will contribute to generate some noise.

```python
def compute_min_flux(M_min):
    '''
    Parameters
    ----------
    M_min : Apparent Magnitude of source

    Returns
    -------
    F : Flux in photon/cm2/s sent by a source with apparent magnitude M_min
    '''
    M_origine = 0

    # Flux of origine in ph/s/cm-2.
    F_origine = 3.6*10**(6)

    Flux_min = F_origine*10**(2/5*(M_origine-M_min))

    # Flux in photons/cm2/s
    Flux_photons_cm_s = Flux_min

    return Flux_photons_cm_s

def compute_F_background():
    '''
    Returns
    -------
    F_background : Flux from background considering it as an apparent magnitude
        of 22mag/degree
    '''
    F_origine = 3.6*10**(6)
    M_origine = 0
    return F_origine*10**(2/5*(M_origine-22))
```

We then import data from a csv file on keplerian parameters of a sample of 1000 asteroids.

```python
neos1000 = 'Atens_Apollos_sup100m_r.csv'
neos1000 = pd.read_csv(neos1000)
```

```python
def compute_period_fromtraj(neo):
    '''
    Parameters
    ----------
    neo : for a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    T : orbital period using Kepler 3rd law T**2/a**3 = 4pi**2/MG
    '''
    M = 1.989*10**30    # Solar mass
    G = 6.67*10**(-11)

    a = neo['a']

    if isinstance(a,float):
        a_axe = np.sqrt(4*np.pi**2*(a*1.496*10**(11))**3/(G*M))

    else:
        a_axe = []
        for i in range(len(neo)):
            a_axe.append(np.sqrt(4*np.pi**2*(a[i]*1.496*10**(11))**3/(G*M)))

    return a_axe
```

#### Characteristics of asteroids sample

In order to better understand the calculs that will be done further in this notebook it is important to have an idea of the parameters of the sample of 1000 asteroids.
That is why we plot the histograms of semi major axis, exentricity and orbital period of the sample of 1000 asteroids.

```python

plt.hist(neos1000['a'],bins=50,facecolor='blue', alpha=0.5)
plt.xlabel('UA')
plt.ylabel('Frequency')
plt.title('Semi major axis histogram of some NEO')
plt.show()

plt.hist(neos1000['e'],bins=50,facecolor='blue', alpha=0.5)
plt.xlabel('Excentricity')
plt.ylabel('Frequency')
plt.title('Excentricty histogram of some NEO')
plt.show()

plt.hist(compute_period_fromtraj(neos1000),bins=50,facecolor='blue', alpha=0.5)
plt.xlabel('Period (years)')
plt.ylabel('Frequency')
plt.title('Period histogram of some NEO')
plt.show()

plt.hist(neos1000['i'],bins=50,facecolor='blue', alpha=0.5)
plt.xlabel('Inclinaison')
plt.ylabel('Frequency')
plt.title('Inclinaison histogram of some NEO')
plt.show()
```

Now that we better understand NEOs ( Near Earth Objects ) that we want to detect, it is important to think of a strategy to detect as much NEO as we can. In this study the goal was to be able to detect 99.9% of NEO with a diameter larger than 100m.

#### SNR needed on telescope detectors

In order to maximize the Signal to Noise Ratio on detectors we decided to fix some detectors characteristics :

| Capacity | Size | IFOV | Dark Current |
| :-: | :-: | :-: | :-:|
| $75$*$10^3$ electrons | 10000*10000pix | 1.08 | 1 |

For the instrument we have :

| FOV | Diameter | Efficiency | Quantum Efficiency |
| :-: | :-: | :-: | :-:|
| 3°x3° | 1m | 0.8 | 0.9 |

In order to compute the SNR the time allowed per sky portion need to be known. Our satellite will be on a Venus like orbit, in order to simplify the study, it has been decided that in 1 period ( 224.7 days ) all the sky must have been scanned and a guard angle of 90° must be left between the sun and the line of sight.

```python
def compute_scanning_time_m(time_to_scan_entier_sky,FOV):
    '''
    Parameters
    ----------
    time_to_scan_entier_sky : time allowed to scan entierely the sky (days)
    FOV : Field of view ( degree )

    Returns
    -------
    Time allowed per image
    '''
    nb_pointing_horiz = 360/FOV
    nb_pointing_vert = 180/FOV
    return (time_to_scan_entier_sky / nb_pointing_horiz)/nb_pointing_vert

print('Time per sky portion (min):',compute_scanning_time_m(224.7,3)*60*24)
```

    Time per sky portion (min): 44.94

In order to respect the criterion : scan all the sky in 1 period, we have 44min94s per sky portion ( 3°x3°).

With all these characteristics it is then possible to compute the signal to noise ratio :
\begin{align} \frac{S}{N} = \frac{FA_{\epsilon}\sqrt{\tau}}{(\frac{N_R^2}{\tau}+FA_{\epsilon}+i_{DC}+F_{\beta}A_{\epsilon}\Omega)^{\frac{1}{2}}}\end{align}

with :
- $F$ = point source signal flux
- $F_{\beta}$ = Background flux from sky
- $\Omega$ = Pixel size
- $\tau$ = Integration time
- $i_{DC}$ = Dark current
- $A_e=A\epsilon Q_e$
- $Q_e$ = Quantum efficiency
- $NR$ = Read out noise

**Source** : *Notes for PHYS 134: Observational Astrophysics / Bill Wolf and Phil Lubin*

We computed with an excel table that for 1 image the SNR needed to have 99.9% probability of detection is 9.6 and for 2 images the SNR needed is 6.1.

As the SNR is related to the incoming flux, it is then possible to compute the minimum magnitude visible with our telescope in order to be capable of detecting an object with accuracy 99.9%. The aim is to maximize the limit detectable magnitude in order to be able to detect fainter objects. After an optimization study, on either doing lot of image with small integration time or longer but fewer images, the chosen strategy is the following :

```python
Image("strategie.png",width=300)
```

The limit magnitude detected is 25.27. That is to says, a 100m diameter asteroid would be visible under the dark curve which corresponds to apparent magnitude 25.21 :

```python
# 100m asteroid
D_SUN = np.linspace(0.00001,6,1000)
D_OBS = np.linspace(0.00001,5,1000)

X, Y = np.meshgrid(D_SUN, D_OBS)

mag_abs=D_to_abs_mag(0.1,0.03)
mag_app = abs_mag_to_mag(mag_abs,X,Y)

fig, ax = plt.subplots()
plt.pcolor(X, Y, mag_app)
level = [18,20,21,22,23,24,25,26,27,28,29,30,31,32,35,40]
cs = ax.contourf(X, Y, mag_app,level)
cs2 = ax.contour(cs,levels=[25.21],colors="black" )

cbar = fig.colorbar(cs)
cbar.ax.set_ylabel('mag')
plt.xlabel('Distance to observer (UA)')
plt.ylabel('Distance to Sun (UA)')
plt.title('Magnitude for 100m diameter and 0.03 geometric albedo')
plt.show()
```

#### Study of asteroid visibility windows

Now that we have the limit magnitude detected it can be interessant to ask the question : How long asteroids are detectable ?

In order to do this, we will use Kepler second law ( area law ). For a given radius of position on the ellipse we have :
\begin{align} r = \frac{p}{1+ecos(\theta)}\end{align}

The area of a section of an ellispe can be compute :
\begin{align} A = \int_{\theta_1}^{\theta_2}\frac{1}{2}r^2d\theta\end{align}

```python
Image("aire.png",width=300)
```

In order to compute time spend by the asteroid in this area we use the 2nd Kepler law :
\begin{align} \frac{dA}{dt}=\frac{C}{2}\end{align}

With  $C=r^2\theta = v_{perihelion}*r_{perihelion} = \sqrt{\mu a (1+e)(1-e)}$

```python
def compute_period_fromtraj(neo):
    '''
    Parameters
    ----------
    neo : for a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    T : orbital period using Kepler 3rd law T**2/a**3 = 4pi**2/MG
    '''
    M = 1.989*10**30    # Solar mass
    G = 6.67*10**(-11)

    a = neo['a']

    if isinstance(a,float):
        a_axe = np.sqrt(4*np.pi**2*(a*1.496*10**(11))**3/(G*M))

    else:
        a_axe = []
        for i in range(len(neo)):
            a_axe.append(np.sqrt(4*np.pi**2*(a[i]*1.496*10**(11))**3/(G*M)))

    return a_axe
```

```python
def compute_b_from_ae(neo):
    '''
    Parameters
    ----------
    neo : a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    b : Semi minor axis ( same units as a, /!\ could be in UA )
    '''
    return neo['a']*np.sqrt(1-neo['e']**2)
```

```python
def compute_theta_from_dist(dist,neo):
    '''
    Parameters
    ----------
    dist : distance between NEO and Sun ( UA )
    neo : a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    theta : angle between apogee and actual position
    '''
    # dist en UA
    dist = dist*1.496*10**(11)     # in m
    a = neo['a']*1.496*10**(11)
    e = neo['e']
    b = compute_b_from_ae(neo)*1.496*10**(11)
    p = b**2/a
    return np.arccos((dist-p)/(dist*e))
```

```python
def compute_theta_from_dist(dist,neo):
    '''
    Parameters
    ----------
    dist : distance between NEO and Sun ( UA )
    neo : a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    theta : angle between apogee and actual position
    '''
    # dist en UA
    dist = dist*1.496*10**(11)     # in m
    a = neo['a']*1.496*10**(11)
    e = neo['e']
    b = compute_b_from_ae(neo)*1.496*10**(11)
    p = b**2/a
    return np.arccos((dist-p)/(dist*e))
```

```python
def arealaw_constant(neo):
    '''
    Parameters
    ----------
    neo : a dataframe containing keplerian parameters of neo(s)

    Returns
    -------
    C : constant of the 2nd Kepler law ( area law )
    '''
    mu = G*M
    a = neo['a']*1.496*10**(11)
    e = neo['e']
    return np.sqrt(mu*a*(1+e)*(1-e))
```

```python
def r2(a,b,e,theta):
    p = b**2/a
    return 1/2*(p/(1-e*np.cos(theta)))**2
```

##### Link between Sun distance and satellite distance

As mentionned before, in order to compute how much time an asteroid will stay in visible area ( in function of its diameter and orbit ) it is necessary to compute the distance from Sun at which it will become visible given the limit magnitude. But to deduce a distance from Sun it is necessary to have the distance from observer.

We will then simplify the study. As mentionned before, we decided to consider a Venus like orbit ( $a = 0.7$ UA, e~0 ) for our satellite, considering that it is possible to detect the asteroid at $n$UA and as we want to keep a 90° angle with the Sun, we are in this configuration :

```python
Image("dist_obs_max_min.png",width=300)
```

That is to say that if an asteroid is visible at nUA from the Sun it is at a maximum distance $d_{max} = \sqrt{nUA^2-0.7^2}$ ( worst case ) from satellite and minimum distance of $d_{min}=nUA - 0.7$ ( best case ). With this simplification, it will be easier to compute the time of visible area for each asteroid. We will consider a best case, a worst case and an average case.

It then now possible from the 1000neos catalogue to compute for each neo the distance from the Sun at which they are visible ( in the best case $ d_{min} $, worst case $d_{max}$ and average case $mean(d_{min},d_{max})$). Then it is possible to deduce with the area of an ellipse portion and 2nd kepler law the time spend for each asteroid in the visibility zone : distance from Sun for which the apparent magnitude is smaller than $25.21$.

```python
def time_nUA_to_peri(dist,neo):
    '''
    Parameters
    ----------
    neo : a dataframe containing keplerian parameters of neo(s)
    dist [UA] : dist from Sun

    Returns
    -------
    Compute time to go from dist UA to perihelion
    '''
    theta = compute_theta_from_dist(dist,neo)
    a = neo['a']*1.496*10**(11)
    e = neo['e']

    b = compute_b_from_ae(neo)*1.496*10**(11)

    if isinstance(a,float):
        area, err = quad(lambda x : r2(a,b,e,x), np.pi, theta)

    else:

        area = np.zeros(len(neo['a']))
        for i in range(len(neo['a'])):

            area[i], err = quad(lambda x : r2(a[i],b[i],e[i],x), np.pi, theta[i])

    C = arealaw_constant(neo)
    period = compute_period_fromtraj(neo)
    airetot = np.pi*a*b

    return period - 2/C*(airetot - np.abs(area))
```

```python
def dist_maxmin_nUAsun_kUAobs(distSUN,print_dist=False):
    '''
    We want to keep a 90 degree angle with the Sun, so if the asteroid is at
    nUA from the Sun and is visible we can deduce the asteroid max distance to observer
    and asteroid min distance to the observer
    '''

    dist_min = distSUN - 0.7
    dist_max = np.sqrt(distSUN**2-0.7**2)
    if print_dist :
        print('Distance max to observer if {} to SUN : '.format(distSUN),dist_max)
        print('Distance min to observer if {} to SUN : '.format(distSUN),dist_min)
    return dist_min,dist_max

```

```python
def time_inside_visible_area_astero(neo,magnitude_lim,case='best'):
    '''
    Parameters
    ----------
    neo : a dataframe containing keplerian parameters of neo(s)
    magnitude_lim : limite magnitude detectable by satellite

    Returns
    -------
    Compute time inside visible area, different for each asteroide depending on size and keplerian parameters
    '''
    d_SUN = np.linspace(0.7,8,3000)
    s = 0
    visible_time = []
    for k in range(len(neo['a'])):

        neok = neo.iloc[k]
        H = neok['H']
        visible_dist = []
        visible_dist_nb = 0
        for i in range(len(d_SUN)):

            # Compute visible distance
            d_SUNi = d_SUN[i]
            d_min , d_max = dist_maxmin_nUAsun_kUAobs(d_SUNi)

            if case == 'best':
                d_to_obs = d_min
            if case == 'worst':
                d_to_obs = d_max
            if case == 'average':
                d_to_obs = np.mean([d_min,d_max])

            magapp = abs_mag_to_mag(H, d_SUNi, d_to_obs)

            if ( magapp < magnitude_lim + 0.01 and magapp > magnitude_lim - 0.01):
                visible_dist.append( d_SUNi )

        if len(visible_dist) > 0 :
            visible_dist_nb = np.max(visible_dist)

        else :
            print('## Either the asteroid is never visible ##')
            print('/!\ Or the d spacing must be changed, or magnitude range comparision')
            s = s+1

        if visible_dist_nb != 0 :
            if visible_dist_nb > neok['a']*(1+neok['e']):
                a=0 # if visible distance > apogee
                #visible_time.append(compute_period_fromtraj(neok)/(60*60*24))
            else :
                if abs_mag_to_D(H,0.03)<100:
                    a=0
                else :
                    # 2*(Compute time to go from visible distance to perihelion)
                    visible_time.append(2*time_nUA_to_peri(visible_dist_nb,neok)/(60*60*24))

    print(s)
    return visible_time
```

```python
case = 'best'
lim_mag = 25.21
visible_times = time_inside_visible_area_astero(neos1000,lim_mag,case)
plt.hist(visible_times,bins=80,range=[0,600],facecolor='blue', alpha=0.5)
plt.xlabel('Visible time distribution : m ={}, case={} '.format(lim_mag,case))
plt.ylabel('Frequency')
plt.title('Time ( Days )')
plt.show()

```

```python
case = 'worst'
lim_mag = 25.21
visible_times = time_inside_visible_area_astero(neos1000,lim_mag,case)
plt.hist(visible_times,bins=80,range=[0,600],facecolor='blue', alpha=0.5)
plt.xlabel('Visible time distribution : m ={}, case={} '.format(lim_mag,case))
plt.ylabel('Frequency')
plt.title('Time ( Days )')
plt.show()

```

```python
case = 'average'
lim_mag = 25.21
visible_times = time_inside_visible_area_astero(neos1000,lim_mag,case)
plt.hist(visible_times,bins=80,range=[0,600],facecolor='blue', alpha=0.5)
plt.xlabel('Visible time distribution : m ={}, case={} '.format(lim_mag,case))
plt.ylabel('Frequency')
plt.title('Time ( Days )')
plt.show()

```

This visible time has to be compared with the satellite period 224.7days. The best case would be that visility time for asteroid is longer than satellite period ( 224.7days ) in order to be able to be sure to detect the asteroid 1 time at least in a period. But this is not the case that is why this study is completed by propagating orbits on CelestLab ( CNES ) in order to determine more precisely the probability to detect an asteroid with this strategy.

#### If all the 1000ateroids where 100m diameter

In order to complete the study we will assume that we are always in the worst case :
- all the asteroid are 100m diameter and $p_v = 0.03$ ( worst case ) and $p_v=0.15$ ( best case )

```python

def time_inside_visible_area_100m_astero(neo,magnitude_lim):
    '''   /!\ Similar to previous function but here we assume each asteroid is 100m diameter ( in order to consider a worst case )
    Parameters
    ----------
    neo : a dataframe containing keplerian parameters of neo(s)
    magnitude_lim : limite magnitude detectable by satellite

    Returns
    -------
    Compute time inside visible area, different for each asteroide depending keplerian parameters
    '''
    d_SUN = np.linspace(0.1,8,3000)
    dist = []
    s = 0
    visible_time_min = []
    visible_time_max = []

    pv_max = 0.15
    pv_min = 0.03
    D = 0.100
    H_min = D_to_abs_mag(D,pv_min)
    H_max = D_to_abs_mag(D,pv_max)

    for k in range(len(neo['a'])):

        neok = neo.iloc[k]

        visible_dist_min = []
        visible_dist_min_nb = 0

        visible_dist_max = []
        visible_dist_max_nb = 0

        for i in range(len(d_SUN)):
            d_SUNi = d_SUN[i]
            d_min , d_max = dist_maxmin_nUAsun_kUAobs(d_SUNi)

            magapp_min = abs_mag_to_mag(H_min, d_SUNi, d_min)

            magapp_max = abs_mag_to_mag(H_max, d_SUNi, d_min)

            if ( magapp_min < magnitude_lim + 0.01 and magapp_min > magnitude_lim - 0.01):
                visible_dist_min.append( d_SUNi )

            if ( magapp_max < magnitude_lim + 0.01 and magapp_max > magnitude_lim - 0.01):
                visible_dist_max.append( d_SUNi )

        if len(visible_dist_min) > 0 :
            visible_dist_min_nb = np.max(visible_dist_min)
            visible_dist_max_nb = np.max(visible_dist_max)

        else :
            print('## Either the asteroid is never visible ##')
            print('/!\ Or the d spacing must be changed, or magnitude range comparision')
            s = s+1

        if visible_dist_min_nb != 0 :
            if visible_dist_min_nb > neok['a']*(1+neok['e']):
                a=0
                #visible_time_min.append(compute_period_fromtraj(neok)/(60*60*24))
            else :
                visible_time_min.append(2*time_nUA_to_peri(visible_dist_min_nb,neok)/(60*60*24))

        if visible_dist_max_nb != 0 :
            if visible_dist_max_nb > neok['a']*(1+neok['e']):
                a=0
                #visible_time_max.append(compute_period_fromtraj(neok)/(60*60*24))
            else :
                visible_time_max.append(2*time_nUA_to_peri(visible_dist_max_nb,neok)/(60*60*24))

    print(s)
    return visible_time_min , visible_time_max

```

```python
lim_mag = 25.21

visible_times_100m_min,visible_times_100m_max = time_inside_visible_area_100m_astero(neos1000,lim_mag)

plt.hist(visible_times_100m_min,bins=80,range=[0,600],facecolor='blue', alpha=0.5)
plt.xlabel('Time (days)')
plt.ylabel('Frequency')
plt.title('Visible time distribution : m ={}, worst case pv=0.03 '.format(lim_mag))
plt.show()

```

```python

plt.hist(visible_times_100m_max,bins=80,range=[0,600],facecolor='blue', alpha=0.5)
plt.xlabel('Time (days)')
plt.ylabel('Frequency')
plt.title('Visible time distribution : m ={}, best case pv=0.15 '.format(lim_mag))
plt.show()

```

This approach has some limits, we can deduce here the detection probability of detecting a NEO or we cannot estimate how much before impact we can detect an asteroid. That is why we added to this study some orbit propagation on CelestLab.

## References

Asteroid size :
- *Multiple asteroid systems: Dimensions and thermal properties from Spitzer Space Telescope and ground-based observations / F. Marchis*

Detection Probability :
- *Survey Simulations of a new near-Earth asteroid detection systeme / A. Mainzer*
- *Detection Systems, Institut d'Optique Graduate School / Julien Moreau and Riad Haïdar*
- *Notes for PHYS 134: Observational Astrophysics / Bill Wolf and Phil Lubin*


```python
from IPython.display import Image
Image('../../Python_probability_statistics_machine_learning_2E.png',width=200)
```

Linear regression gets to the heart of
statistics: Given a set of data points,
what is the relationship of the data in-
hand to data yet seen?
How should information from one data set propagate to
other data? Linear
regression offers the following model to address this
question:

$$
\mathbb{E}(Y|X=x) \approx a x + b
$$

 That is, given specific values for $X$, assume that the conditional
expectation
is a linear function of those specific values.  However, because
the observed
values are not the expectations themselves, the model accommodates
this with an
additive noise term. In other words, the observed variable (a.k.a.
response,
target, dependent variable) is modeled as,

$$
\mathbb{E}(Y| X=x_i) + \epsilon_i \approx a x + b+ \epsilon_i=y
$$

 where $\mathbb{E}(\epsilon_i)=0$ and the $\epsilon_i$ are iid and
where the
distribution function of $\epsilon_i$ depends on the problem, even
though it is
often assumed Gaussian. The $X=x$ values are known as independent
variables,
covariates, or regressors.

Let's see if we can use all of the methods we have
developed so far to
understand this form of regression. The first task is to
determine how to
estimate the unknown linear parameters, $a$ and $b$. To make
this concrete,
let's assume that $\epsilon \sim \mathcal{N}(0,\sigma^2)$.  Bear
in mind that
$\mathbb{E}(Y|X=x)$ is a deterministic function of $x$.  In other
words, the
variable $x$ changes with each draw, but after the data have been
collected
these are no longer random quantities. Thus, for fixed $x$, $y$ is a
random
variable generated by $\epsilon$.  Perhaps we should denote $\epsilon$ as
$\epsilon_x$ to emphasize this, but because $\epsilon$ is an independent,
identically-distributed (iid) random variable at each fixed $x$, this would be
excessive.  Because of Gaussian additive noise, the
distribution of $y$ is
completely characterized by its mean and variance.

$$
\begin{align*}
\mathbb{E}(y) &= a x  + b  \\\
\mathbb{V}(y) &= \sigma^2
\end{align*}
$$

 Using the maximum likelihood procedure, we write
out the log-likelihood
function as

$$
\mathcal{L}(a,b)  = \sum_{i=1}^n \log \mathcal{N}(a x_i +b , \sigma^2)
\propto \frac{1}{2 \sigma^2}\sum_{i=1}^n (y_i-a x_i-b)^2
$$

 Note that we suppressed  the terms that are irrelevent to
the maximum-finding.
Taking the derivative of this with respect to $a$ gives
the following equation:

$$
\frac{\partial \mathcal{L}(a,b)}{\partial a}= 2 \sum_{i=1}^n x_i (b+ a x_i
-y_i) =0
$$

 Likewise, we do the same for the $b$ parameter

$$
\frac{\partial \mathcal{L}(a,b)}{\partial b}=2\sum_{i=1}^n (b+a x_i-y_i) =0
$$

 The following code simulates some data and uses Numpy tools to
compute the
parameters as shown,

```python
import numpy as np
a = 6;b = 1 # parameters to estimate
x = np.linspace(0,1,100)
y = a*x + np.random.randn(len(x))+b
p,var_=np.polyfit(x,y,1,cov=True) # fit data to line
y_ = np.polyval(p,x) # estimated by linear regression
```

```python
%matplotlib inline

from matplotlib.pylab import subplots
fig,axs=subplots(1,2)
fig.set_size_inches((9,3.5))
_=ax =axs[0]
_=ax.plot(x,y,'o',alpha=.3,color='gray',ms=10)
_=ax.plot(x,y_,color='k',lw=3)
_=ax.set_xlabel("$x$",fontsize=22)
_=ax.set_ylabel("$y$",fontsize=22)
_=ax.set_title("Linear regression; a=%3.3g b=%3.3g"%(p[0],p[1]))
_=ax = axs[1]
_=ax.hist(y_-y,color='gray')
_=ax.set_xlabel(r"$\Delta y$",fontsize=22)
fig.tight_layout()
fig.savefig('fig-statistics/Regression_001.png')
```

The panel on the left
shows the data and regression line. The panel on the right shows a histogram of
the regression errors.

 The graph on the left of
[Figure](#fig:Regression_001)
shows the regression line plotted against the
data. The estimated
parameters are noted in the title.  The histogram on the
right of
[Figure](#fig:Regression_001) shows the residual errors in the model.
It is always a good idea to inspect the residuals of any regression
for
normality. These are the differences between the fitted line for
each $x_i$
value and the corresponding $y_i$ value in the data.
Note that the $x$ term does
not have to be uniformly monotone.

To decouple the deterministic variation from
the random variation, we can fix
the index and write separate problems of the
form

$$
y_i = a x_i +b + \epsilon_i
$$

 where $\epsilon_i \sim \mathcal{N}(0,\sigma^2)$. What could we do with
just
this one component of the problem? In other words, suppose we had
$m$-samples of
this component as in $\lbrace y_{i,k}\rbrace_{k=1}^m$. Following
the usual
procedure, we could obtain estimates of the mean of $y_i$ as

$$
\hat{y_i} = \frac{1}{m}\sum_{k=1}^m y_{i,k}
$$

 However, this tells us nothing about the individual parameters $a$
and $b$
because they are not separable in the terms that are computed, namely,
we may
have

$$
\mathbb{E}(y_i) = a x_i +b
$$

 but we still only have one equation and the two unknowns, $a$ and
$b$. How
about if we consider and fix another component $j$ as in

$$
y_j = a x_j +b + \epsilon_i
$$

 Then, we have

$$
\mathbb{E}(y_j) = a x_j +b
$$

 so at least now we have two equations and two unknowns and we know how to
estimate the left hand sides of these equations from the data using the
estimators $\hat{y_i}$ and  $\hat{y_j}$. Let's see how this works in the code
sample below.

```python
x0, xn =x[0],x[80]
# generate synthetic data
y_0 = a*x0 + np.random.randn(20)+b
y_1 = a*xn + np.random.randn(20)+b
# mean along sample dimension
yhat = np.array([y_0,y_1]).mean(axis=1)
a_,b_=np.linalg.solve(np.array([[x0,1],
                                [xn,1]]),yhat)
```

The fitted and true lines are plotted with
the data values. The squares at either end of the solid line show the mean value
for each of the data groups shown.

**Programming
Tip.**

The prior code uses the `solve` function in the Numpy `linalg` module,
which
contains the core linear algebra codes in Numpy that incorporate the
battle-tested LAPACK library.

 We can write out the solution for the
estimated parameters for this
case where $x_0 =0$

$$
\begin{align*}
\hat{a} &= \frac{\hat{y}_i - \hat{y}_0}{x_i} \\\
\hat{b} &=
\hat{y_0} \\\
\end{align*}
$$

 The expectations and variances of these estimators are the following,

$$
\begin{align*}
\mathbb{E}(\hat{a}) &= \frac{a x_i }{x_i}=a  \\\
\mathbb{E}(\hat{b}) &=b \\\
\mathbb{V}(\hat{a}) &= \frac{2 \sigma^2}{x_i^2} \\\
\mathbb{V}(\hat{b}) &= \sigma^2
\end{align*}
$$

 The expectations show that the estimators are unbiased.  The
estimator
$\hat{a}$ has a variance that decreases as larger points $x_i$ are
selected.
That is, it is better to have samples further out along the
horizontal axis for
fitting the line.  This variance quantifies the *leverage*
of those distant
points.

**Regression From Projection Methods.** Let's see if we can apply our
knowledge
of projection methods to the general case.  In vector notation, we can
write
the following:

$$
\mathbf{y} = a \mathbf{x} + b\mathbf{1} + \boldsymbol{\epsilon}
$$

 where $\mathbf{1}$ is the vector of all ones.
Let's use the inner-product
notation,

$$
\langle \mathbf{x},\mathbf{y} \rangle = \mathbb{E}(\mathbf{x}^T \mathbf{y})
$$

 Then, by taking the inner-product with some $\mathbf{x}_1 \in
\mathbf{1}^\perp$
we obtain [^perp],

[^perp]: The space of all vectors, $\mathbf{a}$ such that
$\langle
\mathbf{a},\mathbf{1} \rangle = 0$ is denoted $\mathbf{1}^\perp$.

$$
\langle \mathbf{y},\mathbf{x}_1  \rangle = a \langle \mathbf{x},\mathbf{x}_1
\rangle
$$

 Recall that $\mathbb{E}(\boldsymbol{\epsilon})=\mathbf{0}$.  We
can finally
solve for $a$ as

$$
\begin{equation}
\hat{a} = \frac{\langle\mathbf{y},\mathbf{x}_1 \rangle}{\langle
\mathbf{x},\mathbf{x}_1 \rangle}
\end{equation}
\label{eq:ahat} \tag{1}
$$

 That was pretty neat but now we have the mysterious $\mathbf{x}_1$
vector.
Where does this come from?  If we project $\mathbf{x}$ onto the
$\mathbf{1}^\perp$, then we get the MMSE  approximation to $\mathbf{x}$ in the
$\mathbf{1}^\perp$ space. Thus, we take

$$
\mathbf{x}_1 = P_{\mathbf{1}^\perp} (\mathbf{x})
$$

 Remember that $P_{\mathbf{1}^\perp} $ is a projection matrix so the length of
$\mathbf{x}_1$ is at most $\mathbf{x}$. This means that the denominator in
the
$\hat{a}$ equation above is really just the length of the $\mathbf{x}$
vector in
the coordinate system of $P_{\mathbf{1}^\perp} $. Because the
projection is
orthogonal (namely, of minimum length), the Pythagorean theorem
gives this
length as the following:

$$
\langle \mathbf{x},\mathbf{x}_1 \rangle ^2=\langle \mathbf{x},\mathbf{x}
\rangle- \langle\mathbf{1},\mathbf{x} \rangle^2
$$

 The first term on the right is the length of the $\mathbf{x}$ vector
and last
term is the length of $\mathbf{x}$ in the coordinate system orthogonal
to
$P_{\mathbf{1}^\perp} $, namely that of $\mathbf{1}$.  We
can use this geometric
interpretation to understand what is going on in
typical linear regression in
much more detail. The fact that the denominator is
the orthogonal projection of
$\mathbf{x}$ tells us that the choice of
$\mathbf{x}_1$ has the strongest effect
(i.e., largest value) on reducing
the variance of $\hat{a}$. That is, the more
$\mathbf{x}$ is aligned with
$\mathbf{1}$, the worse the variance of $\hat{a}$.
This makes intuitive sense
because the closer $\mathbf{x}$ is to $\mathbf{1}$,
the more constant it is,
and we have already seen from our one-dimensional
example that distance between
the $x$ terms pays off in reduced variance. We
already know that $\hat{a}$
is an unbiased estimator, and, because we chose
$\mathbf{x}_1$ deliberately as a
projection, we know that it is also of minimum
variance. Such estimators are
known as Minimum-Variance Unbiased Estimators
(MVUE).

In the same spirit, let's examine the numerator of $\hat{a}$ in
Equation [1](#eq:ahat). We can write
$\mathbf{x}_{1}$ as the following

$$
\mathbf{x}_{1} = \mathbf{x} -  P_{\mathbf{1}} \mathbf{x}
$$

 where $P_{\mathbf{1}}$ is projection matrix  of $\mathbf{x}$ onto the
$\mathbf{1}$ vector. Using this, the numerator of $\hat{a}$ becomes

$$
\langle \mathbf{y}, \mathbf{x}_1\rangle  =\langle \mathbf{y},
\mathbf{x}\rangle -\langle \mathbf{y}, P_{\mathbf{1}} \mathbf{x}\rangle
$$

 Note that,

$$
P_{\mathbf{1}}  = \mathbf{1} \mathbf{1}^T \frac{1}{n}
$$

 so that writing this out explicitly gives

$$
\langle \mathbf{y}, P_{\mathbf{1}} \mathbf{x}\rangle = \left(\mathbf{y}^T
\mathbf{1}\right) \left(\mathbf{1}^T \mathbf{x}\right)/n =\left(\sum
y_i\right)\left(\sum x_{i}\right)/n
$$

 and similarly, we have the following for the denominator:

$$
\langle \mathbf{x}, P_{\mathbf{1}} \mathbf{x}\rangle = \left(\mathbf{x}^T
\mathbf{1}\right) \left(\mathbf{1}^T \mathbf{x}\right)/n =\left(\sum
x_i\right)\left(\sum x_{i}\right)/n
$$

 So, plugging all of this together gives the following,

$$
\hat{a} = \frac{\mathbf{x}^T\mathbf{y}-(\sum x_i)(\sum y_i)/n}{\mathbf{x}^T
\mathbf{x}  -(\sum x_i)^2/n}
$$

 with corresponding variance,

$$
\begin{align*}
\mathbb{V}(\hat{a}) &= \sigma^2
\frac{\|\mathbf{x}_1\|^2}{\langle\mathbf{x},\mathbf{x}_1\rangle^2} \\\
&= \frac{\sigma^2}{\Vert \mathbf{x}\Vert^2-n(\overline{x}^2)}
\end{align*}
$$

 Using the same approach with $\hat{b}$ gives,

$$
\begin{equation}
\hat{b}  = \frac{\langle \mathbf{y},\mathbf{x}^{\perp}
\rangle}{\langle \mathbf{1},\mathbf{x}^{\perp}\rangle}
\label{_auto1} \tag{2}
\end{equation}
$$

$$
\begin{equation} \
         = \frac{\langle
\mathbf{y},\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})\rangle}{\langle
\mathbf{1},\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})\rangle}
\label{_auto2} \tag{3}
\end{equation}
$$

$$
\begin{equation} \
         = \frac{\mathbf{x}^T \mathbf{x}(\sum y_i)/n
-\mathbf{x}^T\mathbf{y}(\sum x_i)/n}{\mathbf{x}^T \mathbf{x} -(\sum x_i)^2/n}
\label{_auto3} \tag{4}
\end{equation}
$$

 where

$$
P_{\mathbf{x}} = \frac{\mathbf{\mathbf{x} \mathbf{x}^T}}{\| \mathbf{x} \|^2}
$$

 with variance

$$
\begin{align*}
\mathbb{V}(\hat{b})&=\sigma^2 \frac{\langle
\boldsymbol{\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})},\boldsymbol{\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})}\rangle}{\langle
\mathbf{1},\boldsymbol{\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})}\rangle^2} \\\
&=\frac{\sigma^2}{n-\frac{(n\overline{x})^2}{\Vert\mathbf{x}\Vert^2}}
\end{align*}
$$

**Qualifying the Estimates.** Our formulas for the variance above include the
unknown $\sigma^2$, which we must estimate from the data itself using our
plug-
in estimates.  We can form the residual sum of squares as

$$
\texttt{RSS} = \sum_i (\hat{a} x_i + \hat{b} - y_i)^2
$$

 Thus, the estimate of $\sigma^2$ can be expressed as

$$
\hat{\sigma}^2 = \frac{\texttt{RSS}}{n-2}
$$

 where $n$ is the number of samples. This is also known as the
*residual mean
square*. The $n-2$  represents the *degrees of freedom* (`df`).
Because we
estimated two parameters from the same data we have $n-2$ instead of
$n$. Thus,
in general, $\texttt{df} = n - p$, where $p$ is the number of
estimated
parameters.  Under the assumption that the noise is Gaussian,  the
$\texttt{RSS}/\sigma^2$ is chi-squared distributed with $n-2$ degrees of
freedom. Another important term is the *sum of squares about the mean*, (a.k.a
*corrected* sum of squares),

$$
\texttt{SYY} = \sum (y_i - \bar{y})^2
$$

 The $\texttt{SYY}$ captures the idea of not using the $x_i$ data and
just using
the mean of the $y_i$ data to estimate $y$. These two terms lead
to the $R^2$
term,

$$
R^2=1-\frac{\texttt{RSS}}{ \texttt{SYY} }
$$

 Note that for perfect regression, $R^2=1$.  That is, if the
regression gets
each $y_i$ data point exactly right, then
$\texttt{RSS}=0$ this term equals one.
Thus, this term is used to
measure of goodness-of-fit. The `stats` module in
`scipy` computes
many of these terms automatically,

```python
from scipy import stats
slope,intercept,r_value,p_value,stderr = stats.linregress(x,y)
```

where the square of the `r_value` variable is the $R^2$ above. The
computed
p-value is the two-sided hypothesis test with a null hypothesis that
the slope
of the line is zero. In other words, this tests whether or not the
linear
regression  makes sense for the data for that hypothesis.  The
Statsmodels
module provides a powerful extension to Scipy's stats module by
making it easy
to do regression and keep track of these parameters. Let's
reformulate our
problem using the Statsmodels framework by creating
a Pandas dataframe for the
data,

```python
import statsmodels.formula.api as smf
from pandas import DataFrame
import numpy as np
d = DataFrame({'x':np.linspace(0,1,10)}) # create data
d['y'] = a*d.x+ b + np.random.randn(*d.x.shape)
```

 Now that we have  the input data in the above
Pandas dataframe, we
can perform the regression as in the following,

```python
results = smf.ols('y ~ x', data=d).fit()
```

The $\sim$ symbol is notation for $y = a x + b + \epsilon$, where the
constant
$b$ is implicit in this usage of Statsmodels. The names in the string
are taken
from the columns in the dataframe. This makes it very easy to build
models with
complicated interactions between the named columns in the
dataframe. We can
examine a report of the model fit by looking at the summary,

```python
print (results.summary2())
```

                     Results: Ordinary least squares
    =================================================================
    Model:              OLS              Adj. R-squared:     0.780
    Dependent Variable: y                AIC:                32.6081
    Date:               2019-08-27 12:16 BIC:                33.2132
    No. Observations:   10               Log-Likelihood:     -14.304
    Df Model:           1                F-statistic:        32.89
    Df Residuals:       8                Prob (F-statistic): 0.000436
    R-squared:          0.804            Scale:              1.2790
    -------------------------------------------------------------------
                 Coef.    Std.Err.     t      P>|t|     [0.025   0.975]
    -------------------------------------------------------------------
    Intercept    1.4044     0.6647   2.1128   0.0676   -0.1284   2.9372
    x            6.4271     1.1206   5.7354   0.0004    3.8430   9.0112
    -----------------------------------------------------------------
    Omnibus:              2.415        Durbin-Watson:           2.784
    Prob(Omnibus):        0.299        Jarque-Bera (JB):        0.945
    Skew:                 0.224        Prob(JB):                0.623
    Kurtosis:             1.562        Condition No.:           4
    =================================================================

    /home/unpingco/.conda/envs/pypsml2E/lib/python3.7/site-packages/scipy/stats/stats.py:1416: UserWarning: kurtosistest only valid for n>=20 ... continuing anyway, n=10
      "anyway, n=%i" % int(n))

```

Results: Ordinary least squares
=================================================================
Model:              OLS              Adj. R-squared:     0.808
Dependent Variable: y                AIC:                28.1821
Date:               0000-00-00 00:00 BIC:                00.0000
No. Observations:   10               Log-Likelihood:     -12.091
Df Model:           1                F-statistic:        38.86
Df Residuals:       8                Prob (F-statistic): 0.000250
R-squared:          0.829            Scale:              0.82158
-------------------------------------------------------------------
              Coef.    Std.Err.     t      P>|t|    [0.025   0.975]
-------------------------------------------------------------------
Intercept     1.5352     0.5327   2.8817   0.0205   0.3067   2.7637
x             5.5990     0.8981   6.2340   0.0003   3.5279   7.6701

There is a lot more here than we have discussed so far, but the
Statsmodels
documentation is the best place to go for complete information
about this
report. The F-statistic attempts to capture the contrast between
including the
slope parameter or leaving it off. That is, consider two
hypotheses:
```

$$
\begin{align*}
H_0 \colon  \mathbb{E}(Y|X=x) &= b \\\
H_1 \colon
\mathbb{E}(Y|X=x) &= b + a x
\end{align*}
$$

 In order to quantify how much better adding the slope term is for
the
regression, we compute  the following:

$$
F = \frac{\texttt{SYY} - \texttt{RSS}}{ \hat{\sigma}^2 }
$$

 The numerator computes the difference in the residual squared errors
between
including the slope in the regression or just using the mean of the
$y_i$
values. Once again, if we assume (or can claim asymptotically) that the
$\epsilon$ noise term is Gaussian, $\epsilon \sim \mathcal{N}(0,\sigma^2)$,
then
the $H_0$ hypothesis will follow an F-distribution [^fdist] with degrees of
freedom from the numerator and denominator. In this case, $F \sim F(1,n-2)$.
The
value of this statistic is reported by Statsmodels above. The corresponding
reported probability shows the chance of $F$ exceeding its computed value  if
$H_0$ were true. So, the take-home message from all this is that including the
slope leads to a much smaller reduction in squared error than could be expected
from a favorable draw of $n$ points of this data, under the Gaussian additive
noise assumption. This is evidence that including the slope is meaningful for
this data.

[^fdist]: The $F(m,n)$ F-distribution has two integer degree-of-
freedom parameters, $m$
and $n$.

The Statsmodels report also shows the
adjusted $R^2$ term.
This is a correction to the $R^2$ calculation that accounts
for the number of parameters $p$ that the regression is
fitting and the sample
size $n$,

$$
\texttt{Adjusted } R^2 = 1- \frac{\texttt{RSS}/(n-p)}{\texttt{SYY}/(n-1)}
$$

 This is always lower than $R^2$ except when $p=1$ (i.e., estimating
only $b$).
This becomes a better way to compare regressions when one is
attempting to fit
many parameters with comparatively small $n$.

**Linear Prediction.**  Using
linear regression for prediction introduces
some other issues. Recall the
following expectation,

$$
\mathbb{E}(Y|X=x) \approx \hat{a} x + \hat{b}
$$

 where we have determined $\hat{a}$  and $\hat{b}$  from the data.
Given a new
point of interest, $x_p$, we would certainly compute

$$
\hat{y}_p = \hat{a} x_p + \hat{b}
$$

 as the predicted value for $\hat{ y_p }$. This is the same as saying
that our
best prediction for $y$ based on $x_p$ is the above conditional
expectation. The
variance for this is the following,

$$
\mathbb{V}(y_p) = x_p^2 \mathbb{V}(\hat{a}) +\mathbb{V}(\hat{b})+2 x_p
\texttt{cov}(\hat{a}\hat{b})
$$

 Note that we have the covariance above because $\hat{a}$ and
$\hat{b}$ are
derived from the same data.  We can work this out below using
our previous
notation from [1](#eq:ahat),

$$
\begin{align*}
\texttt{cov}(\hat{a}\hat{b})=&\frac{\mathbf{x}_1^T
\mathbb{V}\lbrace\mathbf{y}\mathbf{y}^T\rbrace\mathbf{x}^{\perp}}{(\mathbf{x}_1^T
\mathbf{x})(\mathbf{1}^T \mathbf{x}^{\perp})} = \frac{\mathbf{x}_1^T
\sigma^2\mathbf{I}\mathbf{x}^{\perp}}{(\mathbf{x}_1^T \mathbf{x})(\mathbf{1}^T
\mathbf{x}^{\perp})}\\\
=&\sigma^2\frac{\mathbf{x}_1^T\mathbf{x}^{\perp}}{(\mathbf{x}_1^T
\mathbf{x})(\mathbf{1}^T \mathbf{x}^{\perp})} =
\sigma^2\frac{\left(\mathbf{x}-P_1\mathbf{x}\right)^T\mathbf{x}^{\perp}}{(\mathbf{x}_1^T
\mathbf{x})(\mathbf{1}^T \mathbf{x}^{\perp})}\\\
=&\sigma^2\frac{-\mathbf{x}^T P_1^T\mathbf{x}^{\perp}}{(\mathbf{x}_1^T
\mathbf{x})(\mathbf{1}^T \mathbf{x}^{\perp})} =
\sigma^2\frac{-\mathbf{x}^T\frac{1}{n}\mathbf{1}
\mathbf{1}^T\mathbf{x}^{\perp}}{(\mathbf{x}_1^T \mathbf{x})(\mathbf{1}^T
\mathbf{x}^{\perp})}\\\
=&\sigma^2\frac{-\mathbf{x}^T\frac{1}{n}\mathbf{1}}{(\mathbf{x}_1^T \mathbf{x})}
= \frac{-\sigma^2\overline{x}}{\sum_{i=1}^n(x_i^2-\overline{x}^2)}\\\
\end{align*}
$$

 After plugging all this in, we obtain the following,

$$
\mathbb{V}(y_p)=\sigma^2 \frac{x_p^2-2 x_p\overline{x}+\Vert
\mathbf{x}\Vert^2/n}{\Vert\mathbf{x}\Vert^2-n\overline{x}^2}
$$

 where, in practice, we use the plug-in estimate for
the  $\sigma^2$.

There is
an important consequence for the confidence interval for
$y_p$. We cannot simply
use the square root of $\mathbb{V}(y_p)$
to form the confidence interval because
the model includes the
extra $\epsilon$ noise term. In particular, the
parameters were
computed using a set of statistics from the data, but now must
include different realizations for the noise term for the
prediction part.  This
means we have to compute

$$
\eta^2=\mathbb{V}(y_p)+\sigma^2
$$

 Then, the 95\% confidence interval $y_p \in
(y_p-2\hat{\eta},y_p+2\hat{\eta})$
is the following,

$$
\mathbb{P}(y_p-2\hat{\eta}< y_p <
y_p+2\hat{\eta})\approx\mathbb{P}(-2<\mathcal{N}(0,1)<2) \approx 0.95
$$

 where $\hat{\eta}$ comes from substituting the
plug-in estimate for $\sigma$.
## Extensions to Multiple Covariates

With all the machinery we have, it is a
short notational hop to consider
multiple regressors as in the following,

$$
\mathbf{Y} = \mathbf{X} \boldsymbol{\beta} +\boldsymbol{\epsilon}
$$

 with the usual $\mathbb{E}(\boldsymbol{\epsilon})=\mathbf{0}$ and
$\mathbb{V}(\boldsymbol{\epsilon})=\sigma^2\mathbf{I}$. Thus, $\mathbf{X}$ is a
$n \times p$ full rank matrix of regressors and $\mathbf{Y}$ is the $n$-vector
of observations. Note that the constant term has been incorporated into
$\mathbf{X}$ as a column of ones.  The corresponding estimated solution for
$\boldsymbol{\beta}$  is the following,

$$
\hat{ \boldsymbol{\beta} } = (\mathbf{X}^T \mathbf{X})^{-1} \mathbf{X}^T
\mathbf{Y}
$$

 with corresponding variance,

$$
\mathbb{V}(\hat{\boldsymbol{\beta}})=\sigma^2(\mathbf{X}^T \mathbf{X})^{-1}
$$

 and with the assumption of Gaussian errors, we have

$$
\hat{\boldsymbol{\beta}}\sim \mathcal{N}(\boldsymbol{\beta},
\sigma^2(\mathbf{X}^T \mathbf{X})^{-1})
$$

 The unbiased estimate of $\sigma^2$ is the following,

$$
\hat{\sigma}^2 = \frac{1}{n-p}\sum \hat{\epsilon}_i^2
$$

 where $\hat{ \boldsymbol{\epsilon}}=\mathbf{X}\hat{\boldsymbol{\beta}}
-\mathbf{Y}$
is the vector of residuals. Tukey christened the following matrix
as the *hat*
matrix (a.k.a. influence matrix),

$$
\mathbf{V}=\mathbf{X}(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T
$$

 because it maps $\mathbf{Y}$ into $\hat{ \mathbf{Y} }$,

$$
\hat{ \mathbf{Y} } = \mathbf{V} \mathbf{Y}
$$

 As an exercise you can check that $\mathbf{V}$ is a projection
matrix. Note
that that matrix is solely a function of $\mathbf{X}$. The
diagonal elements of
$\mathbf{V}$ are called the *leverage values* and are
contained in the closed
interval $[1/n,1]$. These terms measure of distance
between the values of $x_i$
and the mean values over the $n$ observations.
Thus, the leverage terms depend
only on $\mathbf{X}$. This is the
generalization of our initial discussion of
leverage where we had multiple
samples at only two $x_i$ points. Using the hat
matrix, we can compute the
variance of each residual, $e_i = \hat{y}-y_i$ as

$$
\mathbb{V}(e_i) = \sigma^2 (1-v_{i})
$$

 where $v_i=V_{i,i}$. Given the above-mentioned bounds on $v_{i}$,
these are
always less than $\sigma^2$.

Degeneracy in the columns of $\mathbf{X}$ can
become a problem. This is when
two or more of the columns become co-linear. We
have already seen this with our
single regressor example wherein $\mathbf{x}$
close to $\mathbf{1}$ was bad
news. To compensate for this effect we can load
the diagonal elements and solve
for the unknown parameters as in the following,

$$
\hat{ \boldsymbol{\beta} } = (\mathbf{X}^T \mathbf{X}+\alpha \mathbf{I})^{-1}
\mathbf{X}^T \mathbf{Y}
$$

 where $\alpha>0$ is a tunable hyper-parameter. This method is known
as *ridge
regression* and was proposed in 1970 by Hoerl and Kenndard. It can be
shown that
this is the equivalent to minimizing the following objective,

$$
\Vert \mathbf{Y}- \mathbf{X} \boldsymbol{\beta}\Vert^2  + \alpha \Vert
\boldsymbol{\beta}\Vert^2
$$

 In other words, the length of the estimated $\boldsymbol{\beta}$ is
penalized
with larger $\alpha$. This has the effect of stabilizing the
subsequent inverse
calculation and also providing a means to trade bias and
variance, which we will
discuss at length in the section
[ch:ml:sec:regularization](#ch:ml:sec:regularization)

**Interpreting
Residuals.**  Our model assumes an additive Gaussian noise term.
We can check
the voracity of this assumption by examining the residuals after
fitting.  The
residuals are the difference between the fitted values and the
original data

$$
\hat{\epsilon}_i = \hat{a} x_i + \hat{b} - y_i
$$

 While the p-value and the F-ratio provide some indication of whether
or not
computing the slope of the regression makes sense, we can get directly
at the
key assumption of additive Gaussian noise.

For sufficiently small dimensions,
the `scipy.stats.probplot` we discussed in
the last chapter provides quick
visual evidence one way or another by plotting
the standardized residuals,

$$
r_i = \frac{e_i}{\hat{\sigma}\sqrt{1-v_i}}
$$

 The other part of the iid assumption implies homoscedasticity (all
$r_i$ have
equal variances). Under the additive Gaussian noise assumption, the
$e_i$ should
also be distributed according to $\mathcal{N}(0,\sigma^2(1-v_i))$.
The
normalized residuals $r_i$ should then be distributed according to
$\mathcal{N}(0,1)$. Thus, the presence of any $r_i \notin [-1.96,1.96]$ should
not be common at the 5% significance level and is thereby breeds suspicion
regarding the homoscedasticity assumption.

The Levene test in
`scipy.stats.leven` tests the null hypothesis that all the
variances are equal.
This basically checks whether or not the standardized
residuals vary across
$x_i$ more than expected. Under the homoscedasticity
assumption, the variance
should be independent of $x_i$. If not, then this is a
clue that there is a
missing variable in the analysis or that the variables
themselves should be
transformed (e.g., using the $\log$ function) into another
format that can
reduce this effect. Also, we can use weighted least-squares
instead of ordinary
least-squares.

**Variable Scaling.** It is tempting to conclude in a multiple
regression that small coefficients in any of the $\boldsymbol{\beta}$ terms
implies that those terms are not important.  However, simple unit conversions
can cause this effect. For example, if one of the regressors is in
units of
kilometers and the others are in meters, then just the scale
factor can give the
impression of outsized or under-sized effects. The
common way to account for
this is to scale the regressors so that

$$
x^\prime = \frac{x-\bar{x}}{\sigma_x}
$$

 This has the side effect of converting the slope parameters
into correlation
coefficients, which is bounded by $\pm 1$.

**Influential Data.** We have
already discussed the idea
of leverage. The concept of *influence* combines
leverage with
outliers. To understand influence, consider
[Figure](#fig:Regression_005).

The point on the right has
outsized influence in this data because it is the only one used to determine the
slope of the fitted line.

The point on the right in
[Figure](#fig:Regression_005) is the only one that
contributes to the
calculation of the slope for the fitted line. Thus, it
is very influential in
this sense. Cook's distance is a good way to get at this
concept numerically.
To compute this, we have to compute the $j^{th}$
component of the estimated
target variable with the $i^{th}$ point deleted. We
call this $\hat{y}_{j(i)}$.
Then, we compute the following,

$$
D_i =\frac{\sum_j (\hat{y}_j- \hat{y}_{j(i)})^2}{p/n \sum_j (\hat{y}_j-
y_j)^2}
$$

 where, as before, $p$ is the number of estimated terms
(e.g., $p=2$ in the
bivariate case). This calculation emphasizes the
effect of the outlier by
predicting the target variable with and
without each point. In the case of
[Figure](#fig:Regression_005),
losing any of the points on the left cannot
change the estimated
target variable much, but losing the single point on the
right surely
does. The point on the right does not seem to be an outlier (it
*is*
on the fitted line), but this is because it is influential enough to
rotate
the line to align with it. Cook's distance helps capture this
effect by leaving
each sample out and re-fitting the remainder as
shown in the last equation.
[Figure](#fig:Regression_006) shows the
calculated Cook's distance for the data
in [Figure](#fig:Regression_005), showing that the data point on the right
(sample index `5`) has outsized influence on the fitted line. As a
rule of
thumb, Cook's distance values that  are larger than one are
suspect.

The calculated Cook's distance for the data
in [Figure](#fig:Regression_005).

As another
illustration of influence, consider [Figure](#fig:Regression_007) which shows
some data that nicely line up, but
with one outlier (filled black circle) in the
upper panel.  The lower
panel shows so-computed Cook's distance for this data
and emphasizes
the presence of the outlier.  Because the calculation involves
leaving
a single sample out and re-calculating the rest, it can be a
time-
consuming operation suitable to relatively small data sets. There
is always the
temptation to downplay the importance of outliers
because they conflict with a
favored model, but outliers must be
carefully examined to understand why the
model is unable to capture
them. It could be something as simple as faulty data
collection, or it
could be an indication of deeper issues that have been
overlooked. The
following code shows how Cook's distance was compute for
[Figure](#fig:Regression_006) and [Figure](#fig:Regression_007).

```python
fit = lambda i,x,y: np.polyval(np.polyfit(x,y,1),i)
omit = lambda i,x: ([k for j,k in enumerate(x) if j !=i])
def cook_d(k):
    num = sum((fit(j,omit(k,x),omit(k,y))-fit(j,x,y))**2 for j in x)
    den = sum((y-np.polyval(np.polyfit(x,y,1),x))**2/len(x)*2)
    return num/den
```

**Programming Tip.**

The function `omit` sweeps through the data and excludes
the $i^{th}$ data element. The embedded `enumerate` function
associates every
element in the iterable with its corresponding
index.

The upper panel shows data that fit on a line
and an outlier point (filled black circle). The lower panel shows the calculated
Cook's distance for the data in upper panel and shows that the tenth point
(i.e., the outlier) has disproportionate influence.


# Local approximation of a bivariate quadratic expression using linear regression using Keras

Given two features, extract y (the label), perform linear regression
Using the same model as in the main [bivariate linear regression notebook](LinearRegressionBivariate.ipynb)

### Learning goals:
- Learn about Keras
- Port the linear regression gradient descent to Keras
- Get to know about the Kullback-Leibler Divergence metric
- Go further with a two layer neural network

```python
import tensorflow as tf # TF 2.0 required
from tensorflow import keras # TF 2.0 required
import numpy as np
from numpy import random
import matplotlib.pyplot as plt
from sklearn import metrics
import pandas
```

```python
usingTensorBoard = False
```

In 2D, it would require many points to uniformly cover the x plan

It is then prefered to draw x points from a 2D uniform distribution (Monte Carlo experiment)

Ploting y as function of $x_0, x_1$ is then more challenging as $x_0$ and $x_1$ are not continuous monotonic vectors. Scatter plot is favored.

$$\begin{align}
f(x) &= (x_0−0.2)^4 + (x_0−0.1)^3 + 0.1 x_1^2 + 0.35 \\
&= x_0^4 + 0.2 x_0^3 − 0.06 x_0^2 − 0.002 x_0 + 0.1 x_1^2 + 0.3506
\end{align}$$

```python
nFeatures = 2
# f(x) as a bivariate polynom
fPoly = np.array([[-0.002, 0], [-0.06, 0.1], [0.2, 0], [1, 0]])
# Generator
def generateBatch(N):
    #
    xMin = np.array([0, -0.5])
    xMax = np.array([0.5, 0.5])
    #
    b = 0.35
    std = 0.01
    #
    x = random.uniform(xMin, xMax, (N, nFeatures))
    yClean = (x[:,0]-0.2)**4 + (x[:,0]-0.1)**3 + 0.1*x[:,1]**2 + b
    y =  yClean + random.normal(0, std, N)
    return (x, y, yClean)
```

```python
N = 100000
xTrain, yTrain, yTrainClean = generateBatch(N);

xTest, yTest, yTestClean = generateBatch(N);
```

```python
fig = plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.scatter(xTrain[:,0], yTrainClean, marker='.');
plt.xlabel('x0')
plt.subplot(1,2,2)
plt.scatter(xTrain[:,1], yTrainClean, marker='.');
plt.xlabel('x1')
plt.title('Bivariate model without noise');
```

It looks like a tobogan from the side ($x_0$) and from the front ($x_1$)

# Single neuron model with Keras

```python
# Number of epochs
nEpoch = 8
nBatch = 128 # 32 is default

# Model
model1 = keras.models.Sequential([
  keras.layers.Dense(1, activation='linear', input_shape=[nFeatures])
])
model1.compile(optimizer='adam',
              loss=keras.losses.mean_squared_error,
              metrics=['kullback_leibler_divergence'])

# Tensor board
callbacks = []
if usingTensorBoard:
    ks = keras.callbacks.TensorBoard(log_dir="./logs/",
                                     histogram_freq=1, write_graph=True, write_grads=True, batch_size=1)
    callbacks = [ks]
```

Using Kullback-Leibler divergence as an extra measure of the model.

References:
- https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
- (French) https://www.science-emergence.com/Articles/Divergence-de-Kullback-Leibler-avec-python-et-matplotlib/

$D_{KL}(P \Vert Q)=\int_{-\infty}^{+\infty} p(x) log\bigl(\frac{p(x)}{q(x)}\bigr)$

```python
# Fit
hist1 = model1.fit(xTrain, yTrain, epochs=nEpoch, batch_size=nBatch, validation_split = 0.2, verbose=0, callbacks=callbacks)
```

```python
weights1, biases1 = model1.get_weights()
print('Est W =', weights1.reshape(-1), ', b =', biases1[0])
```

    Est W = [0.11365225 0.00045345] , b = 0.34393284

```python
plt.figure(figsize=(15,4))
plt.subplot(1,3,1)
plt.semilogy(hist1.history['loss'])
plt.semilogy(hist1.history['val_loss'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Loss')
plt.subplot(1,3,2)
plt.semilogy(hist1.history['kullback_leibler_divergence'])
plt.semilogy(hist1.history['val_kullback_leibler_divergence'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Kullback-Leibler divergence');
```

### Test model

```python
yEst1 = np.matmul(xTest, weights1) + biases1
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.')
plt.scatter(xTest[:,0], yEst1, marker='.', alpha=0.05)
plt.title('x0')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.')
plt.scatter(xTest[:,1], yEst1, marker='.', alpha=0.05)
plt.title('x1')
plt.legend(('ori clean', 'estimated'))
mse1 = metrics.mean_squared_error(yTest, yEst1)
print('Gradient descent MSE = {:.3e}'.format(mse1));
```

# Using a 2 layer Neural Network for a better fit

References:
- About regularization, see https://www.tensorflow.org/tutorials/keras/overfit_and_underfit

```python
# Number of epochs
nEpoch = 50
nBatch = 256 # 32 is default

# Model
model2 = keras.models.Sequential([
    keras.layers.Dense(8, activation=keras.activations.softmax, input_shape=[nFeatures],      # <---
                        bias_regularizer=keras.regularizers.l1(0.00001),      # <---
                        kernel_regularizer=keras.regularizers.l1(0.00001)), # <----
    keras.layers.Dense(1, activation='linear')
])
model2.compile(optimizer='adam',
              loss=keras.losses.mean_squared_error,
              metrics=['kullback_leibler_divergence'])

# Tensor board
callbacks = []
if usingTensorBoard:
    ks = keras.callbacks.TensorBoard(log_dir="./logs/",
                                     histogram_freq=1, write_graph=True, write_grads=True, batch_size=1)
    callbacks = [ks]

# Fit
hist2 = model2.fit(xTrain, yTrain, epochs=nEpoch, batch_size=nBatch, validation_split = 0.2, verbose=0, callbacks=callbacks)
```

```python
model2.summary()
```

    Model: "sequential_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_1 (Dense)              (None, 8)                 24
    _________________________________________________________________
    dense_2 (Dense)              (None, 1)                 9
    =================================================================
    Total params: 33
    Trainable params: 33
    Non-trainable params: 0
    _________________________________________________________________

```python
model2.get_weights()
```

    [array([[-5.9156574e-04, -9.4933707e-01, -7.1687612e-04, -1.0327047e+00,
             -7.3904550e-04, -6.4879522e-04,  1.8037283e+00, -8.6817262e-04],
            [ 1.1798851e-04,  1.1310233e+00,  8.9924975e-04, -9.8094767e-01,
              3.1008647e-04,  6.1978877e-05, -1.2418784e-03,  5.0762389e-04]],
           dtype=float32),
     array([-0.00043835,  0.00064997, -0.0009306 ,  0.00560384, -0.00041394,
            -0.00052826, -0.00544376, -0.00071103], dtype=float32),
     array([[-0.7158145 ],
            [ 0.9077484 ],
            [ 0.03333439],
            [ 1.0335735 ],
            [-0.542008  ],
            [-0.5094827 ],
            [ 0.9561239 ],
            [-0.12001117]], dtype=float32),
     array([0.21647635], dtype=float32)]

```python
plt.figure(figsize=(15,4))
plt.subplot(1,3,1)
plt.semilogy(hist2.history['loss'])
plt.semilogy(hist2.history['val_loss'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Loss')
plt.subplot(1,3,2)
plt.semilogy(hist2.history['kullback_leibler_divergence'])
plt.semilogy(hist2.history['val_kullback_leibler_divergence'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Kullback-Leibler divergence');
```

### Test model with two layers

```python
yEst2 = model2.predict(xTest).reshape(-1)
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.', alpha=0.05)
plt.scatter(xTest[:,0], yEst2, marker='.', alpha=0.05)
plt.title('x0')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.', alpha=0.05)
plt.scatter(xTest[:,1], yEst2, marker='.', alpha=0.05)
plt.title('x1')
plt.legend(('ori clean', 'estimated'))
mse2 = metrics.mean_squared_error(yTest, yEst2)
print('Neural MSE = {:.3e}'.format(mse2));
```

### Conclusion on the two layer model

Adding a second layers leads to better fitting of the generation function (polynomial degree 4).

This is however mostly due to the non-linear activation function (Softmax), this is shown below with the same model but with a Relu activation. The predicted data is showing the elbows of the multiple Relu.

### Two layer model using Relu

```python
# Number of epochs
nEpoch = 50
nBatch = 256 # 32 is default

# Model
model3 = keras.models.Sequential([
    keras.layers.Dense(8, activation='relu', input_shape=[nFeatures],      # <---
                        bias_regularizer=keras.regularizers.l1(0.00001),
                        kernel_regularizer=keras.regularizers.l1(0.00001)),
    keras.layers.Dense(1, activation='linear')
])
model3.compile(optimizer='adam',
              loss=keras.losses.mean_squared_error,
              metrics=['kullback_leibler_divergence'])

# Tensor board
callbacks = []
if usingTensorBoard:
    ks = keras.callbacks.TensorBoard(log_dir="./logs/",
                                     histogram_freq=1, write_graph=True, write_grads=True, batch_size=1)
    callbacks = [ks]

# Fit
hist3 = model3.fit(xTrain, yTrain, epochs=nEpoch, batch_size=nBatch, validation_split = 0.2, verbose=0, callbacks=callbacks)
```

```python
plt.figure(figsize=(15,4))
plt.subplot(1,3,1)
plt.semilogy(hist3.history['loss'])
plt.semilogy(hist3.history['val_loss'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Loss')
plt.subplot(1,3,2)
plt.semilogy(hist3.history['kullback_leibler_divergence'])
plt.semilogy(hist3.history['val_kullback_leibler_divergence'])
plt.grid()
plt.legend(('train', 'validation'))
plt.title('Kullback-Leibler divergence');
```

```python
yEst3 = model3.predict(xTest).reshape(-1)
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.', alpha=0.05)
plt.scatter(xTest[:,0], yEst3, marker='.', alpha=0.05)
plt.title('x0')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.', alpha=0.05)
plt.scatter(xTest[:,1], yEst3, marker='.', alpha=0.05)
plt.title('x1')
plt.legend(('ori clean', 'estimated'))
mse3 = metrics.mean_squared_error(yTest, yEst3)
print('Neural MSE = {:.3e}'.format(mse3));
```

### To try on your own:
- Add more unit (neurons) to the first layer
- Add an hidden layer

# Where to go from here ?

__Other two feature linear implementation__ [using "raw" Python](LinearRegressionBivariate.html) ([Notebook](LinearRegressionBivariate.ipynb))

__Compare with the two feature binary classification using logistic regression__ [using Keras](../classification/ClassificationContinuous2Features-Keras.html) ([Notebook](../classification/ClassificationContinuous2Features-Keras.ipynb]))