## Rigid body 6 DOF
Devlop a system for a rigid bofy in 6 DOF and do a simualtion

```python
import warnings
#warnings.filterwarnings('ignore')
%matplotlib inline
%load_ext autoreload
%autoreload 2
```

```python
import sympy as sp
import sympy.physics.mechanics as me
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import rigidbody
from rigidbody.substitute_dynamic_symbols import substitute_dynamic_symbols, find_name, find_derivative_name, lambdify, find_derivatives
from rigidbody.plotting import TrackPlot3dWidget,track_plot

from pydy.codegen.ode_function_generators import generate_ode_function
from scipy.integrate import odeint

```

```python
x0,y0,z0 = me.dynamicsymbols('x0 y0 z0')
x01d,y01d,z01d = me.dynamicsymbols('x01d y01d z01d')
u,v,w = me.dynamicsymbols('u v w')

phi,theta,psi = me.dynamicsymbols('phi theta psi')

phi1d,theta1d,psi1d = me.dynamicsymbols('phi1d theta1d psi1d')
```

```python
N = me.ReferenceFrame('N')
```

```python
S = N.orientnew('S', 'Body', [psi,theta,phi],'ZYX')
```

```python
S.ang_vel_in(N)
```

$\displaystyle (- \operatorname{sin}\left(\theta\right) \dot{\psi} + \dot{\phi})\mathbf{\hat{s}_x} + (\operatorname{sin}\left(\phi\right) \operatorname{cos}\left(\theta\right) \dot{\psi} + \operatorname{cos}\left(\phi\right) \dot{\theta})\mathbf{\hat{s}_y} + (- \operatorname{sin}\left(\phi\right) \dot{\theta} + \operatorname{cos}\left(\phi\right) \operatorname{cos}\left(\theta\right) \dot{\psi})\mathbf{\hat{s}_z}$

```python
S.ang_acc_in(N)
```

$\displaystyle (- \operatorname{sin}\left(\theta\right) \ddot{\psi} - \operatorname{cos}\left(\theta\right) \dot{\psi} \dot{\theta} + \ddot{\phi})\mathbf{\hat{s}_x} + (- \operatorname{sin}\left(\phi\right) \operatorname{sin}\left(\theta\right) \dot{\psi} \dot{\theta} + \operatorname{sin}\left(\phi\right) \operatorname{cos}\left(\theta\right) \ddot{\psi} - \operatorname{sin}\left(\phi\right) \dot{\phi} \dot{\theta} + \operatorname{cos}\left(\phi\right) \operatorname{cos}\left(\theta\right) \dot{\phi} \dot{\psi} + \operatorname{cos}\left(\phi\right) \ddot{\theta})\mathbf{\hat{s}_y} + (- \operatorname{sin}\left(\phi\right) \operatorname{cos}\left(\theta\right) \dot{\phi} \dot{\psi} - \operatorname{sin}\left(\phi\right) \ddot{\theta} - \operatorname{sin}\left(\theta\right) \operatorname{cos}\left(\phi\right) \dot{\psi} \dot{\theta} + \operatorname{cos}\left(\phi\right) \operatorname{cos}\left(\theta\right) \ddot{\psi} - \operatorname{cos}\left(\phi\right) \dot{\phi} \dot{\theta})\mathbf{\hat{s}_z}$

```python
M = me.Point('M')
O = M.locatenew('P',0)

M.set_vel(N,0)
O.set_vel(S,u*S.x + v*S.y + w*S.z)

O.v1pt_theory(M,N,S)
```

$\displaystyle u\mathbf{\hat{s}_x} + v\mathbf{\hat{s}_y} + w\mathbf{\hat{s}_z}$

```python
velocity_matrix = O.vel(N).to_matrix(N)
velocity_matrix
```

$\displaystyle \left[\begin{matrix}\left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) w{\left(t \right)} + \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) v{\left(t \right)} + u{\left(t \right)} \cos{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)}\\\left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) w{\left(t \right)} + \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) v{\left(t \right)} + u{\left(t \right)} \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)}\\- u{\left(t \right)} \sin{\left(\theta{\left(t \right)} \right)} + v{\left(t \right)} \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + w{\left(t \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)}\end{matrix}\right]$

```python
rotional_velocity_matrix = sp.simplify(S.ang_vel_in(N).to_matrix(N))
rotional_velocity_matrix
```

$\displaystyle \left[\begin{matrix}- \sin{\left(\psi{\left(t \right)} \right)} \frac{d}{d t} \theta{\left(t \right)} + \cos{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} \frac{d}{d t} \phi{\left(t \right)}\\\sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} \frac{d}{d t} \phi{\left(t \right)} + \cos{\left(\psi{\left(t \right)} \right)} \frac{d}{d t} \theta{\left(t \right)}\\- \sin{\left(\theta{\left(t \right)} \right)} \frac{d}{d t} \phi{\left(t \right)} + \frac{d}{d t} \psi{\left(t \right)}\end{matrix}\right]$

## Mass

```python
mass = sp.symbols('m')
#x_g, y_g, z_g = sp.symbols('x_g y_g z_g')
```

```python
#c_og = me.Point('c_og')
#c_og.set_pos(O,x_g*S.x + y_g*S.y + z_g*S.z)
#c_og.set_vel(S,0)
```

```python
#c_og.v2pt_theory(O,N,S)
```

## Inertia

```python
I_xx, I_yy, I_zz = sp.symbols('I_xx, I_yy, I_zz')
body_inertia_dyadic = me.inertia(S, ixx=I_xx, iyy=I_yy, izz=I_zz)
body_inertia_dyadic
```

$\displaystyle I_{xx}\mathbf{\hat{s}_x}\otimes \mathbf{\hat{s}_x} + I_{yy}\mathbf{\hat{s}_y}\otimes \mathbf{\hat{s}_y} + I_{zz}\mathbf{\hat{s}_z}\otimes \mathbf{\hat{s}_z}$

```python
body_inertia_dyadic.to_matrix(S)
```

$\displaystyle \left[\begin{matrix}I_{xx} & 0 & 0\\0 & I_{yy} & 0\\0 & 0 & I_{zz}\end{matrix}\right]$

```python
body_inertia_dyadic.to_matrix(N)
```

$\displaystyle \left[\begin{matrix}I_{xx} \cos^{2}{\left(\psi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right)^{2} + I_{zz} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right)^{2} & I_{xx} \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) + I_{zz} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) & - I_{xx} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{zz} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)}\\I_{xx} \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) + I_{zz} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) & I_{xx} \sin^{2}{\left(\psi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right)^{2} + I_{zz} \left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right)^{2} & - I_{xx} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{zz} \left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)}\\- I_{xx} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} - \sin{\left(\psi{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{zz} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} + \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} & - I_{xx} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{yy} \left(\sin{\left(\phi{\left(t \right)} \right)} \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} + \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)}\right) \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} + I_{zz} \left(- \sin{\left(\phi{\left(t \right)} \right)} \cos{\left(\psi{\left(t \right)} \right)} + \sin{\left(\psi{\left(t \right)} \right)} \sin{\left(\theta{\left(t \right)} \right)} \cos{\left(\phi{\left(t \right)} \right)}\right) \cos{\left(\phi{\left(t \right)} \right)} \cos{\left(\theta{\left(t \right)} \right)} & I_{xx} \sin^{2}{\left(\theta{\left(t \right)} \right)} + I_{yy} \sin^{2}{\left(\phi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)} + I_{zz} \cos^{2}{\left(\phi{\left(t \right)} \right)} \cos^{2}{\left(\theta{\left(t \right)} \right)}\end{matrix}\right]$

```python
body_central_inertia = (body_inertia_dyadic, O)
```

```python
body = me.RigidBody('Rigid body', masscenter=O, frame = S,
                      mass=mass, inertia=body_central_inertia)
```

```python

```

## Forces

```python
fx, fy, fz, mx, my, mz = sp.symbols('f_x f_y f_z m_x m_y m_z')
```

```python
force_vector = fx*S.x + fy*S.y + fz*S.z
torque_vector = mx*S.x + my*S.y + mz*S.z
```

```python
force = (O, force_vector)
torque = (S, torque_vector)
```

## Equations of Motion

```python
coordinates = [x0, y0, z0, phi, theta, psi]
speeds = [u, v, w, phi1d, theta1d, psi1d]
```

```python
me.kinematic_equations(speeds[0:3],coordinates[0:3],'Body', rot_order='ZYX')
```

    [-(v(t)*sin(z0(t)) + w(t)*cos(z0(t)))/cos(y0(t)) + Derivative(x0(t), t),
     -v(t)*cos(z0(t)) + w(t)*sin(z0(t)) + Derivative(y0(t), t),
     -(v(t)*sin(z0(t)) + w(t)*cos(z0(t)))*sin(y0(t))/cos(y0(t)) - u(t) + Derivative(z0(t), t)]

```python
kinematical_differential_equations = [x0.diff() - velocity_matrix[0],
                                      y0.diff() - velocity_matrix[1],
                                      z0.diff() - velocity_matrix[2],
                                      phi.diff() - phi1d,
                                      theta.diff() - theta1d,
                                      psi.diff() - psi1d,
                                     ]
```

```python
sp.simplify(kinematical_differential_equations)
```

    [-(sin(phi(t))*sin(psi(t)) + sin(theta(t))*cos(phi(t))*cos(psi(t)))*w(t) - (sin(phi(t))*sin(theta(t))*cos(psi(t)) - sin(psi(t))*cos(phi(t)))*v(t) - u(t)*cos(psi(t))*cos(theta(t)) + Derivative(x0(t), t),
     -(-sin(phi(t))*cos(psi(t)) + sin(psi(t))*sin(theta(t))*cos(phi(t)))*w(t) - (sin(phi(t))*sin(psi(t))*sin(theta(t)) + cos(phi(t))*cos(psi(t)))*v(t) - u(t)*sin(psi(t))*cos(theta(t)) + Derivative(y0(t), t),
     u(t)*sin(theta(t)) - v(t)*sin(phi(t))*cos(theta(t)) - w(t)*cos(phi(t))*cos(theta(t)) + Derivative(z0(t), t),
     -phi1d(t) + Derivative(phi(t), t),
     -theta1d(t) + Derivative(theta(t), t),
     -psi1d(t) + Derivative(psi(t), t)]

```python
#?me.KanesMethod
```

```python
kane = me.KanesMethod(N, coordinates, speeds, kinematical_differential_equations)
```

```python
loads = [force,
         torque]
```

```python
bodies = [body]
fr, frstar = kane.kanes_equations(bodies, loads)
```

```python
constants = [I_xx, I_yy, I_zz,mass]

specified = [fx, fy, fz, mx, my, mz]  # External force/torque

right_hand_side = generate_ode_function(kane.forcing_full, coordinates,
                                        speeds, constants,
                                        mass_matrix=kane.mass_matrix_full,specifieds=specified)
```

```python
coordinates_ = [0, 0, 0, 0, 0, 0]
speeds_ = [0, 0, 0, 0, 0, 0]

start = np.array(coordinates_+speeds_)

t_ = 0.
force_torque = [1,0,0,0,0,0]
numerical_specified = np.array(force_torque)

I_xx_ = 1
I_yy_ = 1
I_zz_ = 1
mass_ = 1

numerical_constants = np.array([I_xx_, I_yy_, I_zz_,mass_])

right_hand_side(start,t_,numerical_specified, numerical_constants)
```

    array([0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.])

```python
def simulate(t,force_torque, I_xx,I_yy,I_zz,mass, initial_coordinates = [0, 0, 0, 0, 0, 0],
             initial_speeds = [0, 0, 0, 0, 0, 0]):

    start = np.array(initial_coordinates+initial_speeds)

    numerical_specified = force_torque

    numerical_constants = np.array([I_xx, I_yy, I_zz, mass])

    df = pd.DataFrame(index=t)
    y = odeint(right_hand_side, start, t, args=(numerical_specified, numerical_constants))

    for i,symbol in enumerate(coordinates+speeds):
        name = symbol.name
        df[name] = y[:,i]

    return df
```

```python
t = np.linspace(0,10,100)

df = simulate(t=t, force_torque=[1,0,0,0,0,0],I_xx=1, I_yy=1, I_zz=1, mass=1)

fig,ax = plt.subplots()
df.plot(y='x0', ax=ax);
ax.set_xlabel('time [s]')

fig,ax = plt.subplots()
df.plot(y='u', ax=ax);
ax.set_xlabel('time [s]');
```

```python

radius = 10  # Radius of rotation [m]
w = 0.1  # Angle velocity [rad/s]
V = radius*w  # Speed of point [m/s]
t = np.linspace(0,2*np.pi/w,100)

mass=1

expected_acceleration = -radius*w**2
expected_force = mass*expected_acceleration

df = simulate(t=t, force_torque=[0,-expected_force,0,0,0,0],I_xx=1, I_yy=1, I_zz=1, mass=mass,
              initial_speeds=[V,0,0,0,0,w],initial_coordinates=[0, -radius, 0,0,0,0])

fig,ax = plt.subplots()
track_plot(df,ax, time_step = '5S')

fig,ax = plt.subplots()
df.plot(y='psi', ax=ax);
ax.set_xlabel('time')
ax.set_ylabel('psi')
```

```python
t = np.linspace(0,5,100)

df = simulate(t=t, force_torque=[1,2,0,0,0,0.2],I_xx=1, I_yy=1, I_zz=1, mass=1,
              initial_speeds=[1,0,0,0,0,0.0],initial_coordinates=[0, 0, 0, 0, 0, 0.0])

fig,ax = plt.subplots()
track_plot(df,ax)

fig,ax = plt.subplots()
df.plot(y='psi', ax=ax);
ax.set_xlabel('time')
ax.set_ylabel('psi')

```

```python
t = np.linspace(0,10,100)

l = 1
mass = 1
kzz = 0.3
I_zz = mass*(kzz*l)**2

df = simulate(t=t, force_torque=[0,0.9,0,0,0,0.0],I_xx=1, I_yy=I_zz, I_zz=I_zz, mass=mass,
              initial_speeds=[0,0,0,0,0,0.0],initial_coordinates=[0, 0, 0, 0, 0, np.pi])

fig,ax = plt.subplots()
track_plot(df,ax)

fig,ax = plt.subplots()
df.plot(y='psi', ax=ax);
ax.set_xlabel('time')
ax.set_ylabel('psi')
```

```python
t = np.linspace(0,10,100)

l = 1
mass = 1
kzz = 0.3
I_zz = mass*(kzz*l)**2

df = simulate(t=t, force_torque=[0,0.0,0,0,0,0.0],I_xx=1, I_yy=I_zz, I_zz=I_zz, mass=mass,
              initial_speeds=[0,1,0,0,0,0.0],initial_coordinates=[0, 0, 0, 0, 0, np.pi])

fig,ax = plt.subplots()
track_plot(df,ax)

fig,ax = plt.subplots()
df.plot(y='psi', ax=ax);
ax.set_xlabel('time')
ax.set_ylabel('psi')
```

```python

radius = 2  # Radius of rotation [m]
w = 0.1  # Angle velocity [rad/s]
V = radius*w  # Speed of point [m/s]
t = np.linspace(0,2*np.pi/w,1000)

mass=1

expected_acceleration = -radius*w**2
expected_force = mass*expected_acceleration

df = simulate(t=t, force_torque=[0,-expected_force,0.0,0,0,0],I_xx=1, I_yy=1, I_zz=1, mass=mass,
              initial_speeds=[V,0,0.1,0,0,w],initial_coordinates=[0, 0, 0,0,0,0])

```

```python
scene = TrackPlot3dWidget(df=df)
```

```python
t = np.linspace(0,2*np.pi/w,100)

mass = 1
I_xx = 10
I_yy = 10
I_zz = 10

t = np.linspace(0, 10, 100)

initial_speeds = [0,0,0,0.0,0.0,0.3]

initial_coordinates = [0, 0, 0, np.deg2rad(45), 0, 0]

force_torque = [0,0,0,0,0,0]

# Run an alternative simulation:
df = simulate(t=t, initial_speeds=initial_speeds, initial_coordinates=initial_coordinates,
                             mass=mass,I_xx=I_xx, I_yy=I_yy, I_zz=I_zz,force_torque=force_torque)

fig,ax = plt.subplots()
track_plot(df,ax, time_step = '5S')

fig,ax = plt.subplots()
df.plot(y='psi', ax=ax);
ax.set_xlabel('time')
ax.set_ylabel('psi')
```

```python
scene = TrackPlot3dWidget(df=df)
```

```python
t = np.linspace(0,2*np.pi/w,100)

mass = 1
I_xx = 10
I_yy = 10
I_zz = 10

t = np.linspace(0, 10, 100)

initial_speeds = [0,0,0,0.1,0.0,0.0]

initial_coordinates = [0, 0, 0, 0, 0, np.deg2rad(45)]

force_torque = [0,0,0,0,0,0]

# Run an alternative simulation:
df = simulate(t=t, initial_speeds=initial_speeds, initial_coordinates=initial_coordinates,
                             mass=mass,I_xx=I_xx, I_yy=I_yy, I_zz=I_zz,force_torque=force_torque)

scene = TrackPlot3dWidget(df=df)
```

```python

```


# Problem set 7: Solving the consumer problem with income risk

[](https://mybinder.org/v2/gh/NumEconCopenhagen/exercises-2020/master?urlpath=lab/tree/PS6/problem_set_7.ipynb)

```python
import numpy as np
import scipy as sp
from scipy import linalg
from scipy import optimize
from scipy import interpolate
import sympy as sm

%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
```

# Tasks

<font size="5"
          face="verdana"
          color="red">
            Guide:
</font>
Solving the consumer problem is the **primary task** in this exercise set. You should spend most of the time you have on testing that you understand the different optimizers (problem I) and on solving the intertemporal consumption model (problem III). If for instance you are stuck in plotting, then skip ahead.

## Optimization problem I

Consider the function

$$
f(\boldsymbol{x}) = f(x_1,x_2) = (x_1^2 - x_1x_2 + x_2^2)^2
$$

Define it in **sympy** by:

```python
x1 = sm.symbols('x_1')
x2 = sm.symbols('x_2')
f = (x1**2 - x1*x2 + x2**2)**2
```

The **Jacobian** is

```python
f1 = sm.diff(f,x1)
f2 = sm.diff(f,x2)
sm.Matrix([f1,f2])
```

$\displaystyle \left[\begin{matrix}\left(4 x_{1} - 2 x_{2}\right) \left(x_{1}^{2} - x_{1} x_{2} + x_{2}^{2}\right)\\\left(- 2 x_{1} + 4 x_{2}\right) \left(x_{1}^{2} - x_{1} x_{2} + x_{2}^{2}\right)\end{matrix}\right]$

The **Hessian**  is

```python
f11 = sm.diff(f,x1,x1)
f12 = sm.diff(f,x1,x2)
f21 = sm.diff(f,x2,x1)
f22 = sm.diff(f,x2,x2)
sm.Matrix([[f11,f12],[f21,f22]])
```

$\displaystyle \left[\begin{matrix}2 \left(2 x_{1}^{2} - 2 x_{1} x_{2} + 2 x_{2}^{2} + \left(2 x_{1} - x_{2}\right)^{2}\right) & 2 \left(- x_{1}^{2} + x_{1} x_{2} - x_{2}^{2} - \left(x_{1} - 2 x_{2}\right) \left(2 x_{1} - x_{2}\right)\right)\\2 \left(- x_{1}^{2} + x_{1} x_{2} - x_{2}^{2} - \left(x_{1} - 2 x_{2}\right) \left(2 x_{1} - x_{2}\right)\right) & 2 \left(2 x_{1}^{2} - 2 x_{1} x_{2} + 2 x_{2}^{2} + \left(x_{1} - 2 x_{2}\right)^{2}\right)\end{matrix}\right]$

**Question A:** Lambdify $f(x_1,x_2)$ and use it to create:

$(i)$ a 3D surfaceplot looking like this:

$(ii)$ a contourplot looking like:

```python
_f = sm.lambdify((x1,x2),f)

# write your code here
# Hint: use a np.meshgrid
```

**Answer:** A1.py and A2.py

**Question B:** Construct python functions for the jacobian and the hessian.

```python
f_python = lambda x: _f(x[0],x[1])

# write your code here
```

**Answer:** A3.py

**Question C:** Minimize $f(x_1,x_2)$ using respectively

1. Nelder-Mead,
2. BFGS without analytical jacobian,
3. BFGS with analytical jacobian, and
4. Newton-CG with analytical jacobian and hessian

Compare the results and discuss which optimizer you prefer.

**Optional:** If you wish, you can use the functions defined in the hidden cells below to also track how the optimizers converges to the solution.

```python
def collect(x):

    # globals used to keep track across iterations
    global evals # set evals = 0 before calling optimizer
    global x0
    global x1s
    global x2s
    global fs

    # a. initialize list
    if evals == 0:
        x1s = [x0[0]]
        x2s = [x0[1]]
        fs = [f_python(x0)]

    # b. append trial values
    x1s.append(x[0])
    x2s.append(x[1])
    fs.append(f_python(x))

    # c. increment number of evaluations
    evals += 1
```

```python
def contour():

    global evals
    global x1s
    global x2s
    global fs

    # a. contour plot
    fig = plt.figure(figsize=(10,4))
    ax = fig.add_subplot(1,2,1)
    levels = np.sort([j*10**(-i) for i in [-1,0,1,2,3,4] for j in [0.5,1,1.5]])
    cs = ax.contour(x1_grid,x2_grid,f_grid,levels=levels,cmap=cm.jet)
    fig.colorbar(cs)
    ax.plot(x1s,x2s,'-o',ms=4,color='black')
    ax.set_xlabel('$x_1$')
    ax.set_ylabel('$x_2$')

    # b. function value
    ax = fig.add_subplot(1,2,2)
    ax.plot(np.arange(evals+1),fs,'-o',ms=4,color='black')
    ax.set_xlabel('iteration')
    ax.set_ylabel('function value')
```

```python
x0 = [-2,-1] # suggested initial guess

# write your code here
```

**Answer:** A4.py, A5.py, A6.py, A7.py

## Optimization problem II

Consider the function

$$
f(x_1,x_2) = (4-2.1x_1^2 + \frac{x_1^4}{3})x_1^2 + x_1x_2 + (4x_2^2 - 4)x_2^2)
$$

Define it in **sympy** by:

```python
x1 = sm.symbols('x_1')
x2 = sm.symbols('x_2')
f = (4-2.1*x1**2 + (x1**4)/3)*x1**2 + x1*x2 + (4*x2**2 - 4)*x2**2
_f = sm.lambdify((x1,x2),f)
f
```

$\displaystyle x_{1}^{2} \left(\frac{x_{1}^{4}}{3} - 2.1 x_{1}^{2} + 4\right) + x_{1} x_{2} + x_{2}^{2} \left(4 x_{2}^{2} - 4\right)$

Create **3D plot**:

```python
# a. grids
x1_vec = np.linspace(-2,2,500)
x2_vec = np.linspace(-1,1,500)
x1_grid,x2_grid = np.meshgrid(x1_vec,x2_vec,indexing='ij')
f_grid = _f(x1_grid,x2_grid)

# b. main
fig = plt.figure()
ax = fig.add_subplot(1,1,1,projection='3d')
cs = ax.plot_surface(x1_grid,x2_grid,f_grid,cmap=cm.jet)

# c. add labels
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('$f$')

# d. invert xaxis
ax.invert_xaxis()

# e. remove background
ax.xaxis.pane.fill = False
ax.yaxis.pane.fill = False
ax.zaxis.pane.fill = False

# f. add colorbar
fig.colorbar(cs);
```

**Question A:** Find the minimum of the function starting from each of the suggested initial values below. Print the first 20 solutions, and all solutions aftwards, which is the best yet to be found. Save the solutions and associated function values in `xs` and `fs`.

```python
# a. python function for f
f_python = lambda x: _f(x[0],x[1])

# b. initial guesses
np.random.seed(1986)
K = 1000
x0s = np.empty((K,2))
x0s[:,0] = -2 + 4*np.random.uniform(size=K)
x0s[:,1] = -1 + 2*np.random.uniform(size=K)

# c. solutions and associated values
xs = np.empty((K,2))
fs = np.empty(K)

# write your code here
```

**Answer:** A8.py

**Question B:** Create a 3D scatter plot of where the optimizer converges, and color the dots by the associated function values.

```python
# write your code here
```

**Answer:** A9.py

**Question C:** Plot the function values at the solutions as a function of the starting values.

```python
# write your code here
```

**Answer:** A10.py

# Problem: Solve the consumer problem with income risk I

Define the following **variables** and **parameters**:

* $m_t$ is cash-on-hand in period $t$
* $c_t$ is consumption in period $t$
* $y_t$ is income in period $t$
* $\Delta \in (0,1)$ is income risk
* $r$ is the interest rate
* $\beta > 0$, $\rho > 1$, $\nu > 0 $, $\kappa > 0$, $\xi > 0$ are utility parameters

In the **second period** the household solves:

$$
\begin{aligned}
v_{2}(m_{2}) &= \max_{c_{2}}\frac{c_{2}^{1-\rho}}{1-\rho}+\nu\frac{(m_{2}-c_{2}+\kappa)^{1-\rho}}{1-\rho} \\
 \text{s.t.} \\
c_{2} & \in [0,m_{2}]
\end{aligned}
$$

In the **first period** the household solves:

$$
\begin{aligned}
v_{1}(m_{1}) & =
\max_{c_{1}}\frac{c_{1}^{1-\rho}}{1-\rho}+\beta\mathbb{E}_{1}\left[v_2(m_2)\right] \\
\text{s.t.} \\
m_2 &= (1+r)(m_{1}-c_{1})+y_{2} \\
y_{2} &= \begin{cases}
1-\Delta & \text{with prob. }0.5\\
1+\Delta & \text{with prob. }0.5
\end{cases}\\
c_{1} & \in [0,m_{1}]\\
\end{aligned}
$$

The **basic functions** are:

```python
def utility(c,rho):
    return c**(1-rho)/(1-rho)

def bequest(m,c,nu,kappa,rho):
    return nu*(m-c+kappa)**(1-rho)/(1-rho)

def v2(c2,m2,rho,nu,kappa):
    return utility(c2,rho) + bequest(m2,c2,nu,kappa,rho)

def v1(c1,m1,rho,beta,r,Delta,v2_interp):

    # a. v2 value, if low income
    m2_low = (1+r)*(m1-c1) + 1-Delta
    v2_low = v2_interp([m2_low])[0]

    # b. v2 value, if high income
    m2_high = (1+r)*(m1-c1) + 1+Delta
    v2_high = v2_interp([m2_high])[0]

    # c. expected v2 value
    v2 = 0.5*v2_low + 0.5*v2_high

    # d. total value
    return utility(c1,rho) + beta*v2
```

The **solution functions** are:

```python
def solve_period_2(rho,nu,kappa,Delta):

    # a. grids
    m2_vec = np.linspace(1e-8,5,500)
    v2_vec = np.empty(500)
    c2_vec = np.empty(500)

    # b. solve for each m2 in grid
    for i,m2 in enumerate(m2_vec):

        # i. objective
        obj = lambda x: -v2(x[0],m2,rho,nu,kappa)

        # ii. initial value (consume half)
        x0 = m2/2

        # iii. optimizer
        result = optimize.minimize(obj,[x0],method='L-BFGS-B',bounds=((1e-12,m2),))

        # iv. save
        v2_vec[i] = -result.fun
        c2_vec[i] = result.x[0]

    return m2_vec,v2_vec,c2_vec

def solve_period_1(rho,beta,r,Delta,v1,v2_interp):

    # a. grids
    m1_vec = np.linspace(1e-8,4,100)
    v1_vec = np.empty(100)
    c1_vec = np.empty(100)

    # b. solve for each m1 in grid
    for i,m1 in enumerate(m1_vec):

        # i. objective
        obj = lambda x: -v1(x[0],m1,rho,beta,r,Delta,v2_interp)

        # ii. initial guess (consume half)
        x0 = m1/2

        # iii. optimize
        result = optimize.minimize(obj,[x0],method='L-BFGS-B',bounds=((1e-12,m1),))

        # iv. save
        v1_vec[i] = -result.fun
        c1_vec[i] = result.x[0]

    return m1_vec,v1_vec,c1_vec
```

**Question A:** Find optimal consumption in the first period as funcition of cash-on-hand, and plot it.

```python
rho = 8
kappa = 0.5
nu = 0.1
r = 0.04
beta = 0.94
Delta = 0.5

# b. solve
# write your code here

# c. plot
# write your code here
```

**Answer:** A11.py

**Question B:** Find optimal consumption in the first period as funcition of cash-on-hand, and plot it, assuming that

$$
y_{2} = \begin{cases}
1-\sqrt{\Delta} & \text{with prob. }0.1\\
1-\Delta & \text{with prob. }0.4\\
1+\Delta & \text{with prob. }0.4\\
1+\sqrt{\Delta} & \text{with prob. }0.1
\end{cases}
$$

which add some low probability tail events, but does not change mean income. Give an interpretation of the change in the consumption function.

```python
# write your code here
```

**Answer:** A12.py

# Problem: Solve the consumer problem with income risk II

Define the following **variables** and **parameters**:

* $m_t$ is cash-on-hand in period $t$
* $c_t$ is non-durable consumption in period $t$
* $d_t$ is durable consumption in period $t$ (only adjustable in period 1)
* $y_t$ is income in period $t$
* $\Delta \in (0,1)$ is income risk
* $r$ is the interest rate
* $\beta > 0$, $\rho > 1$, $\alpha \in (0,1)$, $\nu > 0 $, $\kappa > 0$, $\xi > 0$ are utility parameters

In the **second period** the household solves:

$$
\begin{aligned}
v_{2}(m_{2},d_{2}) &= \max_{c_{2}}\frac{c_{2}^{1-\rho}}{1-\rho}+\alpha\frac{d_{2}^{1-\rho}}{1-\rho}+\nu\frac{(m_{2}+d_{2}-c_{2}+\kappa)^{1-\rho}}{1-\rho} \\
\text{s.t.} \\
c_{2} & \in [0,m_{2}]
\end{aligned}
$$

In the **first period** the household solves:

$$
\begin{aligned}
v_{1}(m_{1}) &= \max_{c_{1},d_{1}}\frac{c_{1}^{1-\rho}}{1-\rho}+\alpha\frac{d_{1}^{1-\rho}}{1-\rho}+\beta\mathbb{E}_{1}\left[v_2(m_2,d_2)\right]\\&\text{s.t.}&\\
m_2 &= (1+r)(m_{1}-c_{1}-d_{1})+y_{2} \\
y_{2} &= \begin{cases}
1-\Delta & \text{with prob. }0.5\\
1+\Delta & \text{with prob. }0.5
\end{cases}\\
c_{1}+d_{1} & \in [0,m_{1}]\\
\end{aligned}
$$

Choose **parameters**:

```python
rho = 2
alpha = 0.1
kappa = 0.5
nu = 0.1
r = 0.04
beta = 0.94
Delta = 0.5

# b. solve
# write your code here

# c. plot
# write your code here
```

The **basic functions** are:

```python
def utility(c,d,alpha,rho):
    return c**(1-rho)/(1-rho) + alpha*d**(1-rho)/(1-rho)

def bequest(m,c,d,nu,kappa,rho):
    return nu*(m+d-c+kappa)**(1-rho)/(1-rho)

def v2(c2,d2,m2,alpha,rho,nu,kappa):
    return utility(c2,d2,alpha,rho) + bequest(m2,c2,d2,nu,kappa,rho)

def v1(c1,d1,m1,alpha,rho,beta,r,Delta,v2_interp):

    # a. v2 value, if low income
    m2_low = (1+r)*(m1-c1-d1) + 1-Delta
    v2_low = v2_interp([m2_low,d1])[0]

    # b. v2 value, if high income
    m2_high = (1+r)*(m1-c1-d1) + 1+Delta
    v2_high = v2_interp([m2_high,d1])[0]

    # c. expected v2 value
    v2 = 0.5*v2_low + 0.5*v2_high

    # d. total value
    return utility(c1,d1,alpha,rho) + beta*v2
```

The **solution function for period 2** is:

```python
def solve_period_2(alpha,rho,nu,kappa,Delta):

    # a. grids
    m2_vec = np.linspace(1e-8,5,200)
    d2_vec = np.linspace(1e-6,5,100)
    v2_grid = np.empty((200,100))
    c2_grid = np.empty((200,100))

    # b. solve for each m2 in grid
    for i,m2 in enumerate(m2_vec):
        for j,d2 in enumerate(d2_vec):

            # i. objective
            obj = lambda x: -v2(x[0],d2,m2,alpha,rho,nu,kappa)

            # ii. initial value (consume half)
            x0 = m2/2

            # iii. optimizer
            result = optimize.minimize(obj,[x0],method='L-BFGS-B',bounds=((1e-12,m2),))

            # iv. save
            v2_grid[i,j] = -result.fun
            c2_grid[i,j] = result.x[0]

    return m2_vec,d2_vec,v2_grid,c2_grid
```

**Question A:** Solve for consumption in period 2 and plot the consumption function.

```python
# write your code here
```

**Answer:** A13.py

**Question B:** Find optimal consumption and choices of durables in the first period as a function of cash-on-hand and plot it.

```python
# write your code here
```

**Answer:** A14.py

# Extra Problems

**Simulate** a distribution of consumers in either of the two consumption-saving models above. See **section 6.3 in lecture 11** regarding
how this is done.


# Simple Example
With tributary we can easily construct a forward-propagating reactive graph

```python
from display_helper import clean
clean()
from perspective import Plugin
import tributary.streaming as ts
import random, time
```

```python
def foo():
    return random.random()

def long():
    print('long called!')
    time.sleep(1)
    return 5

# A timer calls the given function every interval up to count times
test = ts.Timer(foo, {}, interval=.5, count=5)

test2 = ts.Negate(test)

res2 = test + test2

p2 = ts.Print(res2)
```

```python
ts.GraphViz(p2)
```

```python
x = ts.run(p2)
```

    0.0
    0.0
    0.0
    0.0
    0.0

```python
x.result()
```

# More Complex Example
Let's construct a slightly more complex graph

```python
import asyncio

def foo():
    return random.random()

async def long():
    await asyncio.sleep(1)
    return 5

rand = ts.Timer(foo, interval=0, count=5)
five = ts.Timer(long, interval=0, count=5)
one = ts.Const(1)
five2 = ts.Const(5)
neg_rand = ts.Negate(rand)

x1 = rand + five # 5 + rand
x2 = x1 - five2 # rand
x3 = x2 + neg_rand # 0
res = x3 + one # 1

p = ts.Print(res)

```

```python
p.graphviz()
```

```python
x = ts.run(p)
```

    0.9999999999999999
    0.9999999999999998
    1.0000000000000004
    0.9999999999999997
    0.9999999999999998

```python
x.result()
```

# Rolling Mean

```python
def stream():
    for _ in range(10):
        yield _

f = ts.Foo(stream)
s = f.rollingSum()
c = f.rollingCount()
f3 = ts.Div(s, c)
p = ts.Print(f3)

```

```python
p.graphviz()
```

```python
x = ts.run(p)
```

    0.0
    0.5
    1.0
    1.5
    2.0
    2.5
    3.0
    3.5
    4.0
    4.5

```python
x.result()
```

# Custom Calculations and Window Functions

```python
def myfoo(ds):
    for d in ds:
        vals = d.values()
        d['HIGH'] = max(vals)
        d['LOW'] = min(vals)
        d['MID'] = sum(vals)/len(vals)
    avgs = [x['MID'] for x in ds]
    ds[-1]['SMA'] = sum(avgs)/len(avgs)
    return [ds[-1]]
```

```python
vals = ts.Random(100)
w = ts.Window(vals, size=5)
n = ts.Apply(w, myfoo)
psp1 = ts.Perspective(n, schema={'HIGH': float, 'LOW': float, 'MID': float, 'SMA': float}, plugin=Plugin.YLINE)
x = ts.run(ts.Perspective(psp1, schema={'HIGH': float, 'LOW': float, 'MID': float, 'SMA': float}))
# This will only display in the notebook, not on Github
# it uses https://perspective.finos.org/
```

    PerspectiveWidget(columns=['HIGH', 'LOW', 'MID', 'SMA'], plugin=Plugin.YLINE)

    PerspectiveWidget(columns=['HIGH', 'LOW', 'MID', 'SMA'])

# Symbolic - Simple Example
Using tributary's sympy functionality, we can construct relatively complicated graphs. Here we will construct as simple lazy graph

```python
import tributary.symbolic as ts
import tributary.streaming as tss

# Parse sympy expression
expr = ts.parse_expression("10sin**2 x**2 + 3xyz + tan theta")

# Generate a new class representing the graph
clz = ts.construct_streaming(expr)

# A function to use as an input
def foo(*args):
    for _ in range(5):
        yield _

# Construct with inputs
x = clz(x=tss.Const(1), y=tss.Foo(foo), z=tss.Timer(lambda: 1, count=0), theta=tss.Const(4))
```

```python
# View the graph
x.graphviz()
```

```python
# Run the graph
y = x.run()
y
```

    <Task pending coro=<_run() running at /Users/theocean154/Programs/projects/tributary/tributary/examples/tributary/streaming/__init__.py:11>>

```python
y.result()
```

# Symbolic - More Complicated Example
Here we will construct a streaming pricer for a vanilla european option

```python
import numpy as np
import sympy as sy
from IPython.display import display, HTML
from sympy.stats import Normal as syNormal, cdf
sy.init_printing()

# adapted from https://gist.github.com/raddy/bd0e977dc8437a4f8276
#spot, strike, vol, days till expiry, interest rate, call or put (1,-1)
spot, strike, vol, dte, rate, cp = sy.symbols('spot strike vol dte rate cp')

T = dte / 260.
N = syNormal('N', 0.0, 1.0)

d1 = (sy.ln(spot / strike) + (0.5 * vol ** 2) * T) / (vol * sy.sqrt(T))
d2 = d1 - vol * sy.sqrt(T)

TimeValueExpr = sy.exp(-rate * T) * (cp * spot * cdf(N)(cp * d1) - cp * strike  * cdf(N)(cp * d2))
```

```python
TimeValueExpr
```

```python
import tributary.symbolic as ts
PriceClass = ts.construct_streaming(TimeValueExpr)

def strikes():
    strike = 205
    while strike < 220:
        yield strike
        strike += 2.5

price = PriceClass(spot=tss.Const(210.59),
                   strike=tss.Foo(strikes),
                   vol=tss.Const(14.04),
                   dte=tss.Const(4),
                   rate=tss.Const(.2175),
                   cp=tss.Const(-1))

price.graphviz()

```

Our function `strikes` will stream in strike prices, and we will see our price output change accordingly

```python
x = tss.run(tss.Print(price._node))

```

    124.8198933249367
    126.82508170405075
    128.83501237227932
    130.8496010565144
    132.86876573965452
    134.8924265789232

```python
x.result()
```

```python

```


We want to try and find the Jacobians of the GR shallow water equations.

```python
import sympy
sympy.init_printing()
```

```python
M, J, Q = sympy.symbols('M, J, Q', positive=True)
t, r = sympy.symbols('t, r', positive=True)
th, ph = sympy.symbols('theta, phi')

coords = t, r, th, ph

Ph = sympy.symbols('Phi', positive=True)
u, v, ut, al = sympy.symbols('u, v, u^t, alpha')
bx, by = sympy.symbols('beta^x, beta^y')

ud, vd = sympy.symbols('u_i, v_i')

vsq = u**2 + v**2
#W = al*ut
W = 1 / sympy.sqrt(1 - ud*u + vd*v)
```

```python
w = Ph, u, v, ut
U = Ph*W, Ph*W**2*u, Ph*W**2*v, Ph*W**2*al*(ud * (u - bx/al) + vd * (v - by/al))
F_u = Ph*W*(u - bx/al), Ph*W**2*ud*(u - bx/al) + Ph, Ph*W**2*vd*(u - bx/al), Ph*W**2*(u - bx/al)*(ud*bx + vd*by - al)
```

```python
for x in U:
    print(x.diff(Ph))

```

    1/sqrt(-u*u_i + v*v_i + 1)
    u/(-u*u_i + v*v_i + 1)
    v/(-u*u_i + v*v_i + 1)
    alpha*(u_i*(u - beta^x/alpha) + v_i*(v - beta^y/alpha))/(-u*u_i + v*v_i + 1)

```python
for x in U:
    print(x.diff(u))
```

    Phi*u_i/(2*(-u*u_i + v*v_i + 1)**(3/2))
    Phi*u*u_i/(-u*u_i + v*v_i + 1)**2 + Phi/(-u*u_i + v*v_i + 1)
    Phi*u_i*v/(-u*u_i + v*v_i + 1)**2
    Phi*alpha*u_i*(u_i*(u - beta^x/alpha) + v_i*(v - beta^y/alpha))/(-u*u_i + v*v_i + 1)**2 + Phi*alpha*u_i/(-u*u_i + v*v_i + 1)

```python
for x in U:
    print(x.diff(v))
```

    -Phi*v_i/(2*(-u*u_i + v*v_i + 1)**(3/2))
    -Phi*u*v_i/(-u*u_i + v*v_i + 1)**2
    -Phi*v*v_i/(-u*u_i + v*v_i + 1)**2 + Phi/(-u*u_i + v*v_i + 1)
    -Phi*alpha*v_i*(u_i*(u - beta^x/alpha) + v_i*(v - beta^y/alpha))/(-u*u_i + v*v_i + 1)**2 + Phi*alpha*v_i/(-u*u_i + v*v_i + 1)

```python
for x in U:
    print(x.diff(ut))
```

    Phi*alpha
    2*Phi*alpha**2*u*u^t
    2*Phi*alpha**2*u^t*v
    2*Phi*alpha**3*u^t*(u_i*(u - beta^x/alpha) + v_i*(v - beta^y/alpha))

```python
for x in F_u:
    print(x.diff(Ph))
```

    alpha*u^t*(u - beta^x/alpha)
    alpha**2*u^t**2*u_i*(u - beta^x/alpha) + 1
    alpha**2*u^t**2*v_i*(u - beta^x/alpha)
    alpha**2*u^t**2*(u - beta^x/alpha)*(-alpha + beta^x*u_i + beta^y*v_i)

```python
for x in F_u:
    print(x.diff(ut))
```

    Phi*alpha*(u - beta^x/alpha)
    2*Phi*alpha**2*u^t*u_i*(u - beta^x/alpha)
    2*Phi*alpha**2*u^t*v_i*(u - beta^x/alpha)
    2*Phi*alpha**2*u^t*(u - beta^x/alpha)*(-alpha + beta^x*u_i + beta^y*v_i)

```python
for x in F_u:
    print(x.diff(u))
```

    Phi*u_i*(u - beta^x/alpha)/(2*(-u*u_i + v*v_i + 1)**(3/2)) + Phi/sqrt(-u*u_i + v*v_i + 1)
    Phi*u_i**2*(u - beta^x/alpha)/(-u*u_i + v*v_i + 1)**2 + Phi*u_i/(-u*u_i + v*v_i + 1)
    Phi*u_i*v_i*(u - beta^x/alpha)/(-u*u_i + v*v_i + 1)**2 + Phi*v_i/(-u*u_i + v*v_i + 1)
    Phi*u_i*(u - beta^x/alpha)*(-alpha + beta^x*u_i + beta^y*v_i)/(-u*u_i + v*v_i + 1)**2 + Phi*(-alpha + beta^x*u_i + beta^y*v_i)/(-u*u_i + v*v_i + 1)

```python
for x in F_u:
    print(x.diff(v))
```

    -Phi*v_i*(u - beta^x/alpha)/(2*(-u*u_i + v*v_i + 1)**(3/2))
    -Phi*u_i*v_i*(u - beta^x/alpha)/(-u*u_i + v*v_i + 1)**2
    -Phi*v_i**2*(u - beta^x/alpha)/(-u*u_i + v*v_i + 1)**2
    -Phi*v_i*(u - beta^x/alpha)*(-alpha + beta^x*u_i + beta^y*v_i)/(-u*u_i + v*v_i + 1)**2

```python
ud = u
bx = 0
W = 1 / sympy.sqrt(1 - ud*u)
dudw = sympy.zeros(3,3)
dudw[0,0] = W
dudw[0,1] = Ph*ud*W**3
dudw[0,2] = Ph*al

dudw[1,0] = W**2*ud
dudw[1,1] = Ph*W**2*(1 + 2*ud*ud*W**2)
dudw[1,2] = 2*Ph*al*W*ud

dudw[2,0] = W**2*al*ud*(u-bx/al)
dudw[2,1] = Ph*W**2*al*((1 + 2*W**2*ud*ud)*(u-bx/al) + ud)
dudw[2,2] = 2*Ph*al**2*W*ud*(u-bx/al)
```

```python
dudw
```

```python
dudw_inv = dudw.inv()
dudw_inv.simplify()
```

```python
dudw_inv
```

```python
dfdw = sympy.zeros(3,3)

dfdw[0,0] = W*(u-bx/al)
dfdw[0,1] = Ph*W*(1 + W**2*ud*(u-bx/al))
dfdw[0,2] = Ph*al*(u-bx/al)

dfdw[1,0] = W**2*ud*(u-bx/al) + 1
dfdw[1,1] = Ph*W**2*((1 + 2*W**2*ud*u)*(u-bx/al) + u)
dfdw[1,2] = 2*Ph*al*W*ud *(u-bx/al)

dfdw[2,0] = W**2*(u-bx/al)*(ud*bx - al)
dfdw[2,1] = Ph*W**2*((u-bx/al)*(bx + 2*W**2*u*(ud*bx-al)) + (ud*bx-al))
dfdw[2,2] = 2*Ph*al*W*(u-bx/al)*(ud*bx - al)

dfdw
```

```python
A = dfdw*dudw_inv
```

```python
A.simplify()
```

```python
evals = A.eigenvals()
```

```python
P, D = A.diagonalize()
```

```python
print(evals)
```

    {-((-3*u**2 + 2)**2/u**2 - 3/u**2)/(3*(-1/2 + sqrt(3)*I/2)*(sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)) - (-1/2 + sqrt(3)*I/2)*(sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)/3 - (-3*u**2 + 2)/(3*u): 1, -((-3*u**2 + 2)**2/u**2 - 3/u**2)/(3*(sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)) - (sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)/3 - (-3*u**2 + 2)/(3*u): 1, -((-3*u**2 + 2)**2/u**2 - 3/u**2)/(3*(-1/2 - sqrt(3)*I/2)*(sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)) - (-1/2 - sqrt(3)*I/2)*(sqrt(-4*((-3*u**2 + 2)**2/u**2 - 3/u**2)**3 + (27*(-2*u**2 + 1)/u + 2*(-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/u**3)**2)/2 + 27*(-2*u**2 + 1)/(2*u) + (-3*u**2 + 2)**3/u**3 - 9*(-3*u**2 + 2)/(2*u**3))**(1/3)/3 - (-3*u**2 + 2)/(3*u): 1}

```python
for x in evals:
    print(x.simplify())
```

    (2**(2/3)*u**2*((-54*u**4 + u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 54*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(2/3)*(1 - sqrt(3)*I)**2 + 4*u*((u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 27*u**2*(-2*u**2 + 1) + 27*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(1/3)*(1 - sqrt(3)*I)*(3*u**2 - 2) + 8*2**(1/3)*((3*u**2 - 2)**2 - 3))/(12*u**2*((u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 27*u**2*(-2*u**2 + 1) + 27*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(1/3)*(1 - sqrt(3)*I))
    -3*u**2/(-27*u**3 + 27*u + 3*sqrt(6)*sqrt(27*u**4 - 54*u**2 + 36 - 8/u**2 + u**(-4)) - 9/u - 1/u**3)**(1/3) + u - (-27*u**3 + 27*u + 3*sqrt(6)*sqrt(27*u**4 - 54*u**2 + 36 - 8/u**2 + u**(-4)) - 9/u - 1/u**3)**(1/3)/3 + 4/(-27*u**3 + 27*u + 3*sqrt(6)*sqrt(27*u**4 - 54*u**2 + 36 - 8/u**2 + u**(-4)) - 9/u - 1/u**3)**(1/3) - 2/(3*u) - 1/(3*u**2*(-27*u**3 + 27*u + 3*sqrt(6)*sqrt(27*u**4 - 54*u**2 + 36 - 8/u**2 + u**(-4)) - 9/u - 1/u**3)**(1/3))
    (2**(2/3)*u**2*((-54*u**4 + u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 54*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(2/3)*(1 + sqrt(3)*I)**2 + 4*u*((u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 27*u**2*(-2*u**2 + 1) + 27*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(1/3)*(1 + sqrt(3)*I)*(3*u**2 - 2) + 8*2**(1/3)*((3*u**2 - 2)**2 - 3))/(12*u**2*((u**3*sqrt((-4*((-3*u**2 + 2)**2 - 3)**3 + (27*u**2*(-2*u**2 + 1) + 27*u**2 + 2*(-3*u**2 + 2)**3 - 18)**2)/u**6) + 27*u**2*(-2*u**2 + 1) + 27*u**2 - 2*(3*u**2 - 2)**3 - 18)/u**3)**(1/3)*(1 + sqrt(3)*I))

```python
A
```

```python
dudw.det()
```

```python
evals.simplify()
```

```python
B = sympy.zeros(3,3)
B[0,0] = u
B[0,1] = -1/W
B[0,2] = 1/(al*u*W)
B[1,0] = 2/W
B[1,1] = 2*u - 2/u
B[1,2] = 1/(al*u**2)
B[2,1] = -al
B
```

```python
w = sympy.symbols('w')
B = B.subs(W,w)
```

```python
PB, DB = B.diagonalize()
```

```python
dudw = sympy.zeros(3,3)
dudw[0,0] = w
dudw[0,1] = Ph*ud*w**3
dudw[0,2] = Ph*vd*w**3

dudw[1,0] = w**2*ud
dudw[1,1] = Ph*w**2*(1+2*ud*ud*w**2)
dudw[1,2] = Ph*w**2*2*ud*vd*w**2

dudw[2,0] = w**2*vd
dudw[2,1] = Ph*w**2*2*ud*vd*w**2
dudw[2,2] = Ph*w**2*(1+2*vd**2*w**2)

dudw
```

```python
dudw.det()
```

```python
dudw_inv = dudw.inv()
```

```python
# set bi = 0
dfdw = sympy.zeros(3,3)

dfdw[0,0] = w*(u-bx)
dfdw[0,1] = Ph*w*(1+w**2*ud*(u-bx))
dfdw[0,2] = Ph*w*w**2*vd*(u-bx)

dfdw[1,0] = w**2*ud*(u-bx) + 1
dfdw[1,1] = Ph*w**2*((u-bx)*(1+2*w**2*ud*u)+u)
dfdw[1,2] = Ph*w**2*((u-bx)*(vd*u*w**2))

dfdw[2,0] = w**2*vd*(u-bx)
dfdw[2,1] = Ph*w**2*((u-bx)*(2*w**2*ud*v))
dfdw[2,2] = Ph*w**2*((u-bx)*(1+2*w**2*vd*v)+u)

dfdw
```

```python
a = dfdw * dudw_inv
```

```python
a.simplify()
```

```python
a
```

```python
evals = a.eigenvals()
```

```python
for x in evals:
    print(x.simplify())
```

```python

```


# Assignment 6

This assignment has weighting $3.5$.
The first question about clustering has 35%, and the second question about tiny image classification has 65%.

This is a challenging assignment, so I recommend you start early.

# Clustering for handwritten digits

Supervised learning requires labeled data, which can be expensive to acquire.
For example, a dataset with $N$ samples for classification will require manual labeling $N$ times.

One way to ameliorate this issue is to perform clustering of the raw data samples first, followed by manual inspection and labeling of only a few samples.
Recall that clustering is a form of non-supervised learning, so it does not require any class labels.

For example, say we are given a set of scanned hand-written digit images.
We can cluster them into 10 groups first, manually inspect and label a few images in each cluster, and propagate the label towards the rest of all (unlabeled) samples in each cluster.

The accuracy of such semi-automatic labeling depends on the accuracy of the clustering.
If each cluster (0 to 9) corresponds exactly to hand-written digits 0-9, we are fine.
Otherwise, we have some mis-labeled data.

The goal of this question is to exercise clustering of the scikit-learn digits dataset which has labels, so that we can verify our clustering accuracy.
The specifics are as follows.

You will be judged by the test accuracy of your code, and quality of descriptions of your method.
As a reference, a simple code I (Li-Yi) wrote can achieve about 78% accuracy. Try to beat it as much as you can.

## Training and test data split

We will split the original dataset into training and test datasets
* training for building our clusters
* testing to see if the clusters can predict future data

## Accuracy
What is your clustering accuracy (comparing cluster labels with the ground truth labels), and what are the properties of mis-clustered samples?

## Data preprocessing
Would the original features (pixels) work well, or we need further processing like scaling/standardization or dimensionality-reduction, before clustering?

## Models and hyper-parameters

Let's focus on k-means clustering, as hierarchical and density-based clustering do not provide the predict() method under scikit-learn.

What is the best test performance you can achieve with which hyper-parameters (for k-means, standard scalar, and dimensionality reduction)?

### Hint
We have learned Pipeline and GridSearchCV for cross validation and hyper-parameter tuning.

```python
%load_ext watermark
%watermark -a '' -u -d -v -p numpy,pandas,matplotlib,scipy,sklearn
%matplotlib inline
```

    last updated: 2016-12-14

    CPython 3.5.2
    IPython 5.1.0

    numpy 1.11.2
    pandas 0.19.1
    matplotlib 1.5.3
    scipy 0.18.1
    sklearn 0.18.1

```python
# Added version check for recent scikit-learn 0.18 checks
from distutils.version import LooseVersion as Version
from sklearn import __version__ as sklearn_version
```

## Load data

```python
import numpy as np

from sklearn.datasets import load_digits
digits = load_digits()

X = digits.data # data in pixels
y = digits.target # digit labels

print(X.shape)
print(y.shape)

print(np.unique(y))
```

    (1797, 64)
    (1797,)
    [0 1 2 3 4 5 6 7 8 9]

## Visualize data

```python
import matplotlib.pyplot as plt
import pylab as pl

num_rows = 4
num_cols = 5

fig, ax = plt.subplots(nrows=num_rows, ncols=num_cols, sharex=True, sharey=True)
ax = ax.flatten()
for index in range(num_rows*num_cols):
    img = digits.images[index]
    label = digits.target[index]
    ax[index].imshow(img, cmap='Greys', interpolation='nearest')
    ax[index].set_title('digit ' + str(label))

ax[0].set_xticks([])
ax[0].set_yticks([])
plt.tight_layout()
plt.show()
```

## Data sets: training versus test

```python
if Version(sklearn_version) < '0.18':
    from sklearn.cross_validation import train_test_split
else:
    from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=1)

num_training = y_train.shape[0]
num_test = y_test.shape[0]
print('training: ' + str(num_training) + ', test: ' + str(num_test))
```

    training: 1257, test: 540

```python
import numpy as np

# check to see if the data are well distributed among digits
for y in [y_train, y_test]:
    print(np.bincount(y))
```

    [119 133 128 119 120 135 130 122 128 123]
    [59 49 49 64 61 47 51 57 46 57]

## Answer

We first write a scoring function for clustering so that we can use for GridSearchCV.
Take a look at use_scorer under scikit learn.

```python
## Note: We do not guarantee that there is a one-to-one correspondence, and therefore the toy result is different.
##       See Explanation for more information
def clustering_accuracy_score(y_true, y_pred):
    n_labels = len(list(set(y_true)))
    n_clusters = len(list(set(y_pred)))
    Pre = np.zeros((n_clusters, n_labels))
    Rec = np.zeros((n_clusters, n_labels))
    F = np.zeros((n_clusters, n_labels))
    w = np.zeros((n_clusters))
    F_i = np.zeros((n_clusters))
    P = np.zeros((n_labels))
    C = np.zeros((n_clusters))

    for i in range(n_clusters):
        C[i] = sum(y_pred == i)
    for j in range(n_labels):
        P[j] = sum(y_true == j)

    for i in range(n_clusters):
        F_i_max = 0
        for j in range(n_labels):
            if (C[i]):
                Pre[i][j] = sum(y_pred[y_true == j] == i) / C[i]
            if (P[j]):
                Rec[i][j] = sum(y_true[y_pred == i] == j) / P[j]
            if (Pre[i][j]+Rec[i][j]):
                F[i][j] = 2*Pre[i][j]*Rec[i][j]/(Pre[i][j]+Rec[i][j])
            F_i_max = max(F_i_max, F[i][j])
        F_i[i] = F_i_max
        w[i] = sum(y_pred == i) / len(y_pred)
    return F_i.dot(w)
```

```python
# toy case demonstrating the clustering accuracy
# this is just a reference to illustrate what this score function is trying to achieve
# feel free to design your own as long as you can justify

# ground truth class label for samples
toy_y_true = np.array([0, 0, 0, 1, 1, 2])

# clustering id for samples
toy_y_pred_true = np.array([1, 1, 1, 2, 2, 0])
toy_y_pred_bad1 = np.array([0, 0, 1, 1, 1, 2])
toy_y_pred_bad2 = np.array([2, 2, 1, 0, 0, 0])

toy_accuracy = clustering_accuracy_score(toy_y_true, toy_y_pred_true)
print('accuracy', toy_accuracy, ', should be 1')

toy_accuracy = clustering_accuracy_score(toy_y_true, toy_y_pred_bad1)
print('accuracy', toy_accuracy, ', should be', 5.0/6.0)

toy_accuracy = clustering_accuracy_score(toy_y_true, toy_y_pred_bad2)
print('accuracy', toy_accuracy, ', should be', 4.0/6.0, ', this will be explained in the following content')
```

    accuracy 1.0 , should be 1
    accuracy 0.833333333333 , should be 0.8333333333333334
    accuracy 0.75 , should be 0.6666666666666666 , this will be explained in the following content

## Explanation

I adopt a modified version of F-value selection, that is, for each cluster, select the best label class with highest F-score. This accuracy calculating method supports the condition that number of clusters not equal to number of labels, which supports GridSearchCV on number of clusters.

Formula: Let $C[i]$ denotes cluster $i$ and $P[j]$ denotes label $j$. Then for each $(i, j)$, we have

$$
\begin{align}
\text{Precision}[i][j] & = \frac{\left|C[i]\cap P[j]\right|}{|C[i]|} \\
\text{Recall}[i][j] & = \frac{\left|C[i]\cap P[j]\right|}{|P[j]|} \\
\text{F-value}[i][j] & = \frac{ 2 \times \text{Precision}[i][j] \times \text{Recall}[i][j]}{\text{Precision}[i][j] + \text{Recall}[i][j]}.
\end{align}
$$

Then for each cluster, we search the best F-value for each label, that is,
$$\text{F-value}[i] = \max\limits_{j} \text{F-value}[i][j].$$
We also store the weight $w$ for each cluster, that is,
$$w[i] = \frac{|C[i]|}{n}$$

Hence the final score is simply the dot product of $\text{F-value}$ and $w$, which is the weighted F-score.

Again, since this accuracy calculating method supports the condition that number of clusters not equal to number of labels, we __do not__ guarantee that there is a bijection between clusters and labels, therefore there are cases that some labels have no corresponding clusters, as the second toy example shows.

Build a pipeline with standard scaler, PCA, and clustering.

```python
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import KernelPCA
from sklearn.cluster import KMeans
from sklearn.metrics import make_scorer
from scipy.stats import mode

pipe = Pipeline([('scl', StandardScaler()),
                 ('pca', KernelPCA()),
                 ('km', KMeans(random_state=1))])

# map cluster number to acutal label
def cluster_mapping(y_true, y_pred):
    mapping = {}
    n_labels = len(list(set(y_true)))
    n_clusters = len(list(set(y_pred)))
    Pre = np.zeros((n_clusters, n_labels))
    Rec = np.zeros((n_clusters, n_labels))
    F = np.zeros((n_clusters, n_labels))
    P = np.zeros((n_labels))
    C = np.zeros((n_clusters))

    for i in range(n_clusters):
        C[i] = sum(y_pred == i)
    for j in range(n_labels):
        P[j] = sum(y_true == j)

    for i in range(n_clusters):
        F_i_max = 0
        F_i_max_label = 0
        for j in range(n_labels):
            if (C[i]):
                Pre[i][j] = sum(y_pred[y_true == j] == i) / C[i]
            if (P[j]):
                Rec[i][j] = sum(y_true[y_pred == i] == j) / P[j]
            if (Pre[i][j]+Rec[i][j]):
                F[i][j] = 2*Pre[i][j]*Rec[i][j]/(Pre[i][j]+Rec[i][j])
            if (F_i_max < F[i][j]):
                F_i_max_label = j
                F_i_max = F[i][j]
        mapping[i] = F_i_max_label
    return mapping
```

Use GridSearchCV to tune hyper-parameters.

```python
if Version(sklearn_version) < '0.18':
    from sklearn.grid_search import GridSearchCV
else:
    from sklearn.model_selection import GridSearchCV

pcs = list(range(1, 60))
kernels = ['linear', 'rbf', 'cosine']
initTypes = ['random', 'k-means++']
clusters = list(range(10, 20))
tfs = [True, False]

param_grid = [{'pca__n_components': pcs,
               'pca__kernel': kernels,
               'km__init' : initTypes,
               'km__n_clusters' : clusters,
               'scl__with_std' : tfs,
               'scl__with_mean' : tfs}]

gs = GridSearchCV(estimator=pipe,
                  param_grid=param_grid,
                  scoring=make_scorer(clustering_accuracy_score),
                  cv=10,
                  n_jobs=-1,
                  verbose=False)

gs = gs.fit(X_train, y_train)
print(gs.best_score_)
print(gs.best_params_)
```

    0.824823005662
    {'pca__n_components': 15, 'scl__with_mean': True, 'pca__kernel': 'cosine', 'km__init': 'k-means++', 'km__n_clusters': 11, 'scl__with_std': False}

```python
best_model = gs.best_estimator_

print('Test accuracy: %.3f' % clustering_accuracy_score(y_test, best_model.predict(X_test)))
```

    Test accuracy: 0.844

Visualize mis-clustered samples, and provide your explanation.

```python
mapping = cluster_mapping(y_train, best_model.predict(X_train))

y_test_pred = np.array(list(map(lambda x: mapping[x], best_model.predict(X_test))))
miscl_img = X_test[y_test != y_test_pred][:25]
correct_lab = y_test[y_test != y_test_pred][:25]
miscl_lab = y_test_pred[y_test != y_test_pred][:25]

fig, ax = plt.subplots(nrows=5, ncols=5, sharex=True, sharey=True)
ax = ax.flatten()
for i in range(25):
    img = miscl_img[i].reshape(8, 8)
    ax[i].imshow(img, cmap='Greys', interpolation='nearest')
    ax[i].set_title('%d) t: %d p: %d' % (i+1, correct_lab[i], miscl_lab[i]))

ax[0].set_xticks([])
ax[0].set_yticks([])
plt.tight_layout()
plt.show()
```

## Explanation
Since the accuracy is 84.4%, which means more than 1 digit will be incorrectly clustered in a group of 10 digits, the error is still considered to be high (compared with using neural networks or other methods).
The mis-clustered samples, as we can observe from the picture above, are generally two kinds:
1. Hard to differentiate: e.g. No. 14, No. 21 and No, 22. It is even difficult for a human to tell which classes they should belong to.
2. Blurred to much: e.g. No. 4, No. 18 and No. 19. Those can be differentiated by a human but seems too vague to be correctly clustered.

Furthermore, we can find that digit '5' tends to be mis-clustered to digit '9' (e.g. No. 5, No. 17 and No. 18), and digit '9' tends to be mis-clustered to digit '1' and '7' (e.g. No. 1, No. 2, No. 8, No. 19, No. 23 and No. 24). The reason may be that the distance between those digit clusters are not far, and a digit lying around the border can be easily mis-clustered. For example, when the digit is blurred/vague.

Also, we can find from No. 10, No. 14, No. 15 that the digit '1' in the dataset are sometimes twisted, tilted or blurred. It's even hard for a human to tell whether it is digit '1' or not. From the examples, we can see that they tend to be clustered as digit '6' or '2' with respect to their shape and level of clarity.

# Tiny image classification

We will use the [CIFAR-10 dataset](https://www.cs.toronto.edu/~kriz/cifar.html) for image object recognition.
The dataset consists of 50000 training samples and 10000 test samples in 10 different classes (airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck; see the link above for more information).
The goal is to maximize the accuracy of your classifier on the test dataset after being optimized via the training dataset.

You can use any learning models (supervised or unsupervised) or optimization methods (e.g. search methods for hyper-parameters).
The only requirement is that your code can run inside an ipynb file, as usual.
Please provide a description of your method, in addition to the code.
Your answer will be evaluated not only on the test accuracy but also on the creativity of your methodology and the quality of your explanation/description.

# Description

## 1. Introduction

### 1.1. High-level approach
I adopted a Wide Residual Network with 28 convolutional layers and 10x width. As stated Zagoruyko et al's paper [1](#myfootnote1), a 96.11% testing accuracy can be obtained.

### 1.2. Reason
Without cifar-10 data size-augmentation[2](#myfootnote2), one of the best test results - a 93.57% testing accuracy is reported[3](#myfootnote3) with the famous deep residual netwook[4](#myfootnote4). Since the desired testing accuracy is high, I decided to start from resnet. However, inspired by Zagoruyko et al, I choose to use a modified version of resent - the Wide Residual Network.

Wide resent basically increases the channel of each convolutional layer while decreasing the depth of the network. With result data from their work, a structure of 28 convolution layers and 10x width with dropout achieves the best result - 96.11% accuracy.

When training my network, however, I only found the result of their early version's paper[5](#myfootnote5), which is training without dropout layer achieves better accuracy (which is also modified from 95.61% to 96.00% in the lastest version). Moreover, due to assignment time constraint, I simply use the default adadelta optimizer instead of [sgd + selected learning rate] in the first 100 itertaions, which may accounts for my less accurate result. After finetuning, in the end, I only get a 94.14% accuarcy, which is still higher than the original resnet.

## 2. Method

### 2.1 Data Preprocessing

Using `keras.preprocessing.image.ImageDataGenerator`, the data preprocessing scheme I use is
1. `featurewise_center`: Set feature-wise input mean to 0 over the dataset
2. `featurewise_std_normalization`: Divide inputs by feature-wise std of the dataset
3. `rotation_range`: 10$^{\circ}$ range for random rotations
4. `width_shift_range`: 15.625% random horizontal shifts
5. `height_shift_range`: 15.625% random vertical shifts
6. `horizontal_flip`: Randomly flip inputs horizontally

Since `vertical_flip` looks unrealistic for a real-world picture, I did not use it. Also, `zca_whitening` may results in loss of image info, and other methods may more or less lead to problems in training.

Moreover, I did not adopte cifar-10 data size-augmentation[2](#myfootnote2), which requires structural changes to my network, which may results in more time cost in training and finetuning. As the paper suggests, the test accuracy should increase compared with non-size-augmentated data.

### 2.2 Model

Since we use 28 convolutional layers, by structure of wide resnet, the layer blocks are:
1. A initial block with 1 convolutional layers
2. 4 conv1 block with 160 channels. All blocks contains 2 convolutional layers. The first block contains one additional residual convolutional layer. In total 3+2+2+2=9 convolutional layers. Each block ends with a residual merge.
3. 4 conv2 block with 320 channels. All blocks contains 2 convolutional layers. The first block contains one additional residual convolutional layer. In total 3+2+2+2=9 convolutional layers. Each block ends with a residual merge.
4. 4 conv3 block with 640 channels. All blocks contains 2 convolutional layers. The first block contains one additional residual convolutional layer. In total 3+2+2+2=9 convolutional layers. Each block ends with a residual merge.

All convolutional layers are followed by a batch normalization layer and a relu layer.

After all layer blocks, we use average pooling to 8-by-8, followed by a flatten layer, and finalliy a fully connected layer (with softmax) of 10 outputs, which corresponds to each class.

### 2.3 Hyper-parameter

As discussed in 1.2, the most important hyper-parameter of wide resnet, also as Zagoruyko et al discribed, is N (the number of layers), K (the width of the convolutional channel), and dropout ratio. I adopted the best result from version 1 of their work, that is, N = 28, K = 10 (times original resnet), dropout ratio = 0. However, as their version 2 points out, this model with dropout can achieve a better result. Since the difference is just 0.11%, we can see that dropout may not influence much in wide resnet.

### 2.4 Detailed Model

You may view the detailed model in the following code output.

## 3. Result

| Network | Accuracy |
|------------------------------------------|--------|
| Kaiming He et al Resent | 93.57% |
| My WRN-28-10 w/o dropout (Adadelta only) | 93.69% |
| My WRN-28-10 w/o dropout (Fine-tuned) | 94.14% |
| Zagoruyko et al WRN-28-10 v1 w/ dropout | 95.61% |
| Zagoruyko et al WRN-28-10 v1 w/o dropout | 95.83% |
| Zagoruyko et al WRN-28-10 v2 w/o dropout | 96.00% |
| Zagoruyko et al WRN-28-10 v2 w/ dropout | 96.11% |

With adadelta optimizer only, my training accuracy finally becomes 100%, which denotes an over-fitting and thus making the testing accuracy fail to improve. With SGD fine-tuning on the model, the testing accuracy increases around 0.45%. However, my model fail to achieve a testing accuracy above 95% as the papaer suggestes. See 4.2 Limitations for more analysis.

Note that the code I provided below simply uses the fine-tuned result of my network, and the training process is just for showing the result. They are not the original processes.

## 4. Conclusion

### 4.1 Traning Conclusion
1. Wide Residual Network achieves better result than the original Residual Network.
2. Optimizer choice, fine-tuning and data augmentation are curtial for improving the testing rate.
3. Ideas that worked several years ago may not work nowadays, ideas that work on a specific model may not work on another model (e.g. importance of dropout, improving the depth always work on resnet)
4. The importance of keeping track of the latest news and models on machine learning.

### 4.2 Limitations
1. Optimizer, Regulation and Over-fitting:
With more time provided, I would try to use [different optimizer (adagrad, RMSprop, adam, sgd+specific training rate) + different Regulation (l1, l2)] instead of using adadelta only. This may account for the 1.5% accuracy below the paper's result.
2. Dropout:
As the second version of the paper suggested, with dropout the testing accuracy improved 0.11%. However, this should not be the major contraint of the result.
3. Fine-tuning:
I only ran 50 iterations of fine-tuning based on the training rate I set. The peak testing accuracy I obtained is 94.49% but was not saved during fine-tuning.
4. Data Augmentation:
Cifar-10 data size-augmentation[2](#myfootnote2) is proved to be useful to improve the testing rate, which I did not try during my training.
5. FCN:
Fully Convolutional Network[6](#myfootnote6) is proved to be useful to improve the testing rate. Also, R-FCN[7](#myfootnote7) provides a nice implimentation of Fully Convolutional Resnet. Maybe a wide, fully convolved residual network can provides a better result.
6. Use of GPU:
AWS refused to provide me with a GPU instance since I am a new user. I trained my network on my Nvidia GeForce GTX 980M, which takes 7 days to run 100 iterations. This heavily limited adjustments to my model.

## 5. References
1: [Zagoruyko, Sergey, and Nikos Komodakis. "Wide Residual Networks." arXiv preprint arXiv:1605.07146v2 (2016).](https://arxiv.org/pdf/1605.07146v2.pdf)

2: [Graham, Benjamin. "Fractional max-pooling." arXiv preprint arXiv:1412.6071 (2015).](https://arxiv.org/pdf/1412.6071v4.pdf)

3: [Benenson. What is the class of this image ?" rodrigob@github](http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html)

4: [He, Kaiming, et al. "Deep residual learning for image recognition." arXiv preprint arXiv:1512.03385 (2015).](https://arxiv.org/pdf/1512.03385v1.pdf)

5: [Zagoruyko, Sergey, and Nikos Komodakis. "Wide Residual Networks." arXiv preprint arXiv:1605.07146v1 (2016).](https://arxiv.org/pdf/1605.07146v1.pdf)

6: [Long, Jonathan, Evan Shelhamer, and Trevor Darrell. "Fully convolutional networks for semantic segmentation." IEEE Conference on CVPR. 2015.](http://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Long_Fully_Convolutional_Networks_2015_CVPR_paper.pdf)

7: [Dai, Jifeng, et al. "R-FCN: Object Detection via Region-based Fully Convolutional Networks." arXiv preprint arXiv:1605.06409 (2016).](https://arxiv.org/pdf/1605.06409.pdf)

All codes referenced have been specified in their context.

```python
# Functions to build a user-defined wide resnet for cifa-10
# Author: Somshubra Majumdar https://github.com/titu1994/Wide-Residual-Networks
# Modified By: Gao Chang, HKU

from keras.models import Model
from keras.layers import Input, merge, Activation, Dropout, Flatten, Dense
from keras.layers.convolutional import Convolution2D, MaxPooling2D, AveragePooling2D
from keras.layers.normalization import BatchNormalization
from keras import backend as K

def initial_conv(input):
    x = Convolution2D(16, 3, 3, border_mode='same')(input)

    channel_axis = 1 if K.image_dim_ordering() == "th" else -1

    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)
    return x

def conv1_block(input, k=1, dropout=0.0):
    init = input

    channel_axis = 1 if K.image_dim_ordering() == "th" else -1

    # Check if input number of filters is same as 16 * k, else create convolution2d for this input
    if K.image_dim_ordering() == "th":
        if init._keras_shape[1] != 16 * k:
            init = Convolution2D(16 * k, 1, 1, activation='linear', border_mode='same')(init)
    else:
        if init._keras_shape[-1] != 16 * k:
            init = Convolution2D(16 * k, 1, 1, activation='linear', border_mode='same')(init)

    x = Convolution2D(16 * k, 3, 3, border_mode='same')(input)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    if dropout > 0.0: x = Dropout(dropout)(x)

    x = Convolution2D(16 * k, 3, 3, border_mode='same')(x)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    m = merge([init, x], mode='sum')
    return m

def conv2_block(input, k=1, dropout=0.0):
    init = input

    channel_axis = 1 if K.image_dim_ordering() == "th" else -1

    # Check if input number of filters is same as 32 * k, else create convolution2d for this input
    if K.image_dim_ordering() == "th":
        if init._keras_shape[1] != 32 * k:
            init = Convolution2D(32 * k, 1, 1, activation='linear', border_mode='same')(init)
    else:
        if init._keras_shape[-1] != 32 * k:
            init = Convolution2D(32 * k, 1, 1, activation='linear', border_mode='same')(init)

    x = Convolution2D(32 * k, 3, 3, border_mode='same')(input)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    if dropout > 0.0: x = Dropout(dropout)(x)

    x = Convolution2D(32 * k, 3, 3, border_mode='same')(x)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    m = merge([init, x], mode='sum')
    return m

def conv3_block(input, k=1, dropout=0.0):
    init = input

    channel_axis = 1 if K.image_dim_ordering() == "th" else -1

    # Check if input number of filters is same as 64 * k, else create convolution2d for this input
    if K.image_dim_ordering() == "th":
        if init._keras_shape[1] != 64 * k:
            init = Convolution2D(64 * k, 1, 1, activation='linear', border_mode='same')(init)
    else:
        if init._keras_shape[-1] != 64 * k:
            init = Convolution2D(64 * k, 1, 1, activation='linear', border_mode='same')(init)

    x = Convolution2D(64 * k, 3, 3, border_mode='same')(input)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    if dropout > 0.0: x = Dropout(dropout)(x)

    x = Convolution2D(64 * k, 3, 3, border_mode='same')(x)
    x = BatchNormalization(axis=channel_axis)(x)
    x = Activation('relu')(x)

    m = merge([init, x], mode='sum')
    return m

def WRN(nb_classes, N, k, dropout):
    """
    Creates a Wide Residual Network with specified parameters
    :param nb_classes: Number of output classes
    :param N: Depth of the network. Compute N = (n - 4) / 6.
              Example : For a depth of 16, n = 16, N = (16 - 4) / 6 = 2
              Example2: For a depth of 28, n = 28, N = (28 - 4) / 6 = 4
              Example3: For a depth of 40, n = 40, N = (40 - 4) / 6 = 6
    :param k: Width of the network.
    :param dropout: Adds dropout if value is greater than 0.0
    """
    init = Input(shape=(3, 32, 32))

    x = initial_conv(init)

    for i in range(N):
        x = conv1_block(x, k, dropout)

    x = MaxPooling2D((2,2))(x)

    for i in range(N):
        x = conv2_block(x, k, dropout)

    x = MaxPooling2D((2,2))(x)

    for i in range(N):
        x = conv3_block(x, k, dropout)

    x = AveragePooling2D((8,8))(x)
    x = Flatten()(x)
    x = Dense(nb_classes, activation='softmax')(x)

    model = Model(init, x)
    return model
```

    Using Theano backend.
    Using gpu device 0: GeForce GTX 980M (CNMeM is disabled, cuDNN 5005)

```python
import numpy as np
import sklearn.metrics as metrics
from keras.datasets import cifar10
from keras.models import Model
from keras.layers import Input
from keras.optimizers import SGD
import keras.callbacks as callbacks
import keras.utils.np_utils as kutils
from keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import accuracy_score

batch_size = 64
nb_epoch = 5

(X_train, y_train), (X_test, y_test) = cifar10.load_data()

X_train = X_train.astype('float32')
X_train /= 255.0
X_test = X_test.astype('float32')
X_test /= 255.0

y_train = kutils.to_categorical(y_train)
y_test = kutils.to_categorical(y_test)

generator = ImageDataGenerator(featurewise_center=True,
                               featurewise_std_normalization=True,
                               rotation_range=10,
                               width_shift_range=5./32,
                               height_shift_range=5./32,
                               horizontal_flip=True)

generator.fit(X_train, seed=0, augment=True)

test_generator = ImageDataGenerator(featurewise_center=True,
                                    featurewise_std_normalization=True)

test_generator.fit(X_test, seed=0, augment=True)

model = WRN(nb_classes=10, N=4, k=10, dropout=0.0)

model.summary()
```

    ____________________________________________________________________________________________________
    Layer (type)                     Output Shape          Param #     Connected to
    ====================================================================================================
    input_1 (InputLayer)             (None, 3, 32, 32)     0
    ____________________________________________________________________________________________________
    convolution2d_1 (Convolution2D)  (None, 16, 32, 32)    448         input_1[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_1 (BatchNormal(None, 16, 32, 32)    32          convolution2d_1[0][0]
    ____________________________________________________________________________________________________
    activation_1 (Activation)        (None, 16, 32, 32)    0           batchnormalization_1[0][0]
    ____________________________________________________________________________________________________
    convolution2d_3 (Convolution2D)  (None, 160, 32, 32)   23200       activation_1[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_2 (BatchNormal(None, 160, 32, 32)   320         convolution2d_3[0][0]
    ____________________________________________________________________________________________________
    activation_2 (Activation)        (None, 160, 32, 32)   0           batchnormalization_2[0][0]
    ____________________________________________________________________________________________________
    convolution2d_4 (Convolution2D)  (None, 160, 32, 32)   230560      activation_2[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_3 (BatchNormal(None, 160, 32, 32)   320         convolution2d_4[0][0]
    ____________________________________________________________________________________________________
    convolution2d_2 (Convolution2D)  (None, 160, 32, 32)   2720        activation_1[0][0]
    ____________________________________________________________________________________________________
    activation_3 (Activation)        (None, 160, 32, 32)   0           batchnormalization_3[0][0]
    ____________________________________________________________________________________________________
    merge_1 (Merge)                  (None, 160, 32, 32)   0           convolution2d_2[0][0]
                                                                       activation_3[0][0]
    ____________________________________________________________________________________________________
    convolution2d_5 (Convolution2D)  (None, 160, 32, 32)   230560      merge_1[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_4 (BatchNormal(None, 160, 32, 32)   320         convolution2d_5[0][0]
    ____________________________________________________________________________________________________
    activation_4 (Activation)        (None, 160, 32, 32)   0           batchnormalization_4[0][0]
    ____________________________________________________________________________________________________
    convolution2d_6 (Convolution2D)  (None, 160, 32, 32)   230560      activation_4[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_5 (BatchNormal(None, 160, 32, 32)   320         convolution2d_6[0][0]
    ____________________________________________________________________________________________________
    activation_5 (Activation)        (None, 160, 32, 32)   0           batchnormalization_5[0][0]
    ____________________________________________________________________________________________________
    merge_2 (Merge)                  (None, 160, 32, 32)   0           merge_1[0][0]
                                                                       activation_5[0][0]
    ____________________________________________________________________________________________________
    convolution2d_7 (Convolution2D)  (None, 160, 32, 32)   230560      merge_2[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_6 (BatchNormal(None, 160, 32, 32)   320         convolution2d_7[0][0]
    ____________________________________________________________________________________________________
    activation_6 (Activation)        (None, 160, 32, 32)   0           batchnormalization_6[0][0]
    ____________________________________________________________________________________________________
    convolution2d_8 (Convolution2D)  (None, 160, 32, 32)   230560      activation_6[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_7 (BatchNormal(None, 160, 32, 32)   320         convolution2d_8[0][0]
    ____________________________________________________________________________________________________
    activation_7 (Activation)        (None, 160, 32, 32)   0           batchnormalization_7[0][0]
    ____________________________________________________________________________________________________
    merge_3 (Merge)                  (None, 160, 32, 32)   0           merge_2[0][0]
                                                                       activation_7[0][0]
    ____________________________________________________________________________________________________
    convolution2d_9 (Convolution2D)  (None, 160, 32, 32)   230560      merge_3[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_8 (BatchNormal(None, 160, 32, 32)   320         convolution2d_9[0][0]
    ____________________________________________________________________________________________________
    activation_8 (Activation)        (None, 160, 32, 32)   0           batchnormalization_8[0][0]
    ____________________________________________________________________________________________________
    convolution2d_10 (Convolution2D) (None, 160, 32, 32)   230560      activation_8[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_9 (BatchNormal(None, 160, 32, 32)   320         convolution2d_10[0][0]
    ____________________________________________________________________________________________________
    activation_9 (Activation)        (None, 160, 32, 32)   0           batchnormalization_9[0][0]
    ____________________________________________________________________________________________________
    merge_4 (Merge)                  (None, 160, 32, 32)   0           merge_3[0][0]
                                                                       activation_9[0][0]
    ____________________________________________________________________________________________________
    maxpooling2d_1 (MaxPooling2D)    (None, 160, 16, 16)   0           merge_4[0][0]
    ____________________________________________________________________________________________________
    convolution2d_12 (Convolution2D) (None, 320, 16, 16)   461120      maxpooling2d_1[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_10 (BatchNorma(None, 320, 16, 16)   640         convolution2d_12[0][0]
    ____________________________________________________________________________________________________
    activation_10 (Activation)       (None, 320, 16, 16)   0           batchnormalization_10[0][0]
    ____________________________________________________________________________________________________
    convolution2d_13 (Convolution2D) (None, 320, 16, 16)   921920      activation_10[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_11 (BatchNorma(None, 320, 16, 16)   640         convolution2d_13[0][0]
    ____________________________________________________________________________________________________
    convolution2d_11 (Convolution2D) (None, 320, 16, 16)   51520       maxpooling2d_1[0][0]
    ____________________________________________________________________________________________________
    activation_11 (Activation)       (None, 320, 16, 16)   0           batchnormalization_11[0][0]
    ____________________________________________________________________________________________________
    merge_5 (Merge)                  (None, 320, 16, 16)   0           convolution2d_11[0][0]
                                                                       activation_11[0][0]
    ____________________________________________________________________________________________________
    convolution2d_14 (Convolution2D) (None, 320, 16, 16)   921920      merge_5[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_12 (BatchNorma(None, 320, 16, 16)   640         convolution2d_14[0][0]
    ____________________________________________________________________________________________________
    activation_12 (Activation)       (None, 320, 16, 16)   0           batchnormalization_12[0][0]
    ____________________________________________________________________________________________________
    convolution2d_15 (Convolution2D) (None, 320, 16, 16)   921920      activation_12[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_13 (BatchNorma(None, 320, 16, 16)   640         convolution2d_15[0][0]
    ____________________________________________________________________________________________________
    activation_13 (Activation)       (None, 320, 16, 16)   0           batchnormalization_13[0][0]
    ____________________________________________________________________________________________________
    merge_6 (Merge)                  (None, 320, 16, 16)   0           merge_5[0][0]
                                                                       activation_13[0][0]
    ____________________________________________________________________________________________________
    convolution2d_16 (Convolution2D) (None, 320, 16, 16)   921920      merge_6[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_14 (BatchNorma(None, 320, 16, 16)   640         convolution2d_16[0][0]
    ____________________________________________________________________________________________________
    activation_14 (Activation)       (None, 320, 16, 16)   0           batchnormalization_14[0][0]
    ____________________________________________________________________________________________________
    convolution2d_17 (Convolution2D) (None, 320, 16, 16)   921920      activation_14[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_15 (BatchNorma(None, 320, 16, 16)   640         convolution2d_17[0][0]
    ____________________________________________________________________________________________________
    activation_15 (Activation)       (None, 320, 16, 16)   0           batchnormalization_15[0][0]
    ____________________________________________________________________________________________________
    merge_7 (Merge)                  (None, 320, 16, 16)   0           merge_6[0][0]
                                                                       activation_15[0][0]
    ____________________________________________________________________________________________________
    convolution2d_18 (Convolution2D) (None, 320, 16, 16)   921920      merge_7[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_16 (BatchNorma(None, 320, 16, 16)   640         convolution2d_18[0][0]
    ____________________________________________________________________________________________________
    activation_16 (Activation)       (None, 320, 16, 16)   0           batchnormalization_16[0][0]
    ____________________________________________________________________________________________________
    convolution2d_19 (Convolution2D) (None, 320, 16, 16)   921920      activation_16[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_17 (BatchNorma(None, 320, 16, 16)   640         convolution2d_19[0][0]
    ____________________________________________________________________________________________________
    activation_17 (Activation)       (None, 320, 16, 16)   0           batchnormalization_17[0][0]
    ____________________________________________________________________________________________________
    merge_8 (Merge)                  (None, 320, 16, 16)   0           merge_7[0][0]
                                                                       activation_17[0][0]
    ____________________________________________________________________________________________________
    maxpooling2d_2 (MaxPooling2D)    (None, 320, 8, 8)     0           merge_8[0][0]
    ____________________________________________________________________________________________________
    convolution2d_21 (Convolution2D) (None, 640, 8, 8)     1843840     maxpooling2d_2[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_18 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_21[0][0]
    ____________________________________________________________________________________________________
    activation_18 (Activation)       (None, 640, 8, 8)     0           batchnormalization_18[0][0]
    ____________________________________________________________________________________________________
    convolution2d_22 (Convolution2D) (None, 640, 8, 8)     3687040     activation_18[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_19 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_22[0][0]
    ____________________________________________________________________________________________________
    convolution2d_20 (Convolution2D) (None, 640, 8, 8)     205440      maxpooling2d_2[0][0]
    ____________________________________________________________________________________________________
    activation_19 (Activation)       (None, 640, 8, 8)     0           batchnormalization_19[0][0]
    ____________________________________________________________________________________________________
    merge_9 (Merge)                  (None, 640, 8, 8)     0           convolution2d_20[0][0]
                                                                       activation_19[0][0]
    ____________________________________________________________________________________________________
    convolution2d_23 (Convolution2D) (None, 640, 8, 8)     3687040     merge_9[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_20 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_23[0][0]
    ____________________________________________________________________________________________________
    activation_20 (Activation)       (None, 640, 8, 8)     0           batchnormalization_20[0][0]
    ____________________________________________________________________________________________________
    convolution2d_24 (Convolution2D) (None, 640, 8, 8)     3687040     activation_20[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_21 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_24[0][0]
    ____________________________________________________________________________________________________
    activation_21 (Activation)       (None, 640, 8, 8)     0           batchnormalization_21[0][0]
    ____________________________________________________________________________________________________
    merge_10 (Merge)                 (None, 640, 8, 8)     0           merge_9[0][0]
                                                                       activation_21[0][0]
    ____________________________________________________________________________________________________
    convolution2d_25 (Convolution2D) (None, 640, 8, 8)     3687040     merge_10[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_22 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_25[0][0]
    ____________________________________________________________________________________________________
    activation_22 (Activation)       (None, 640, 8, 8)     0           batchnormalization_22[0][0]
    ____________________________________________________________________________________________________
    convolution2d_26 (Convolution2D) (None, 640, 8, 8)     3687040     activation_22[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_23 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_26[0][0]
    ____________________________________________________________________________________________________
    activation_23 (Activation)       (None, 640, 8, 8)     0           batchnormalization_23[0][0]
    ____________________________________________________________________________________________________
    merge_11 (Merge)                 (None, 640, 8, 8)     0           merge_10[0][0]
                                                                       activation_23[0][0]
    ____________________________________________________________________________________________________
    convolution2d_27 (Convolution2D) (None, 640, 8, 8)     3687040     merge_11[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_24 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_27[0][0]
    ____________________________________________________________________________________________________
    activation_24 (Activation)       (None, 640, 8, 8)     0           batchnormalization_24[0][0]
    ____________________________________________________________________________________________________
    convolution2d_28 (Convolution2D) (None, 640, 8, 8)     3687040     activation_24[0][0]
    ____________________________________________________________________________________________________
    batchnormalization_25 (BatchNorma(None, 640, 8, 8)     1280        convolution2d_28[0][0]
    ____________________________________________________________________________________________________
    activation_25 (Activation)       (None, 640, 8, 8)     0           batchnormalization_25[0][0]
    ____________________________________________________________________________________________________
    merge_12 (Merge)                 (None, 640, 8, 8)     0           merge_11[0][0]
                                                                       activation_25[0][0]
    ____________________________________________________________________________________________________
    averagepooling2d_1 (AveragePoolin(None, 640, 1, 1)     0           merge_12[0][0]
    ____________________________________________________________________________________________________
    flatten_1 (Flatten)              (None, 640)           0           averagepooling2d_1[0][0]
    ____________________________________________________________________________________________________
    dense_1 (Dense)                  (None, 10)            6410        flatten_1[0][0]
    ====================================================================================================
    Total params: 36489290
    ____________________________________________________________________________________________________

```python
print ("Start Training:")

sgd = SGD(lr = 0.001, decay = 0.1, momentum = 0.9, nesterov = True)

model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["acc"])

# model.load_weights("WRN-28-10.h5")

model.fit_generator(generator.flow(X_train, y_train, batch_size=batch_size),
                    samples_per_epoch=len(X_train),
                    nb_epoch=nb_epoch,
#                     callbacks=[callbacks.ModelCheckpoint("WRN-28-10-Best.h5", monitor="val_acc", save_best_only=True)],
                    validation_data=test_generator.flow(X_test, y_test, batch_size=batch_size),
                    nb_val_samples=X_test.shape[0],
                    verbose = True)
```

    Start Training:
    Epoch 1/5
    50000/50000 [==============================] - 5208s - loss: 3.1280e-04 - acc: 1.0000 - val_loss: 0.3443 - val_acc: 0.9409
    Epoch 2/5
    50000/50000 [==============================] - 5009s - loss: 0.0023 - acc: 0.9992 - val_loss: 0.3471 - val_acc: 0.9406
    Epoch 3/5
    50000/50000 [==============================] - 5013s - loss: 0.0080 - acc: 0.9972 - val_loss: 0.3442 - val_acc: 0.9408
    Epoch 4/5
    50000/50000 [==============================] - 5013s - loss: 0.0086 - acc: 0.9970 - val_loss: 0.3287 - val_acc: 0.9444
    Epoch 5/5
    50000/50000 [==============================] - 5002s - loss: 0.0080 - acc: 0.9972 - val_loss: 0.3495 - val_acc: 0.9393

```python
print ("Start Testing:")

# model.load_weights("WRN-28-10.h5")
results = model.evaluate_generator(test_generator.flow(X_test, y_test, batch_size=batch_size), X_test.shape[0])

print ("Results:")
print ("Test loss: {0}".format(results[0]))
print ("Test accuracy: {0}".format(results[1]))
```

    Start Testing:
    Results:
    Test loss: 0.3487206139005721
    Test accuracy: 0.9414