```
from __future__ import division

import numpy as np
import matplotlib.pyplot as plt

import pydy
import sympy as sym
import sympy.physics.mechanics as me

from scipy.linalg import solve, lstsq
from scipy.integrate import odeint

import time
import functools

me.Vector.simp = False  # to increase computation speed

from sympy.physics.vector import init_vprinting
init_vprinting()

from matplotlib import animation
from matplotlib.patches import Ellipse, Rectangle

from IPython.html import widgets
from IPython.html.widgets import interact, interactive

# %matplotlib inline
```

```
from IPython.display import Image
Image(filename='snake-schematic.png')
```

```
# number of links (not currently used)
n = 3

# dynamic variables
q0, q1, q2 = me.dynamicsymbols('q0 q1 q2')
u0, u1, u2 = me.dynamicsymbols('u0 u1 u2')

q3, q4 = me.dynamicsymbols('q3 q4')
u3, u4 = me.dynamicsymbols('u3 u4')

# variables for bodies
g, t = sym.symbols('g t')
m0, m1, m2 = sym.symbols('m_0 m_1 m_2')
l0, l1, l2 = sym.symbols('l_0 l_1 l_2')
Izz0, Izz1, Izz2 = sym.symbols('I_zz_0 I_zz_1 I_zz_2')

# inertial reference frame
frN = me.ReferenceFrame('fr_N')

# body-fixed frames
fr0 = frN.orientnew('fr_0', 'Axis', [q0, frN.z])
fr1 = fr0.orientnew('fr_1', 'Axis', [q1, fr0.z])
fr2 = fr1.orientnew('fr_2', 'Axis', [q2, fr1.z])

# angular velocities of frames
fr0.set_ang_vel(frN, u0 * frN.z)
fr1.set_ang_vel(fr0, u1 * fr0.z)
fr2.set_ang_vel(fr1, u2 * fr1.z)

# origin of inertial frame
Nor = me.Point('N_or')

# joints
jt0 = Nor.locatenew('jt_0', q3 * frN.x + q4 * frN.y)
jt1 = jt0.locatenew('jt_1', l0 * fr0.x)
jt2 = jt1.locatenew('jt_2', l1 * fr1.x)
jt3 = jt2.locatenew('jt_3', l2 * fr2.x)

# mass centers
mc0 = jt0.locatenew('mc_0', l0 / 2 * fr0.x)
mc1 = jt1.locatenew('mc_1', l1 / 2 * fr1.x)
mc2 = jt2.locatenew('mc_2', l2 / 2 * fr2.x)

# constrained point velocities
Nor.set_vel(frN, 0)
jt0.set_vel(frN, u3 * frN.x + u4 * frN.y)

# unconstrained point velocities
jt1.v2pt_theory(jt0, frN, fr0)
jt2.v2pt_theory(jt1, frN, fr1)
jt3.v2pt_theory(jt2, frN, fr2)

mc0.v2pt_theory(jt0, frN, fr0)
mc1.v2pt_theory(jt1, frN, fr1)
mc2.v2pt_theory(jt2, frN, fr2)

# inertia dyads
in0 = (me.inertia(fr0, 0, 0, Izz0), mc0)
in1 = (me.inertia(fr1, 0, 0, Izz1), mc1)
in2 = (me.inertia(fr2, 0, 0, Izz2), mc2)

# bodies
bd0 = me.RigidBody('bd_0', mc0, fr0, m0, in0)
bd1 = me.RigidBody('bd_1', mc1, fr1, m1, in1)
bd2 = me.RigidBody('bd_2', mc2, fr2, m2, in2)

# kinematic differential equations
kd = [q0.diff(t) - u0,
      q1.diff(t) - u1,
      q2.diff(t) - u2,
      q3.diff(t) - u3,
      q4.diff(t) - u4]

# forces
fl = [(mc0, -m0 * g * frN.z),
      (mc1, -m1 * g * frN.z),
      (mc2, -m2 * g * frN.z)]

# fl = [(mc0, -m0 * g * frN.y),
#       (mc1, -m1 * g * frN.y),
#       (mc2, -m2 * g * frN.y)]

# bodies
bl = [bd0, bd1, bd2]

# q_ind and u_ind
coords = [q0, q1, q2, q3, q4]
speeds = [u0, u1, u2, u3, u4]

# # try adding some configuration level and configuration velocity constraints
# zero = jt0.pos_from(Nor)
# conlist_coor = [zero & frN.x, zero & frN.y]

# # Define the velocity level constraints
# dzero = me.time_derivative(zero, frN)
# conlist_speed = [dzero & frN.x, dzero & frN.y]

# q3, q4 = me.dynamicsymbols('q3 q4')
# u3, u4 = me.dynamicsymbols('u3 u4')

# equations of motion using Kane's method
kane = me.KanesMethod(frame=frN,
                      q_ind=coords,
                      u_ind=speeds,
                      kd_eqs=kd,
                      q_dependent=[],
                      configuration_constraints=[],
                      u_dependent=[],
                      velocity_constraints=[])

(fr, frstar) = kane.kanes_equations(fl, bl)
kanezero = fr + frstar

# these take to long...
# solve Kane's equations for the derivatives of the speeds
#eom = sym.solvers.solve(kanezero, q0d, q1d, q2d)

# add the kinematical differential equations to get the equations of motion
#eom.update(kane.kindiffdict())

# print the results
#me.mprint(eom)

#rhs = kane.mass_matrix.inv() * kane.forcing
#me.mprint(rhs)

#j3pos = jt3.pos_from(jt0).express(frN).simplify()
#sym.trigsimp(kanezero)
```

```
now = time.time()
#eom = sym.solvers.solve(kanezero, q1, q2)
eom = kane.rhs()
print('Elapsed time: {0:.2f} sec'.format(time.time() - now))
```

    Elapsed time: 0.33 sec

```
#eom
```

```
kanezero
```

```
kz = sym.trigsimp(kanezero)
```

```
type(kz)
```

    sympy.matrices.immutable.ImmutableMatrix

```
kz[0]
```

```
kz[3]
```

```
sym.trigsimp(frstar[3])
```

```

```

```

```

```
# parameter values
ls = .25                          # segment lengths
ms = .15                          # segment masses
ws_ratio = .1                     # width is a function of length
ws = ls * ws_ratio                # segment widths
Izzs = ms * (ls**2 + ws**2) / 12  # flat plate about com

# parameter arrays for numerical simulation
param_names = [g, m0, l0, Izz0, m1, l1, Izz1, m2, l2, Izz2]
param_vals = [9.81, ms, ls, Izzs, ms, ls, Izzs, ms, ls, Izzs]

# dummy symbols used for dynamamic variables
dyn_symbols = coords + speeds
dummy_symbols = [sym.Dummy() for i in dyn_symbols]
dummy_dict = dict(zip(dyn_symbols, dummy_symbols))

# substitute dummy symbols into mass matrix and forcing vector
kd_dict = kane.kindiffdict()                              # solved kinematical differential equations
F = kane.forcing_full.subs(kd_dict).subs(dummy_dict)      # substitute into the forcing vector
M = kane.mass_matrix_full.subs(kd_dict).subs(dummy_dict)  # substitute into the mass matrix

# callable matrices
F_func = sym.lambdify(param_names + dummy_symbols, F)
M_func = sym.lambdify(param_names + dummy_symbols, M)

# partially evaluate matrices to run faster (maybe 15% or so)
ffunc = functools.partial(F_func, *param_vals)
mfunc = functools.partial(M_func, *param_vals)

def rhs(x, t, args):
    arguments = np.hstack((args, x))
    dx = np.array(solve(M_func(*arguments), F_func(*arguments))).flatten()
    return dx

def rhs_partial(x, t, args):

    #x[1] = np.interp(t, ts, theta1)
    #x[2] = np.interp(t, ts, theta2)

    dx = np.array(solve(mfunc(*x), ffunc(*x))).flatten()
    return dx

# initial conditions
qstart = (np.pi / 2, np.pi / 2, np.pi / 2, 2, 2)
ustart = (2, 0, 2, 0, 0)  # (0.5, 0.5, 0.5)
#ustart = (0, -1.5, 1.5)

# stable ICs
# qstart = (-np.pi / 2, 0, 0)
# ustart = (0, 0, 0)

ts, dt = linspace(0, 30, 5001, retstep=True)

# theta1 = np.cos(ts)
# theta2 = np.cos(ts + np.pi/8)
# qstart = (np.pi / 4, theta1[0], theta2[0])

x0 = np.hstack((qstart, ustart))
```

```
# numerically integrate
now = time.time()
y1 = odeint(rhs_partial, x0, ts, args=(None,), full_output=False)
y1 = y1.T
print('Elapsed time: {0:.2f} sec'.format(time.time() - now))
```

    Elapsed time: 1.28 sec

```
np.diff(y1[0]) / dt
```

    array([ 2.0090272 ,  2.02728354,  2.0458476 , ...,  3.48340694,
            3.48886136,  3.49414025])

```
u0 = y1[0]
dq0 = np.gradient(y1[0]) / dt
abs_diff = (u0 - dq0) / dq0
```

```
np.rad2deg((u0 - dq0).max())
```

```
fig, ax = plt.subplots()
ax.plot(abs_diff)
```

    [<matplotlib.lines.Line2D at 0x9fb4ad0>]

```
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6, 6))

ax1.plot(y1[0])
ax1.plot(y1[1])
ax1.plot(y1[2])

ax2.plot(y1[5])
ax2.plot(y1[6])
ax2.plot(y1[7])

# ax1.plot(y2[0])
# ax1.plot(y2[1])
# ax1.plot(y2[2])

# ax2.plot(y2[3])
# ax2.plot(y2[4])
# ax2.plot(y2[5])

# ax1.set_ylim(-15, 15)
# ax2.set_ylim(-50, 50)

ax2.set_xlabel('iteration')
ax1.set_ylabel('q', rotation='horizontal')
ax2.set_ylabel('u', rotation='horizontal')

fig.tight_layout()

#display(fig)
```

```
#%matplotlib inline

def plot_chain(ii=0, hst=100):

    fig, ax = plt.subplots(1, 1, sharex=True, sharey=True, figsize=(9, 9))

    ax.axis('image')
    ax.set_xlim(-.8, 2.5)
    ax.set_ylim(-4, 2.5)
    ax.set_ylim(0, 2.5)

    fig.tight_layout()

    time_text = ax.text(0.05, 0.9, '{0:.3} sec'.format(ts[ii]), transform=ax.transAxes)

    ellipse_params = dict(alpha=.5, edgecolor='None', color='green')
    e0 = Ellipse(xy=c0[ii], width=ws, height=1.05 * ls, angle=dg0[ii] - 90, **ellipse_params)
    e1 = Ellipse(xy=c1[ii], width=ws, height=1.05 * ls, angle=dg1[ii] - 90, **ellipse_params)
    e2 = Ellipse(xy=c2[ii], width=ws, height=1.05 * ls, angle=dg2[ii] - 90, **ellipse_params)
    [ax.add_patch(e) for e in [e0, e1, e2]]

    st = ii - hst
    #st if st > 0 else 0
    if st < 0:
        st = 0

    ax.plot(j1[st:ii, 0], j1[st:ii, 1], 'b--')
    ax.plot(j2[st:ii, 0], j2[st:ii, 1], 'g--')
    ax.plot(j3[st:ii, 0], j3[st:ii, 1], 'r--')

    ax.plot(c0[st:ii, 0], c0[st:ii, 1], 'b')
    ax.plot(c1[st:ii, 0], c1[st:ii, 1], 'g')
    ax.plot(c2[st:ii, 0], c2[st:ii, 1], 'r')

    ax.plot(com[st:ii, 0], com[st:ii, 1], 'k-', lw=2, alpha=.9)
```

```
plt.close('all')
```

```
# organize the data
th0, th1, th2 = y1[:3]
th1 = th1 + th0
th2 = th2 + th1

dg0, dg1, dg2 = np.rad2deg(y1[:3])
dg1 = dg1 + dg0
dg2 = dg2 + dg1

j0 = np.c_[0, 0]
j0 = y1[3:5].T
j1 = j0 + np.c_[ls   * np.cos(th0), ls   * np.sin(th0)]
j2 = j1 + np.c_[ls   * np.cos(th1), ls   * np.sin(th1)]
j3 = j2 + np.c_[ls   * np.cos(th2), ls   * np.sin(th2)]
c0 = j0 + np.c_[ls/2 * np.cos(th0), ls/2 * np.sin(th0)]
c1 = j1 + np.c_[ls/2 * np.cos(th1), ls/2 * np.sin(th1)]
c2 = j2 + np.c_[ls/2 * np.cos(th2), ls/2 * np.sin(th2)]

# center of mass
mms = np.array([ms, ms, ms])
cx = np.c_[c0[:, 0], c1[:, 0], c2[:, 0]]
cy = np.c_[c0[:, 1], c1[:, 1], c2[:, 1]]
com = np.c_[(cx * mms).sum(axis=1), (cy * mms).sum(axis=1)] / mms.sum()

# make it interactive!
interactive(plot_chain, ii=(0, len(y1.T)-1), hst=(0, 500, 10))
```

```
fig, ax = plt.subplots()

ax.plot(j0[:, 0], j0[:, 1], 'r')
ax.plot(j3[:, 0], j3[:, 1], 'g')
ax.plot(com[:, 0], com[:, 1], 'k')
```

    [<matplotlib.lines.Line2D at 0x6df0510>]

```
M = 3 * ms
1 / M * np.sum()
```

```
fig, ax = plt.subplots(1, 1, sharex=True, sharey=True, figsize=(6, 6))

fig.tight_layout()

# display the current time
time_text = ax.text(0.05, 0.9, 'time here', transform=ax.transAxes)

# create a rectangular cart
ell = [Ellipse(xy=(0, 0), width=.05*ls, height=1.05 * ls, angle=100 * np.random.random(),
                    alpha=.5, edgecolor='None', color='green')
            for i in range(n)]
[ax.add_patch(ellis) for ellis in ell]

lim = 3 * ls
ax.set_xlim(-lim, lim)
ax.set_ylim(-lim, lim)

th0, th1, th2 = y1[:3]
dg0, dg1, dg2 = np.rad2deg(y1[:3])

j0 = np.c_[0, 0]
c0 = j0 + np.c_[ls/2 * np.cos(th0), ls/2 * np.sin(th0)]

j1 = j0 + np.c_[ls   * np.cos(th0), ls   * np.sin(th0)]
c1 = j1 + np.c_[ls/2 * np.cos(th1), ls/2 * np.sin(th1)]

j2 = j1 + np.c_[ls   * np.cos(th1), ls   * np.sin(th1)]
c2 = j2 + np.c_[ls/2 * np.cos(th2), ls/2 * np.sin(th2)]

j3 = j2 + np.c_[ls   * np.cos(th2), ls   * np.sin(th2)]

ii = 100

ell[0].center = c0[ii]
ell[1].center = c1[ii]
ell[2].center = c2[ii]

ell[0].angle = dg0[ii] - 90
ell[1].angle = dg1[ii] - 90
ell[2].angle = dg2[ii] - 90

ax.axis('image')
ax.set_xlim(-.7, .7)
ax.set_ylim(-.7, .7)

ax.plot(j1[:ii, 0], j1[:ii, 1], 'b--')
ax.plot(j2[:ii, 0], j2[:ii, 1], 'g--')
ax.plot(j3[:ii, 0], j3[:ii, 1], 'r--')

ax.plot(c0[:ii, 0], c0[:ii, 1], 'b')
ax.plot(c1[:ii, 0], c1[:ii, 1], 'g')
ax.plot(c2[:ii, 0], c2[:ii, 1], 'r')
```

    [<matplotlib.lines.Line2D at 0xdfbaf10>]

```
-dg2[:20] + 90
```

    array([ 0.        , -0.05208   , -0.09370436, -0.12483333, -0.14542379,
           -0.15542785, -0.15479371, -0.14346486, -0.12138054, -0.08847643,
           -0.0446831 ,  0.01007321,  0.0758705 ,  0.15279123,  0.24092226,
            0.34035504,  0.45118562,  0.5735147 ,  0.70744765,  0.85309458])

```
fig, ax = plt.subplots(figsize=(7, 7))
ax.plot(j3[:, 0], j3[:, 1], alpha=.4)
ax.plot(j2[:, 0], j2[:, 1], alpha=.7)
ax.plot(j1[:, 0], j1[:, 1], alpha=.7)
ax.plot(j0[:, 0], j0[:, 1], alpha=.7)

ax.axis('equal')

fig.tight_layout()
```

```
    # the number of pendulum bobs
    numpoints = states.shape[1] / 2

    # first set up the figure, the axis, and the plot elements we want to animate
    fig = plt.figure()

    # some dimesions
    cart_width = 0.4
    cart_height = 0.2

    # set the limits based on the motion
    xmin = around(states[:, 0].min() - cart_width / 2.0, 1)
    xmax = around(states[:, 0].max() + cart_width / 2.0, 1)

    # create the axes
    ax = plt.axes(xlim=(xmin, xmax), ylim=(-1.1, 1.1), aspect='equal')

    # display the current time
    time_text = ax.text(0.04, 0.9, '', transform=ax.transAxes)

    # create a rectangular cart
    rect = Rectangle([states[0, 0] - cart_width / 2.0, -cart_height / 2],
        cart_width, cart_height, fill=True, color='red', ec='black')
    ax.add_patch(rect)

    # blank line for the pendulum
    line, = ax.plot([], [], lw=2, marker='o', markersize=6)

    # initialization function: plot the background of each frame
    def init():
        time_text.set_text('')
        rect.set_xy((0.0, 0.0))
        line.set_data([], [])
        return time_text, rect, line,

    # animation function: update the objects
    def animate(i):
        time_text.set_text('time = {:2.2f}'.format(t[i]))
        rect.set_xy((states[i, 0] - cart_width / 2.0, -cart_height / 2))
        x = hstack((states[i, 0], zeros((numpoints - 1))))
        y = zeros((numpoints))
        for j in arange(1, numpoints):
            x[j] = x[j - 1] + length * cos(states[i, j])
            y[j] = y[j - 1] + length * sin(states[i, j])
        line.set_data(x, y)
        return time_text, rect, line,

    # call the animator function
    anim = animation.FuncAnimation(fig, animate, frames=len(t), init_func=init,
            interval=t[-1] / len(t) * 1000, blit=True, repeat=False)

    # save the animation if a filename is given
    if filename is not None:
        anim.save(filename, fps=30, codec='libx264')

##############

from matplotlib import animation
ani = animation.FuncAnimation(fig, updatefig, frames=1000, interval=75,
                              repeat=True, blit=True)

# video save information
metadata = dict(author='Isaac J. Yeaton',
                description='Standard and Otsu threholded images.')
Writer = animation.writers['avconv']
writer = Writer(fps=15)  # , extra_args=['-vcodec', 'libx264'], metadata=metadata)

if True:
    #ani.save('/home/isaac/Desktop/tmp1.mp4', writer=writer)
    ani.save('/home/isaac/Desktop/processed_shadowgraph_fields.mp4',
             writer=writer)
```

```

```

```
# check that the partial evaulation works
xx = x0
#xx = y.T[696]
arguments = np.hstack((param_vals, xx))

mm = M_func(*arguments)
ff = F_func(*arguments)

# this does not work...
# aargs = dict(zip(param_names, dummy_symbols, arguments))
# mm = M_func(*aargs)
# ff = F_func(*aargs)

print solve(mm, ff).flatten()
print lstsq(mm, ff)[0].flatten()

#part_args = dict(zip(param_names, param_vals))

print solve(mfunc(*xx), ffunc(*xx)).flatten()
```

    [  0.           0.           0.         -58.78688199  88.03408696
     -48.64785093]
    [  0.           0.           0.         -58.78688199  88.03408696
     -48.64785093]
    [  0.           0.           0.         -58.78688199  88.03408696
     -48.64785093]

```
frame0 = I.orientnew('frame0', 'Axis', [q[0], I.z])

mc0 = O.locatenew('mc0', q[1] * frame0.x)
mc0.v2pt_theory(O, I, frame0)

in0 = me.inertia(frame0, 0, 0, Izz[0])
body0 = me.RigidBody('body0', mc0, frame0, m[0], (in0, mc0))

frame1 = frame0.orientnew('frame1', 'Axis', [q[2], I.z])
frame2 = frame1.orientnew('frame2', 'Axis', [q[3], I.z])
frame1.set_ang_vel(I, u[2] * I.z)
frame2.set_ang_vel(I, u[3] * I.z)

in1 = me.inertia(frame1, 0, 0, Izz[1])
in2 = me.inertia(frame2, 0, 0, Izz[2])

# mc1 = mc0.locatenew('mc1', l[0] * frame0.x +
#                            l[1] * sym.cos(q[2]) * frame1.x +
#                            l[1] * sym.sin(q[2]) * frame1.y)
# mc2 = mc1.locatenew('mc2', l[1] * frame1.x +
#                            l[2] * sym.cos(q[3]) * frame2.x +
#                            l[2] * sym.sin(q[3]) * frame2.y)

# after looking at the n-pendulum example
mc1 = mc0.locatenew('mc1', l[0] * frame0.x + l[1] * frame1.x)
mc2 = mc1.locatenew('mc2', l[1] * frame1.x + l[2] * frame2.x)
mc1.v2pt_theory(mc0, I, frame1)
mc2.v2pt_theory(mc1, I, frame2)

body1 = me.RigidBody('body1', mc1, frame1, m[1], (in1, mc1))
body2 = me.RigidBody('body2', mc2, frame2, m[2], (in2, mc2))

# each one might need four entries...
frames = [I, frame0, frame1, frame2]
points = [O, mc0, mc1, mc2]
bodies = [body0, body1, body2]
forces = [0, 0, 0]
kindiffs = [qq.diff(t) - uu for qq, uu in zip(q, u)]

# need to figure out forces and how to setup me.KanesMethod

# frames = [I]                              # List to hold the n + 1 frames
# points = [P0]                             # List to hold the n + 1 points
# particles = [Pa0]                         # List to hold the n + 1 particles
# forces = [(P0, f * I.x - m[0] * g * I.y)] # List to hold the n + 1 applied forces, including the input force, f
# kindiffs = [q[0].diff(t) - u[0]]          # List to hold kinematic ODE's
# for i in range(n):
#     Bi = I.orientnew('B' + str(i), 'Axis', [q[i + 1], I.z])   # Create a new frame
#     Bi.set_ang_vel(I, u[i + 1] * I.z)                         # Set angular velocity
#     frames.append(Bi)                                         # Add it to the frames list

#     Pi = points[-1].locatenew('P' + str(i + 1), l[i] * Bi.x)  # Create a new point
#     Pi.v2pt_theory(points[-1], I, Bi)                         # Set the velocity
#     points.append(Pi)                                         # Add it to the points list

#     Pai = Particle('Pa' + str(i + 1), Pi, m[i + 1])           # Create a new particle
#     particles.append(Pai)                                     # Add it to the particles list

#     forces.append((Pi, -m[i + 1] * g * I.y))                  # Set the force applied at the point

#     kindiffs.append(q[i + 1].diff(t) - u[i + 1])              # Define the kinematic ODE:  dq_i / dt - u_i = 0
```


The main package used in this notebook is **CasADi**. It has automatic differentiacion capabilities, focuses in optimal control. Its following integration are used:
   - IDAS, for diferential algebraic equations
   - IPOPT for non linear optimization

```python
%matplotlib inline

from casadi import SX,DM,solve,substitute,kron, vec, trace, vertcat,\
                    horzcat, inv, reshape, Function, nlpsol, integrator,\
                    jacobian,hessian,exp,log,tanh,sqrt,det
import numpy as np
import matplotlib.pyplot as plt
from collections import namedtuple
from time import time

np.set_printoptions(precision=3)
```

```python
#SX in casadi stand for simbolic, and DM stands for numeric.
#I guess there is a easier way of doing this, but I don't know
SX2DM = lambda x: Function('f',[],[x],[],['x'])()['x']
```

```python
#state space type
ss = namedtuple('ss', ['A', 'B', 'C', 'D'])
```

## Conversion between transfer function to state space system

Here use the controllable canonical for to make the conversion of $G=(A,B,C,D)$.

$$G = \frac{b_0s^n+s^{n-1}b_1+\dots+sb_{n-1}+b_n}{s^n+s^{n-1}a_1+\dots+sa_{n-1}+a_n}$$

$$
\begin{equation}\begin{aligned}
A &=& \begin{bmatrix}
0 & 0 & \dots & 0 &-a_n\\
1 & 0 & \dots & 0 & -a_{n-1}\\
0 & 1 & \dots & 0 & -a_{n-2}\\
\vdots & \vdots & \ddots & \vdots & \vdots\\
0 & 0 & \dots & 1 & -a_1
\end{bmatrix}
B &=& \begin{bmatrix}
0 \\ 0 \\ \dots \\ 0 \\ 1
\end{bmatrix}\\
C &=& \begin{bmatrix}
b_n-a_nb_0 & \dots  & b_1-a_1b_0
\end{bmatrix}
D &=& \begin{bmatrix}b_0\end{bmatrix}
\end{aligned}\end{equation}
$$

```python
def tf2ss(b,a):
    """Conversion from tranfer function to state space system.

    Convert a continuous system from Laplace transfer function form to
    state space form. Uses the controllable canonic form realization of
    a transfer system given in polinomial fraction form, without delay.

    Parameters
    ----------
    param1 : b
        List of numerator coefficients from bigger to smaller s expoent.
    param2 : a
        List of denominator coefficients from bigger to smaller s expoent.

    Returns
    -------
    ss
        Namedtuple wiht A, B, C and D matrix of state space system.
    """
    n = a.shape[0]-1;
    #normalize by a[0] and pad b vector
    b,a = vertcat(SX(a.shape[0]-b.shape[0],1),b/a[0]), a/a[0]

    A,B,C,D = SX(n,n),SX(n,1),SX(1,n),SX(1,1)

    for i in range(n) : A[n-1,i] = -a[n-i]
    for i in range(n-1) : A[i,i+1] = 1
    B[n-1] = 1
    for i in range(n) : C[i] = b[n-i]-a[n-i]*b[0]
    D = b[0]

    return ss(A,B,C,D)
```

## Step of state space system

For get the matrix of step response of a MIMO state space system the IDAS for diferential algebraic equation is used. For each input variable is performed a simulation with a unitary step in that variable, while the other inputs are keeped in zero.

```python
def step(Gs,tf,dt=0.1,plot=True):
    """Calculates the matrix step response of a list of systems.

    Uses the controllable canonic form realization of a
    transfer system given in polinomial fraction form, without delay.

    Parameters
    ----------
    param1 : Gs
        List of system in state space form, or a sigle system.
    param2 : tf
        End time of simulation
    param3 : dt
        Time step
    param4 : plot
        Boolean indicatin if plot or return the simulated steps.

    Returns
    -------
    numpy.array
        'number of systems' X 'number of outputs' X 'number of inputs'
        X 'number of times' 4D numpy array with simulation.
    numpy.array
        1D array with times.
    """
    if isinstance(Gs,ss): Gs = [Gs]
    ny,nu = Gs[0].D.shape     # number of outputs and inputs
    ts = np.arange(0,tf,dt)   # array of times
    N = len(ts)               # size of array of times
    ys = np.zeros((len(Gs),ny,nu,N))  # array of outputs
    for k,G in enumerate(Gs): # for each system in the list
        A,B,C,D = G
        nx = A.shape[0]
        x,y = SX.sym('x',nx),SX.sym('y',ny)
        ys[k,:,:,0] = DM(D)
        for i in range(nu): # for each input, perform a unitary step
            dae = {'x':x, 'z':y, 'ode':A@x+B[:,i],'alg':y-C@x-D[:,i]}
            opts={'tf':dt}
            F = integrator('F','idas',dae,opts)
            x0 = DM.zeros(nx);
            for j in range(0,N-1): # perform time steps
                r = F(x0=x0)
                x0 = r['xf']
                ys[k,:,i,j+1] = np.array(r['zf'])[:,0]
    if plot: #plot unit step response matrix
        fig,axes = plt.subplots(ny, nu,
                       sharex='col', sharey='row', squeeze=False)
        for i,row in enumerate(axes):
            for j,ax in enumerate(row):
                ax.plot(ts,ys[:,i,j].T)
        plt.show()
    else:
        return ys,ts
```

### Test step

Test step function to system: $$G_n=\frac{-s+1}{s^2+s+1}$$

```python
#test step
G_n= tf2ss(b=SX([-1, 1]), a=SX([1,1,1]))
step(G_n,dt=0.1,tf=10)

```

## PI and PID tranfer function
$$PI \Rightarrow Kp+\frac{Ki}{s}=\frac{sKp+Ki}{s}$$
$$PID \Rightarrow Kp+\frac{Ki}{s}+Kd\frac{s}{\alpha s+1} = \frac{Kd(1+\alpha Kp)s^2+(Kp+\alpha Ki Kd)s+Ki}{\alpha Kd s^2+s}$$

```python
def pi(pc):
    """Creates PI controller from parameters in paralell form (Kp + Ki/s).

    Parameters
    ----------
    param1 : pc
        List of parameters: Kp, proportional gain, and Ki, integral gain.

    Returns
    -------
    ss
        Namedtuple wiht A, B, C and D matrix of state space controller
    """
    kp,ki = pc[0],pc[1]
    return tf2ss(vertcat(kp,ki),
                 vertcat(SX(1),SX(1,1)));

def pid(pc,alpha=0.1):
    """Creates PID controller from parameters (Kp+Ki/s+Kd*s/(alpha*s+1)).

    Parameters
    ----------
    param1 : pc
        List of parameters: Kp, proportional gain; Ki, integral gain;
        and Kd, derivative gain.
    param2 : alpha
        Derivative filter factor

    Returns
    -------
    ss
        Namedtuple wiht A, B, C and D matrix of state space controller
    """
    kp,ki,kd = pc[0],pc[1],pc[2]
    return tf2ss(vertcat(kd*(1+alpha*kp),kp+alpha*ki*kd,ki),
                 vertcat(alpha*kd,1,SX(1,1)))
```

```python
C = pid(DM([0.3,0.3,0.01]))
print(C)
```

    ss(A=SX(
    [[00, 1],
     [00, -1000]]), B=SX([00, 1]), C=SX([[300, -9999.7]]), D=SX(10.3))

## System transformation

### Series

Plant:

\begin{eqnarray}
\dot{x}	& = & Ax + Bu \\
y 		& = & Cx + Du
\end{eqnarray}

Controller:
\begin{eqnarray}
\dot{z}	& = & \tilde{A}z + \tilde{B}e  \\
u 		& = & \tilde{C}z + \tilde{D}e
\end{eqnarray}

Serie:
\begin{equation}\begin{aligned}
\begin{bmatrix}\dot{x}\\\dot{z}\end{bmatrix} & = \begin{bmatrix} A  & B\tilde{C} \\ 0 & \tilde{A} \end{bmatrix}\begin{bmatrix}x \\ z\end{bmatrix} + \begin{bmatrix} B\tilde{D} \\ \tilde{B}\end{bmatrix}e\\
y & =  \begin{bmatrix}C & D\tilde{C}\end{bmatrix}\begin{bmatrix}x \\ z\end{bmatrix}+D\tilde{D}e
\end{aligned}\end{equation}

```python
def series(G,C):
    """Computes the series system (GC) of Plant (G) and Controller (C),
    all in state space form.

    Parameters
    ----------
    param1 : G
        Plant in state space form
    param2 : C
        Controller in state space form

    Returns
    -------
    ss
        Resultant series system, in open loop.
    """
    return ss(vertcat(horzcat(G.A,G.B@C.C),
                      horzcat(SX(C.A.shape[0], G.A.shape[0]),C.A)),
              vertcat(G.B@C.D,C.B),
              horzcat(G.C,G.D@C.C),
              G.D@C.D)

```

### Feedback

Open-loop:
\begin{eqnarray}
\dot{x}	& = & Ax + Be \\
y 		& = & Cx + De
\end{eqnarray}

Doing:
$$e = r-y$$
$$F = (I+D)^{-1}$$

Close loop:
\begin{eqnarray}
\dot{x}	& = & (A-BFC)x + (BF)r \\
y 		& = & (C-DFC)x + (DF)r
\end{eqnarray}

```python
def feedback(G,C):
    """Computes the feebback system (inv(I+GC)GC) of Plant and Controller,
    all in state space form.

    Parameters
    ----------
    param1 : G
        Plant in state space form
    param2 : C
        Controller in state space form

    Returns
    -------
    ss
        Resultant feedback system, in open loop.
    """
    A,B,C,D = series(G,C)
    F = inv(SX.eye(D.shape[0])+D)
    return ss(A-B@F@C,B@F,C-D@F@C,D@F)
```

```python
T = feedback(G_n,C)
step(T,25)
```

# ISE Tunning

### Transform to $\dot{r}-e$ form

The H2 norm of the system in form $\dot{r}-e$ is the ISE for the unitary step scenario of the original system. That is why it is so important.

\begin{eqnarray}
\dot{x}	& = & Ax + Br \\
y 		& = & Cx + Dr
\end{eqnarray}

assuming $y = r$ in steady state and doing $w = \dot{x}$:

\begin{equation}\begin{aligned}
\dot{w}	& = & Aw + B\dot{r}  \\
e 		& = & -CA^{-1}w
\end{aligned}
\end{equation}

```python
def transform(T):
    """Converts a system from 'r-y' form to 'dr-e' form.

    Parameters
    ----------
    param1 : T
        Feedback state space system  in 'r-y' form, that is,
        the reference as input and the controlable variables as output.

    Returns
    -------
    ss
        Feedback state space system  in 'dr-e' form: the time derivative
        of reference as input and the follow-up error as output.
    """
    A,B,C,D = T
    return ss(A,B,-C@inv(A),SX(*D.shape))
```

## Analytical ISE calculation

### H2 norm calculation

Be a system $G$ given by this state space equations:

\begin{equation}\begin{aligned}
\dot{x}	& = & Ax + Bu  \\
y 		& = & Cx + Du
\end{aligned}
\end{equation}

Its H2 norm, assuming it is finite, can be calculated by the following equation:

\begin{equation}
\Vert G\Vert_2^2 =  \text{tr}(P)
\end{equation}

\begin{equation}
P = CQC^\top + DD^\top
\end{equation}

\begin{equation}
AQ+QA^\top+BB^\top=0\\
\end{equation}

\begin{equation}
(I_n\otimes A+A^\top\otimes I_n)vec(Q) = -vec(BB^\top)
\end{equation}

```python
def h2norm(G):
    """Calculates the H2 norm of a system in state space form,
    assuming it is finite.

    Parameters
    ----------
    param1 : G
        System in state space form

    Returns
    -------
    float
        H2 norm. If H2 norm is not finite, this value has no meaning.
    """
    A,B,C,D = G
    nx,nu = B.shape
    ny,nu = D.shape
    Q = reshape(solve(kron(SX.eye(nx),A)+kron(A,SX.eye(nx)),
                      -vec(B@B.T)),nx,nx)
    P = C@Q@C.T + D@D.T
    return trace(P)

def ISE(G,C=None):
    """Calculates analytical ISE of a MIMO state space system for a
    scenario of unit steps in the input.

    Uses the 'dr-e' form of the system, whose H2 norm is the desired ISE.

    Parameters
    ----------
    param1 : G
        System in state space form. If no controller provided,
        it is assumed that it its already in feedback.
    param2 : C
        Controller in state space form.

    Returns
    -------
    float
        ISE. If ISE is infinite, this value has no meaning.
    """
    if not C is None: G = feedback(G,C)
    return h2norm(transform(G))

def stable(G,C=None):
    """Test stability of feedback system, using eigvalues test.

    Parameters
    ----------
    param1 : G
        System in state space form. If no controller provided,
        it is assumed that it its already in feedback.
    param2 : C
        Controller in state space form.

    Returns
    -------
    bool
        True if the system is stable.
    """
    if not C is None: G = feedback(G,C)
    return np.all(np.real(np.linalg.eigvals(SX2DM(G.A)))< 0)

```

```python
#plot ISE contour varying Kp and Ki PI controller parameters
pc = SX.sym('pc',2)
fISE = Function('fISE',[pc],[ISE(G_n,pi(pc))],['pc'],['ISE'])
f = lambda x: max(min(x,10),-10) # function to limitate the output

N = 100
kps, kis = np.linspace(-1.5, 1.5, N), np.linspace(-.1, .6, N)
ISEs = np.array([[f(fISE([kp,ki])) for kp in kps] for ki in kis])

plt.contourf(kps,kis,ISEs,100,cmap=plt.cm.seismic),plt.colorbar()
plt.title('ISE Level curves'),plt.xlabel('Kp'),plt.ylabel('Ki')
plt.show()
```

## Minimize ISE

```python
opts = {'print_time':0,
        'ipopt':{'max_iter':100,'tol':1e-3,'print_level':0,'sb':'yes'}}

def minise(G,C,pc,pc0,pc_min,pc_max):
    """ise tunning. Find parameters of controller that minimize ISE
    of feedback system.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : C
        Controller in state space form.
    param2 : pc
        Symbolic controller parameters.
    param2 : pc0
        Initial .
    param2 : pc_min
        Controller parameters lower bound for optimization.
    param2 : pc_max
        Controller parameters upper bound for optimization.

    Returns
    -------
    dict
        'pc': optimal controller parameters, 'f': optimal ISE
    """
    res = nlpsol('S','ipopt',{'x':pc,'f':ISE(G,C),'g':pc},opts) \
            (x0=pc0,lbg=pc_min,ubg=pc_max)
    return {'pc':res['x'],'f':res['f']}

```

```python
# Test PI ise tunning
pc,pc0,pc_min,pc_max = SX.sym('pc',2),DM([.2,.2]),DM([.1,.1]),DM([1.,1.])
res_pi = minise(G_n,pi(pc),pc,pc0,pc_min,pc_max)
# Test PID ise tunning
pc,pc0 = SX.sym('pc',3),DM([.3,.3,.3])
pc_min,pc_max = DM([.1,.1,.1]),DM([1.,1.,1.])
res_pid = minise(G_n,pid(pc),pc,pc0,pc_min,pc_max)
print('PI ótimo:',res_pi['pc'],'\nPID ótimo:',res_pid['pc'])

#Systems to plot
Ts = [feedback(G_n,pi([0.2,0.2])),
      feedback(G_n,pi(res_pi['pc'])),
      feedback(G_n,pid(res_pid['pc'])),
      G_n]
ISEs = [SX2DM(ISE(T)).full()[0,0] for T in Ts]
step(Ts,tf=15)
```

```python
def sensitivity(f,x,dx=0.1,N=10):
    """Sensitivity analysis in function for each parameter.

    Parameters
    ----------
    param1 : f
        Function that receives a vector and return a number.
    param2 : x
        Point around which the sensitivity analysis will be performed.
    param3 : dx
        Number that especifies of change in the parameters.
    param4 : N
        Number of points to evaluate the function in both sides.
    """
    x = np.ravel(x)
    Mdx = np.diag(dx*x) #create matrix of changes in each parameter
    t = range(-N,N+1)
    y = np.array([[f(x+Mdx[i]*j/N) for j in t] for i in range(len(x))])
    plt.plot(t,y.T)
    percentage = '{:.2f}%'.format(100*dx)
    plt.xticks((-N, N),('-'+percentage, percentage)
    plt.xlabel('Percentage of parameter change')
    plt.show()
```

```python
# Apply sensitivity analysis to ISE around optimum PID controler
f = Function('f',[pc],[ISE(G_n,pid(pc))],['pc'],['ISE'])
plt.legend(['Kp','Ki','Kd'])
plt.title('ISE sensitivity analysis around optimum')
sensitivity(f,res_pid['pc'],0.5)
[H, g] = substitute(hessian(ISE(G_n,pid(pc)),pc),[pc],[res_pid['pc']])
print('Hessian eigen values:', np.linalg.eigvals(SX2DM(H)))
print('Gradient:', SX2DM(g).full().flatten())
```

# EISE Tunning

## Sensitivity system

Given $G=(A(p),B(p),C(p),D(p))$, a state space model depending on parameter vector $p$:

\begin{eqnarray}
\dot{x}	& = & A(p)x + B(p)u \\
y 		& = & C(p)x + D(p)u
\end{eqnarray}

We define ${\partial G}/{\partial p_i}$, the associeted sensitivity system in relation to parameter $p_i$, as:

\begin{eqnarray}
\frac{\partial \dot{x}}{\partial p}	& = & \frac{\partial A}{\partial p}x +A \frac{\partial x}{\partial p} + \frac{\partial B}{\partial p} u \\
\frac{\partial y}{\partial p}	& = & \frac{\partial C}{\partial p}x +C \frac{\partial x}{\partial p} + \frac{\partial D}{\partial p} u \\
\end{eqnarray}

```python
def sensys(G,p,p_n,i):
    """Calculates the state space sensitivity system of model.

    Parameters
    ----------
    param1 : G
        Mode in state space form.
    param2 : p
        Simbolic parameters of model.
    param3 : p_n
        Value of parameters.
    param4 : i
        Parameter inded for which sensitivity system will be calculated.

    Returns
    -------
    ss
        State space sensibility system in relation to parameter p_i
    """
    nx = G.A.shape[0]
    ny,nu = G.D.shape
    A = vertcat(horzcat(G.A,SX(nx,nx)),
                horzcat(reshape(jacobian(G.A,p[i]),nx,nx),G.A))
    B = vertcat(G.B,reshape(jacobian(G.B,p[i]),nx,nu))
    C = horzcat(reshape(jacobian(G.C,p[i]),ny,nx),G.C)
    D = reshape(jacobian(G.D,p[i]),ny,nu)
    return ss(*substitute([A,B,C,D],[p],[p_n]))
```

## Output covariance in flunction of Parameters covariance

Using uncertainty propagation, the variance of output, $\sigma_y^2$, can be calculated in function of output sentivity in relation to parameters, ${\partial y}/{\partial p}$, and covariance matrix of parameters, $\sigma_p^2$, using the following equation:

\begin{equation}
\sigma_y^2 = \left(\frac{\partial y}{\partial p}\right)\sigma_p^2\left(\frac{\partial y}{\partial p}\right)^\top
\end{equation}

```python
def step_inc(Gs,p,p_n,V,tf,dt=0.1):
    """Plot step of system together with associeted uncertainty.
    Uses analytical propagation of parameters covariance matrix.

    Parameters
    ----------
    param1 : Gs
        List of models in state space form.
    param2 : p
        Simbolic parameters of models.
    param3 : p_n
        Value of parameters.
    param4 : V
        Covariance matrix of parameters.
    param5 : tf
        End time of simulation.
    param6 : dt
        Time step.
    """
    if isinstance(Gs, ss): Gs = [Gs]
    ny,nu = Gs[0].D.shape
    fig,axes = plt.subplots(ny, nu,
                   sharex='col', sharey='row', squeeze=False)
    # Loop in the systems and system colors
    for G,color in zip(Gs,plt.rcParams['axes.prop_cycle']):
        ys, ts =  step([ss(*substitute(G,[p],[p_n]))],tf,dt,plot=False)
        ys = ys[0]
        nk = len(ts)
        dydp = step([sensys(G,p,p_n,i) for i in range(p.shape[0])]
                    ,tf,dt,plot=False)[0]
        sig_ys=np.sqrt(np.array([[[dydp[:,i,j,k].dot(V@dydp[:,i,j,k])[0]
            for k in range(nk)] for j in range(nu)] for i in range(ny)]))
        for i,row in enumerate(axes):
            for j,ax in enumerate(row):
                ax.plot(ts,ys[i,j],
                        ts,ys[i,j]+sig_ys[i,j],':',
                        ts,ys[i,j]-sig_ys[i,j],':',**color)

```

```python
#test step_inc
p,p_n = SX.sym('p',4),DM([-1,1,1,1])
V = 0.16*DM([[1,.5,0,0],[.5,1,.5,0],[0,.5,1,.5],[0,0,.5,1]])
G = tf2ss(SX(p[:2]),SX(vertcat(1,p[2:])))
step_inc([G],p,p_n,V,tf=10)
```

## Ortonormal parametrization

Be $\sigma_p^2 = UDU^\top$ the singular value decomposition of $\sigma_p^2$, covariance matrix of parameter vetor $p$. I we do a coordinates change $q = D^{-1/2}U^\top p$, the parameters $q$ in the coodinates has identity covariance matrix.

```python
def reparam(G,p,V):
    """Convert system parameters to a ortonormal parameters.

    Parameters
    ----------
    param1 : G
        System in state space form.
    param2 : p
        Simbolic parameters of system.
    param3 : V
        Covariance matrix of parameters

    Returns
    -------
    ss
        System reparameterizided
    SX
        New parameters for the system
    numpy.array
        Matrix to convert beetween parametrizations: p = sV@q
    """
    w,v = np.linalg.eig(np.array(V))
    sV = v@np.diag(np.sqrt(w))
    q = SX.sym('q',p.shape[0])

    return ss(*substitute(G,[p],[sV@q])),q,sV
```

## EISE calculation

Suposing we have ortonormal parametrization, the EISE, expectation of ISE, can be calculated as follows:

\begin{equation}\begin{aligned}
EISE & = E\left( ISE \right) = E \left( \sum_i\sum_j\int_0^\infty e^2_{ij}(t)dt \right) = \sum_i\sum_j\int_0^\infty E(e^2_{ij}(t))dt = \\
& = \sum_i\sum_j\int_0^\infty E(e_{ij}(t))^2+VAR(e_{ij}(t))dt= \\
&= \sum_i\sum_j\int_0^\infty E(e_{ij}(t))^2 dt+\sum_i\sum_j\int_0^\infty VAR(e_{ij}(t))dt = \\
& = \sum_i\sum_j\int_0^\infty E(e_{ij}(t))^2 dt+\sum_i\sum_j\int_0^\infty \sum_k  \left(
\frac{\partial e_{ij}}{\partial p_k} \right)^2 dt = \\
& = \sum_i\sum_j\int_0^\infty E(e_{ij}(t))^2 dt+\sum_k \left( \sum_i\sum_j\int_0^\infty   \left(
\frac{\partial e_{ij}}{\partial p_k} \right)^2 dt \right) = \\
&=\left\Vert G \right\Vert_2^2 + \sum_k \left\Vert \frac{\partial G}{\partial p_k} \right\Vert_2^2  = ISE + IVE
\end{aligned}\end{equation}

```python
def EISE(G,p,p_n,V,C):
    """Calculates EISE (Expected ISE) of a pair plant-controller.

    Parameters
    ----------
    param1 : G
        System in state space form.
    param2 : p
        Simbolic parameters of system.
    param3 : p_n
        Value of parameters of system.
    param4 : V
        Covariance matrix of parameters
    param5 : C
        Controler in state space form

    Returns
    -------
    float
        EISE value
    """
    T,q,sV = reparam(feedback(G,C),p,V) #p = sV*q
    q_n = inv(sV)@p_n
    Tt = transform(T)
    return h2norm(ss(*substitute(Tt,[q],[q_n]))) + \
            sum(h2norm(sensys(Tt,q,q_n,i)) for i in range(q.shape[0]))

def mineise(G,p,p_n,V,C,pc,pc0,pc_min,pc_max):
    """eise tunning. Find parameters of controller that minimize EISE
    of feedback system.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : p
        Simboli plant parameters.
    param3 : p_n
        Value of plant parameters.
    param4 : V
        Covariance matrix of plant parameters
    param2 : C
        Controller in state space form.
    param2 : pc
        Symbolic controller parameters.
    param2 : pc0
        Initial guess for controller parameters.
    param2 : pc_min
        Controller parameters lower bound for optimization.
    param2 : pc_max
        Controller parameters upper bound for optimization.

    Returns
    -------
    dict
        'pc': optimal controller parameters, 'f': optimal EISE
    """
    res = nlpsol('S','ipopt',{'x':pc,'f':EISE(G,p,p_n,V,C),'g':pc},opts)\
            (x0=pc0,lbg=pc_min,ubg=pc_max)
    return {'pc':res['x'], 'f':res['f']}
```

```python
#test eise tunning
p,p_n = SX.sym('p',4),DM([-1,1,1,1])
V = 0.16*DM([[1,.5,0,0],[.5,1,.5,0],[0,.5,1,.5],[0,0,.5,1]])
G = tf2ss(SX(p[:2]),SX(vertcat(1,p[2:])))
pc,pc0,pc_min,pc_max = SX.sym('pc',2),DM([.2,.2]),DM([.1,.1]),DM([1.,1.])
res_pi = mineise(G,p,p_n,V,pi(pc),pc,pc0,pc_min,pc_max)
pc,pc0 = SX.sym('pc',3),DM([.3,.3,.3])
pc_min,pc_max = DM([.1,.1,.1]),DM([1.,1.,1.])
res_pid = mineise(G,p,p_n,V,pid(pc),pc,pc0,pc_min,pc_max)
print('PI EISE ótimo:',res_pi['pc'])
print('PID EISE ótimo:',res_pid['pc'])
Cs = [pi(res_pi['pc']),pid(res_pid['pc'])]
EISEs = [SX2DM(EISE(G,p,p_n,V,C)).full()[0,0] for T in Ts]
step_inc([feedback(G,C) for C in Cs],p,p_n,V,tf=15)
plt.legend(['{} EISE={:.4}'.format(*x)
            for x in zip(['pi_opt','pid_opt'],EISEs)])
plt.show()
```

```python
# Apply sensitivity analysis to EISE around optimum PID controler
eise = EISE(G,p,p_n,V,pid(pc))
f = Function('f',[pc],[eise],['pc'],['ISE'])
sensitivity(f,res_pid['pc'],0.5)
[H, g] = substitute(hessian(eise,pc),[pc],[res_pid['pc']])
print('Hessian eigen values:', np.linalg.eigvals(SX2DM(H)))
print('Gradient:', SX2DM(g).full().flatten())
```

```python
#plot EISE contour varying Kp and Ki PI controller parameters
pc = SX.sym('pc',2)
fEISE = Function('fEISE',[pc],[EISE(G,p,p_n,V,pi(pc))],['pc'],['EISE'])
f = lambda x: max(min(x,20),-20) # function to limitate the output

N = 100
kps, kis = np.linspace(-1.5, 1.5, N), np.linspace(-.1, .6, N)
EISEs = np.array([[f(fEISE([kp,ki])) for kp in kps] for ki in kis])

plt.contourf(kps,kis,EISEs,100,cmap=plt.cm.seismic),plt.colorbar()
plt.title('Curvas de Nível do EISE'),plt.xlabel('Kp'),plt.ylabel('Ki')
plt.show()
```

```python
def worstcase(G,p,p_n,V,n_ic,C,p0=None):
    """Calculates the worst case plant for the controller considering
    a uncertain region on plant G.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : p
        Symbolic plant parameters.
    param3 : p_n
        Value of plant parameters.
    param4 : V
        Covariance matrix of plant parameters.
    param5 : C
        Controller in state space form.
    param6 : p0
        Initial guess for parameters of worst case plant.

    Returns
    -------
    dict
        'p': worst case plant parameters, 'f': worst case ISE,
        'lambda': associated Lagrange multipliers
    """
    if p0 is None: p0 = 1.1*p_n
    p0 = p_n + (p0-p_n)*(n_ic/sqrt((p0-p_n).T@inv(V)@(p0-p_n)))
    mahalanobis = (p-p_n).T@inv(V)@(p-p_n)
    nlp = {'x':p,'f':-ISE(G,C),'g':mahalanobis}
    res = nlpsol('S','ipopt',nlp,opts)(x0=p0,lbg=n_ic**2,ubg=n_ic**2)
    return {'p':res['x'], 'f':-res['f'], 'lambda':-res['lam_g']}
```

```python
n_ics = np.arange(0,1.0,0.2)
w_cases = [worstcase(G,p,p_n,V,n_ic,pi(res_pi['pc'])) for n_ic in n_ics]

tf,dt = 15,0.1
step([ss(*substitute(G,[p],[w_case['p']])) for w_case in w_cases],tf,dt)
step([feedback(ss(*substitute(G,[p],[w_case['p']])),
               pi(res_pi['pc'])) for w_case in w_cases],tf,dt)
```

## Min-max formulation

\begin{equation}
\min_{\mathbf{C}} \max_{\mathbf{G}} ISE(\mathbf{C},\mathbf{G})
\end{equation}

Using KKT first order necessary conditions:

\begin{equation}\begin{aligned}
\min_{\mathbf{C},\mathbf{G},\lambda} & ISE(\mathbf{C},\mathbf{G}) \\
\text{s.a.} & \frac{\partial ISE(\mathbf{C},\mathbf{G})}{\partial \mathbf{G}}+\lambda \frac{\partial D_M(\mathbf{G})}{\partial \mathbf{G}} = 0 \\
 & D_M(\mathbf{G}) - n_\sigma = 0
\end{aligned}\end{equation}

```python
def minmax(G,p,p_n,V,n_ic,C,pc,pc0):
    """minmax tunning, finds the controller parameters that minimize
    the worst case inside a confidence region.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : p
        Symbolic plant parameters.
    param3 : p_n
        Value of plant parameters.
    param4 : V
        Covariance matrix of plant parameters.
    param5 : C
        Controller in state space form.
    param6 : pc
        Symbolic controller parameters.
    param7 : pc0
        Initial guess for controller parameters.

    Returns
    -------
    dict
        'p': worst case plant parameters, 'f': worst case ISE,
        'pc': optimum controler parameters
    """
    w_case = worstcase(G,p,p_n,V,n_ic,ss(*substitute(C,[pc],[pc0])))
    lambd = SX.sym('lambd')
    mahalanobis = (p-p_n).T@inv(V)@(p-p_n)
    ise = ISE(T)
    kkt = jacobian(ise,p)+lambd*jacobian(mahalanobis,p)
    nlp = {'x':vertcat(p,pc,lambd),'f':-ise,
           'g':vertcat(mahalanobis,kkt.T)}
    bg = vertcat(n_ic**2,DM(p.shape[0],1))
    res = nlpsol('S','ipopt',nlp,opts) \
        (x0=vertcat(w_case['p'],pc0,w_case['lambda']),lbg=bg,ubg=bg)
    n_p = p.shape[0]
    return {'p':res['x'][:n_p],'pc':res['x'][n_p:],'f':-res['f']}
```

```python
#test minmax tunning
pc,pc0 = SX.sym('pc',3),DM([.3,.3,.3])
pc_min,pc_max = DM([.1,.1,.1]),DM([1.,1.,1.])
n_ic=0.6
res = minmax(G,p,p_n,V,n_ic,C,pc,pc0)
G_pior = ss(*substitute(G,[p],[res['p']]))
#step of open loop nominal model and worst case plant
step([G_n,G_pior],10)
#step of close loop nominal model and worst case plant
step([feedback(G_n,pid(res['pc'])), feedback(G_pior,pid(res['pc']))],10)
```

# ISE restricted optimization

### Complex determinant

Be a complex matrix $L$, for which we want to compute determinant.

\begin{equation}\begin{aligned}
L = L^0+jL^1 \\
L = \begin{bmatrix}L_1 & L_2 & \cdots & L_n\end{bmatrix} \\
L^0 = \begin{bmatrix}L^0_1 & L^0_2 & \cdots & L^0_n\end{bmatrix} \\
L^1 = \begin{bmatrix}L^1_1 & L^1_2 & \cdots & L^1_n\end{bmatrix} \\
\end{aligned}\end{equation}

We have:

\begin{equation}
det(L) = \sum_{x \in \{0,1\}^n}det(\begin{bmatrix}L^{x_1}_1 & L^{x_2}_2 & \cdots & L^{x_n}_n\end{bmatrix})
\end{equation}

```python
from itertools import product
def complexdet(A,B):
    """Computes de determinant of the complex matrix A+jB.

    Parameters
    ----------
    param1 : A
        Real part of matrix.
    param2 : B
        Imaginary part of matrix.

    Returns
    -------
    SX
        Symbolic array with two values: real and imag part of determinant.
    """
    n = A.shape[0]
    M = horzcat(A,B)
    C,D = 0,0
    for i in product([0,1],repeat=n):
        d = det(M[:,np.arange(n)+n*np.array(i)])
        if sum(i)%4 == 0: C += d
        if sum(i)%4 == 1: D += d
        if sum(i)%4 == 2: C -= d
        if sum(i)%4 == 3: D -= d
    return vertcat(C,D)
```

```python
#test det(A+jB)
print(complexdet(SX.sym('A',2,2),SX.sym('B',2,2)))
```

    [(((A_0*A_3)-(A_2*A_1))-((B_0*B_3)-(B_2*B_1))), (((A_0*B_3)-(B_2*A_1))+((B_0*A_3)-(A_2*B_1)))]

Be $L=(A,B,C,D)$ a open loop state space system. We have frequency response $G(j\omega)$: $$L(j\omega)=-C(\omega^2 I+A^2)^{-1}AB+ D - j\omega C(\omega^2 I+A^2)^{-1}B$$
The Nyquist diagram is the plot for $\omega$ varying beetween $-\infty$ and $+\infty$ of the function: $$det(I+L(j\omega))$$

```python
def nyquist(L,w):
    """finds det(I+L(j*omega)) to plot Nyquist diagram.

    Parameters
    ----------
    param1 : L
        Open loop in state space form. Plant in series
    param2 : w
        Frequency.

    Returns
    -------
    SX
        Real and imaginary parts of point in Nyquist diagram.
    """
    A,B,C,D = L
    M = inv(w**2*SX.eye(A.shape[0])+A@A)
    return complexdet(-C@A@M@B+D+SX.eye(D.shape[0]), -w*C@M@B)
```

```python
#test nyquist
ws = np.logspace(-.5,3,100)
rfs = [nyquist(series(G_n,pi(pc0)),w) for w in ws]
re,im = zip(*((rf[0],rf[1]) for rf in rfs))
plt.plot(re,im)
plt.plot(re,-np.array(im))
plt.show()
```

##  Controller robustiness rate

Defining $N(\mathbf{C},\mathbf{G},\omega)$:
\begin{equation}
N(\mathbf{C},\mathbf{G},\omega) = \begin{bmatrix}Re(det(I+\mathbf{G}(j\omega)\mathbf{C}(j\omega))) \\ Im(det(I+\mathbf{G}(j\omega)\mathbf{C}(j\omega))) \end{bmatrix}
\end{equation}

The condition $N(\mathbf{C},\mathbf{G},\omega) = 0 $ is equivalent to the system be marginally stable. So we define robustiness rate as:

\begin{equation}\begin{aligned}
rate(\mathbf{C}) = \min_G D_M(\mathbf{G}) \\
\text{subject to} \: N(\mathbf{C},\mathbf{G},\omega) = 0
\end{aligned}\end{equation}

```python
def rate(G,p,p_n,V,C):
    """Calculates the controler robustness rate.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : p
        Symbolic plant parameters.
    param3 : p_n
        Value of plant parameters.
    param4 : V
        Covariance matrix of plant parameters.
    param5 : C
        Controller in state space form.

    Returns
    -------
    dict
        'rate': controller robustness rate, 'p': destabilizer plant,
        'w': unstable frequency, 'lambda': Lagrange multiplier
    """
    mahalanobis = (p-p_n).T@inv(V)@(p-p_n)
    ln_w = SX.sym('ln_w')
    nlp = {'x':vertcat(p,ln_w),'f':mahalanobis,
           'g':nyquist(series(G,C),exp(ln_w))}
    res =  nlpsol('S','ipopt',nlp,opts) \
                 (x0=vertcat(p_n*1.1,0),lbg=[0,0],ubg=[0,0])
    return {'rate':sqrt(res['f']),'lambda':res['lam_g'],
            'p':res['x'][:-1],'w':exp(res['x'][-1])}

```

```python
#test rate
C = pi([.3,.3])
res = rate(G,p,p_n,V,C)

step([G_n,ss(*substitute(G,[p],[res['p']]))],10)
step([feedback(G_n,C),feedback(ss(*substitute(G,[p],[res['p']])),C)],10)

```

## iseres tunning

\begin{equation}\begin{aligned}
\min_\mathbf{C} ISE(\bar{\mathbf{G}},\mathbf{C}) \\
\text{subject to} \: \text{rate}(\mathbf{C}) \geq n_\sigma
\end{aligned}\end{equation}

Can be rewritten as:

\begin{equation}\begin{aligned}
\min_{\mathbf{C},\mathbf{G},\omega,\lambda} & ISE(\mathbf{C},\bar{\mathbf{G}}) \\
\text{subject to} \; & D_M(\mathbf{G}) = n_\sigma \\
& \frac{\partial D_M(\mathbf{G})}{\partial \mathbf{G}} + \lambda^\top \frac{\partial N(\mathbf{C},\mathbf{G},\omega)}{\partial \mathbf{G}} = 0 \\
& \lambda^\top\frac{\partial N(\mathbf{C},\mathbf{G},\omega)}{\partial \omega} = 0 \\
& N(\mathbf{C},\mathbf{G},\omega) = 0\\
\end{aligned}\end{equation}

```python
def iseres(G,p,p_n,V,n_ic,C,pc,pc0):
    """iseres tunning. Find optimal control parameters that minimize ISE
    while keeping a minimum robustness rate.

    Parameters
    ----------
    param1 : G
        Plant in state space form.
    param2 : p
        Symbolic plant parameters.
    param3 : p_n
        Value of plant parameters.
    param4 : V
        Covariance matrix of plant parameters.
    param5 : n_ic
        Size of confidence interval, in number of standard deviations.
    param6 : C
        Controller in state space form.
    param7 : pc
        Symbolic controller parameters.
    param8 : pc0
        Initial guess for controller parameters.

    Returns
    -------
    dict
        'pc': optimum controller parameters, 'p': destabilizer plant,
        'f': ISE of controller
    """
    res = rate(G,p,p_n,V,ss(*substitute(C,[pc],[pc0])))
    ise = ISE(ss(*substitute(G,[p],[p_n])),C)
    mahalanobis = (p-p_n).T@inv(V)@(p-p_n)
    ln_w = SX.sym('ln_w')
    N = nyquist(series(G,C),exp(ln_w))
    lambd = SX.sym('lambd',2)
    nlp = {'x':vertcat(p,pc,ln_w,lambd),'f':ise,
           'g':vertcat(mahalanobis-n_ic**2,
                       (jacobian(mahalanobis,p)+lambd.T@jacobian(N,p)).T,
                       lambd.T@jacobian(N,ln_w),N)}
    bg = DM(p.shape[0]+4,1)
    x0=vertcat(res['p'],pc0,log(res['w']),res['lambda'])
    res = nlpsol('S','ipopt',nlp,opts)(x0=x0,lbg=bg,ubg=bg)
    n_p,n_pc = p.shape[0],pc.shape[0]
    return {'p':res['x'][:n_p], 'pc':res['x'][n_p:n_p+n_pc],'f':res['f']}

```

```python
#test iseres
pc,pc0 = SX.sym('pc',3),DM([.3,.3,.3])
pc_min,pc_max = DM([.1,.1,.1]),DM([1.,1.,1.])
C = pid(pc)
n_ic = 1.5
res = iseres(G,p,p_n,V,n_ic,C,pc,pc0)
G_inst = ss(*substitute(G,[p],[res['p']]))
step([G_n,G_inst],10)
step([feedback(G_n,pid(res['pc'])),feedback(G_inst,pid(res['pc']))],25)
```


# Running a statistical trial for a machine learning regression model

Imagine you have been given an imaging dataset and you have trained a [convolutional neural network](https://en.wikipedia.org/wiki/Convolutional_neural_network) to count the number of cells in the image for a medical-based task. On a held-out test set you observe an average error ±5 cells. In order to be considered reliable enough for clinical use, the algorithm needs to be further validated on a prospective trial. The purpose of a such a trial is to 1) ensure that the algorithm's performance is close to what was observed on a test set, and 2) increase the rigour of assessment through a pre-specified protocol. How should such a statistical trial be conducted and what statistical quantities should be estimated? In this post I outline a two-stage method to conduct a validation study for a regression model. This strategy amounts to answering two questions:

1. What is the upper-bound error the algorithm could have before it is was no longer of practical use (e.g. ±10 cells)?
2. Conditional on this upper bound, how many samples will be needed in a prospective trial to establish that the error is *at most* a certain size?

In a previous [post](http://www.erikdrysdale.com/threshold_and_power) I discussed how to calibrate a machine learning (ML) model for a binary classification task in the context of a statistical trial. The classical ML pipeline is to train and tune a model on a training and validation set, and then make predictions (only once) on a test set to get an "unbiased" estimate of a specific performance metric.[[^1]] A statistical trial represents a further iteration on the ML pipeline: collecting data prospectively to "confirm" that the model works as well as you expect. For binary classifiers there were two statistical procedures when preparing for a prospective trial: 1) using the test set to establish a conservative threshold for a target performance level (e.g. 90% sensitivity), 2) picking a slightly worse trial goal (e.g. 80% sensitivity) and calculating a sample size necessary based on this spread. The first procedure relied on the statistical properties of the threshold (which is a random variable) for a given fixed hypothesis. The second procedure could be trivially calculated using statistical tests for the difference in two binomial proportions.

The regression case is more complicated because the desired performance cannot be chosen in advance: the result is what it is. One possibility is to pre-specify a null hypothesis (e.g. R-squared greater than 10%), and only run prospective trials for algorithms that rejected this null. However, such an approach would create a statistical significance [filter](http://www.erikdrysdale.com/winners_curse) that would, conditional on success (i.e. rejection of null), cause the expected test set performance to be biased upwards. Such a bias would lead to algorithms which fail to generalize and underestimate the prospective sample size that will be needed.

I have developed a two-stage testing strategy that avoids the problem of statistical significance filters and relies of classical statistical hypothesis testing paradigms. This approach has several advantages:

1. Model performance will be unbiased
2. Classical statistical techniques can be used to obtain valid inference
3. The upper bound can be chosen with respect to power considerations or the application use case, or both
4. **The analysis applies to any ML algorithm and any performance metric** (conditional on some regularity conditions)

The rest of the post is structured as follows: section (1) provides the statistical framework for a two-stage testing strategy for estimating the mean of a Gaussian, section (2) shows how two common regression performance metrics can be used and approximated by a Gaussian distribution, and section (3) provides an example pipeline of how this framework can be used and statistical simulation results.

[^1]: If the test set is non-representative of the future data generating process then the results of this subsequent analysis will not hold. Dealing with dataset shift is a large topic area that is beyond the scope of this post.

## (1) Two-stage testing approach

Imagine you will have access to two independent datasets, and your goal is to establish an upper-bound on the "true" mean of a Gaussian distribution. In the first stage, a sample is drawn, and the distribution of the sample mean is used to estimate the null hypothesis. In the second stage, a new sample is drawn, and the null from stage 1 is used to calculate a test statistic and a p-value. Assume the data is IID and comes from a normal distribution with a known variance: $X_i \sim N(\mu, \sigma^2)$.

$$
\begin{align*}
&\text{Step 1: Establish null} \\
\hat{\mu}_0 &= \hat\mu_1 + k \sqrt{\sigma^2 / n_1} \\
H_0&: \mu \geq \hat{\mu}_0 \\
H_A&: \mu < \hat{\mu}_0 \\
&\text{Step 2: Test on second sample} \\
s_2 &= \frac{\hat\mu_2 - \hat\mu_0}{\sqrt{\sigma^2 / n_2}} \\
\text{Reject }H_0&: s_2 < \Phi_{s_2}^{-1}(\alpha|H_0) = t_\alpha
\end{align*}
$$

In the first stage, the null is estimated as the point estimate of the sample mean plus $k$ standard deviations above it. As the value of $k$ increases, the power of the second-stage test increases. However, the "information" about the true mean decreases since the bound becomes larger. Consider the distribution of $s_2$, which is the test statistic in the second stage of the procedure:

$$
\begin{align*}
s_2&= \frac{\hat\mu_2-[\hat\mu_1+k\sqrt{\sigma^2/n_1}]}{\sigma^2/n_2} \\
&= z_2 - \sqrt{n_2/n_1}(z_1 + k) \\
z_i &= \frac{\hat\mu_i - \mu}{\sqrt{\sigma^2 / n_i}}
\end{align*}
$$

The unconditional distribution of the statistic can be seen to have a normal distribution:

$$
\begin{align*}
s_2&\sim N\big(-\sqrt{n_2/n_1}\cdot k, 1+n_2/n_1\big)
\end{align*}
$$

For a given $n_2$, as $n_1 \to \infty$, then the probability of rejecting the null, $P(s_2 < t_\alpha)$, approaches $\alpha$ since $s_2 \to N(0,1)$. For a given $n_1$, as $n_2 \to \infty$, then the probability of rejecting the null approaches $\Phi(k)$.[[^2]] However, in order to calculate the type-I and type-II errors of a testing procedure we need to know the distribution of $s_2$ conditional on the status of the null. Such as distribution is more complex:

$$
\begin{align}
s_2 | \{\text{$H_0$ is false}\} &\sim \frac{\hat\mu_2-\hat\mu_0}{\sqrt{\sigma^2/n_2}} \hspace{1mm} \Big|\hspace{1mm} \hat\mu_0 > \mu \nonumber \\
&\sim z_2 - r\cdot z_1^k | z_1^k > 0 \label{eq:dist_cond} \\
z_1^k &= z_1 + k, \hspace{3mm} r = \sqrt{n_2/n_1} \nonumber
\end{align}
$$

The conditional test statistic in \eqref{eq:dist_cond} is equivalent to a weighted sum of a standard normal and a truncated normal distribution. How can we characterize this distribution? The first step is to define a bivariate normal distribution as a function of $z_2\sim N(0,1)$ and $z_1^k~\sim(k,1)$.

$$
\begin{align}
X&= z_2 - r\cdot Y \nonumber \\
Y&= z_1^k \nonumber\\
\begin{pmatrix} X \\ Y \end{pmatrix} &\sim \text{MVN}\Bigg[ \begin{pmatrix} -rk \\ k \end{pmatrix}, \begin{pmatrix} 1+r^2 & -r \\ -r & 1 \end{pmatrix} \Bigg] \label{eq:dist_MVN} \\
\rho &= \frac{-r}{\sqrt{1+r^2}} \nonumber
\end{align}
$$

How does this help us? Luckily the distribution of a truncated bivariate normal distribution for one of the variables has been [characterized](https://link.springer.com/article/10.1007/BF02294652) already by *Arnold et. al*. After working out some of the math, the marginal density function of $X$ in \eqref{eq:dist_MVN}, conditional on $Y > 0$ or $Y<0$, can be written as follows:

$$
\begin{align*}
f_X(x) &= \frac{1}{\Phi(s\cdot k)} \cdot \frac{1}{\sqrt{1+r^2}} \cdot \phi\Bigg(\frac{x+rk}{\sqrt{1+r^2}} \Bigg) \cdot \Phi\Bigg(s\cdot \Bigg[ -r\cdot \frac{x+rk}{\sqrt{1+r^2}} + \sqrt{1+r^2}\cdot  k \Bigg] \Bigg) \\
s &= \begin{cases}
+1 &\text{ if } \text{$H_0$ is false, } \hspace{2mm} (Y>0) \\
-1 &\text{ if } \text{$H_0$ is true, }  \hspace{2mm} (Y<0)
\end{cases} \\
f_W(w) &= \frac{1}{\Phi(k)} \cdot \frac{1}{\sigma_W} \cdot \phi(w) \cdot  \Phi(a + b\cdot w), \hspace{3mm} w=\frac{x+rk}{\sqrt{1+r^2}}
\end{align*}
$$

Next, we can use the result from Owen's classic [paper](https://www.tandfonline.com/doi/abs/10.1080/03610918008812164) which shows that that [integral](https://mathoverflow.net/questions/283928/closed-form-solution-for-an-integral-involving-the-p-d-f-and-c-d-f-of-a-n0-1) needed for calculating $F_W$ can be calculated from the CDF of a bivariate normal:

$$
\begin{align*}
F_W(w;s) &= \frac{1}{\Phi(s\cdot k)} \int_{-\infty}^w \frac{1}{\sigma_W} \cdot \phi(u) \cdot  \Phi(a(s) + b(s)\cdot u) du \\
    &= \frac{1}{\Phi(s\cdot k)}  \text{BVN}\Big( X_1 \leq \frac{a(s)}{\sqrt{1+b(s)^2}}, X_2 \leq w, \rho=-b(s)/\sqrt{1+b(s)^2} \Big) \\
F_X(x;s) &= F_W\big((x+rk) / \sqrt{1+r^2};s\big) \label{eq:cdf_X}
\end{align*}
$$

The first code block below shows how to calculate the CDF of \eqref{eq:dist_cond} using \eqref{eq:cdf_X}. Simulations are run to demonstrate the accuracy of this approach.

[^2]: Throughout this post $\Phi$ and $\phi$ denote the standard normal CDF and PDF, respectively.

```python
from time import time
import numpy as np
from scipy.stats import norm
from scipy.stats import multivariate_normal as MVN
import pandas as pd
import plotnine
from plotnine import *
from scipy.optimize import minimize_scalar

class cond_dist():
    def __init__(self, k, n1, n2, null=False):
        self.s = +1
        if null is True:
            self.s = -1
        self.k, self.n1, self.n2 = k, n1, n2
        self.r = np.sqrt(n2 / n1)

    def cdf_w(self, w):
        a = np.sqrt(1+self.r**2) * self.k * self.s
        b = -self.r * self.s
        rho = -b/np.sqrt(1+b**2)
        Sigma = np.array([[1,rho],[rho,1]])
        dist_MVN = MVN(mean=np.repeat(0,2),cov=Sigma)
        x1 = a / np.sqrt(1+b**2)
        if isinstance(w, float):
            X = [x1, w]
        else:
            X = np.c_[np.repeat(x1,len(w)), w]
        pval = dist_MVN.cdf(X)
        return pval

    def cdf_x(self, x):
        const = 1 / norm.cdf(self.s * self.k)
        w = (x + self.r * self.k) / np.sqrt(1+self.r**2)
        pval = self.cdf_w(w) * const
        return pval

    def quantile(self, p):
        res = minimize_scalar(fun=lambda x: (self.cdf_x(x)-p)**2, method='brent').x
        return res

seed = 1234
n1 = 100
sig2 = 2
mu = 3
nsim = 5000

n2_seq = [100, 300, 500]
k_seq = list(np.arange(0,2.5,0.5))

np.random.seed(seed)
holder = []
for k in k_seq:
    for n2 in n2_seq:
        c = k * np.sqrt(sig2/n1)
        # Draw samples for two-stages
        x1 = mu + np.sqrt(sig2)*np.random.randn(n1, nsim)
        x2 = mu + np.sqrt(sig2)*np.random.randn(n2, nsim)
        xbar1, xbar2 = x1.mean(0), x2.mean(0)
        null_mu = xbar1 + c
        s2 = (xbar2 - null_mu)/np.sqrt(sig2/n2)
        # Calculate the p-values
        pval_h0false = cond_dist(k=k, n1=n1, n2=n2, null=False).cdf_x(s2)
        pval_h0true = cond_dist(k=k, n1=n1, n2=n2, null=True).cdf_x(s2)
        pval_uncond = norm(loc=-np.sqrt(n2/n1)*k,scale=np.sqrt(1+n2/n1)).cdf(s2)
        tmp = pd.DataFrame({'s2':s2,'mu0':null_mu,'n2':n2,'k':k,
                            'h0_false':pval_h0false, 'h0_true':pval_h0true,'pval_uncond':pval_uncond})
        holder.append(tmp)
        del tmp
df_res = pd.concat(holder).assign(null=lambda x: np.where(x.mu0 > mu, False, True))
cn_gg = ['null','n2','k']
df_res = df_res.sort_values(cn_gg+['s2']).reset_index(None,True)
df_res = df_res.assign(idx=df_res.groupby(cn_gg).cumcount())
df_res.idx = df_res.groupby(cn_gg).apply(lambda x: x.idx/x.idx.max()).values
# Compare the conditional distribution
df_res = df_res.assign(pval_cond=lambda x: np.where(x.null==False,x.h0_false,x.h0_true))
df_res_long = df_res.melt(cn_gg+['idx'],['pval_cond','pval_uncond'],'tt')

# Make a pp-plot
tmp = df_res_long.groupby(cn_gg+['tt']).sample(n=250, random_state=seed,replace=True)
plotnine.options.figure_size = (8, 7)
gg_pp = (ggplot(tmp, aes(x='value',y='idx',color='tt')) + theme_bw() +
         geom_point() + labs(x='Theoretical percentile',y='Empirical percentile') +
         ggtitle('Figure 1: P-P plot for test statistic') +
         facet_grid('n2+null~k',labeller=label_both) +
         scale_color_discrete(name='Distribution',labels=['Conditional','Unconditional']) +
         geom_abline(slope=1,intercept=0,linetype='--',color='black',size=1))
gg_pp
```

Figure 1 shows that the CDF for the conditional distribution in \eqref{eq:cdf_X} accurately captures the distribution of the test statistic when the null is both false and true. When the null is false ($z_1^k > 0)$, for larger values of $k$, the unconditonal distribution of $s_2$ is a close approximation. This result makes sense since when the null hypothesis is set many standard deviations above the point estimate, the null will be false for almost all realizations so the conditioning event excludes very few realizations.

In classical statistics we pick a critical value to reject the null such that when the null is true, then rejection event happens at most $\alpha$ percent of the time. We can use \eqref{eq:cdf_X} to find the $\alpha-$quantile of the distribution when the null is true so that we reject it at most $a$-percent of the time:

$$
\begin{align}
F^{-1}_W(\alpha;-1) &= \sup_w: \{ F_W(w;-1)\leq \alpha \} \label{eq:quantile} \\
&= t_\alpha \nonumber
\end{align}
$$

```python
np.random.seed(seed)
# Calculate power for a range of n2's/k's
alpha = 0.05
n2_seq = np.arange(50,1001, 50)
k_seq = np.arange(0.0, 2.51, 0.50)
holder = []
for k in k_seq:
    for n2 in n2_seq:
        dd_true = cond_dist(k=k, n1=n1, n2=n2, null=True)
        dd_false = cond_dist(k=k, n1=n1, n2=n2, null=False)
        crit = dd_true.quantile(alpha)
        power_theory = dd_false.cdf_x(crit)
        # --- simulation --- #
        c = k * np.sqrt(sig2/n1)
        # Draw samples for two-stages
        x1 = mu + np.sqrt(sig2)*np.random.randn(n1, nsim)
        x2 = mu + np.sqrt(sig2)*np.random.randn(n2, nsim)
        xbar1, xbar2 = x1.mean(0), x2.mean(0)
        null_mu = xbar1 + c
        s2 = (xbar2 - null_mu)/np.sqrt(sig2/n2)
        power_emp = np.mean(s2[null_mu > mu] < crit)
        type1_emp = np.mean(s2[null_mu < mu] < crit)
        # ------------------ #
        tmp = pd.Series({'k':k,'n2':n2,'Critical-Value':crit,'Power':power_theory,
                         'emp_power':power_emp, 'emp_type1':type1_emp})
        holder.append(tmp)
        del tmp
df_power = pd.concat(holder,1).T
df_power_long = df_power.melt(['k','n2'],['Critical-Value','Power'],'measure')
df_power_long.measure = pd.Categorical(df_power_long.measure,['Power','Critical-Value'])

plotnine.options.figure_size = (8, 4)
gg_power = (ggplot(df_power_long,aes(x='n2',y='value',color='k',group='k')) + theme_bw() +
            geom_line() + labs(x='n2',y='Value') +
            facet_wrap('~measure',labeller=label_both,scales='free_y') +
            ggtitle('Figure 1B: Power by second-stage sample size and k') +
            theme(subplots_adjust={'wspace': 0.20}) +
            scale_x_continuous(limits=[0,1001]))
gg_power
```

Figure 1B reveals that as the second-stage sample size ($n_2$) or the value of $k$ grows, the power of the test increases. Higher values of $k$ ensures that the expected value between $\hat\mu_2 - \hat\mu_0$ becomes increasingly negative, raising the probability of rejection. A higher second-stage sample size decreases the variation of $\hat\mu_2$, ensuring that the average negative difference is more consistently around the expectation, once again increasing the probability of rejection.

```python
df_emp_long = df_power.melt(['k','n2'],['emp_power','emp_type1'],'tt')
df_emp_long.tt = df_emp_long.tt.map({'emp_power':'Empirical Power','emp_type1':'Empirical Type-I'})
tmp = pd.DataFrame({'tt':'Empirical Type-I', 'vv':0.05},index=[0])
plotnine.options.figure_size = (8, 4)
gg_emp = (ggplot(df_emp_long,aes(x='n2',y='value',color='k',group='k')) + theme_bw() +
            geom_line() + labs(x='n2',y='Value') + facet_wrap('~tt',scales='free_y') +
            ggtitle('Figure 1C: Empirical results match theory') +
            theme(subplots_adjust={'wspace': 0.15}) +
            scale_x_continuous(limits=[0,1001]) +
            geom_hline(aes(yintercept='vv'),data=tmp,inherit_aes=False,linetype='--'))
gg_emp
```

Figure 1C shows that the empirical power curves line up with the theoretical expectation, and that the type-I error rates average to the expected level: 5%. Note that the empirical type-I error rates are not exactly 5% by random chance alone. For a sufficiently large number of simulation draws, the estimates will converge to the 5% line.

## (2) Regression statistic inference example

This section will show how to apply the principle of two-stage testing to example regression performance metrics: mean absolute error (MAE) and mean square error (MSE). In addition to being common metrics, these statistics also have known distributional properties when a linear regression model is used with Gaussian data. Hence, the statistical simulations can be benchmarked against a ground truth. However, in practice any regression statistic whose density function is reasonably smooth, and any regression model can be used. To repeat, the simple linear model and choice of statistics is only for convenience and does not signify a loss of generality to any other regression instance. Formally we are interested in the risk of the MAE & MSE loss functions:

$$
\begin{align*}
R_{MAE}(\theta) = E_{y\sim g(x)}[\text{MAE}(y,f_\theta(x)] \\
R_{MSE}(\theta) = E_{y\sim g(x)}[\text{MSE}(y,f_\theta(x)]
\end{align*}
$$

Where $y$ and $x$ are the response and the feature set, respectively, and $f_\theta(x)$ is a linear regression model indexed by its coefficient vector: $\theta$. Assume that the joint distribution is Gaussian:

$$
\begin{align*}
y &= x^T \theta^0 + u \\
u_i&\sim N(0,\sigma^2_u), \hspace{3mm} x \sim \text{MVN}(0,I) \\
e &= y - f_\theta(x) \\
&= x^T(\theta^0 - \theta) + u \\
&\sim N(0, \sigma^2_u + \| \theta^0 - \theta \|^2_2 )
\end{align*}
$$

The errors of such a model have a Gaussian distribution, with a variance equal to some irreducible error ($\sigma^2_u$) and the L2-norm of the coefficient error: $\sum_j (\theta_j^0 - \theta_j^2)$. The risk for the MSE or MAE can be easily calculated since the error, $e$, has a known distribution:

$$
\begin{align}
R_{MSE}(\theta) &= E(e^2) = \sigma^2_u + \| \theta^0 - \theta \|^2_2 \label{eq:risk_mse} \\
R_{MAE}(\theta) &= E( |e| ) = \sqrt{\sigma^2_u + \| \theta^0 - \theta \|^2_2}\cdot\sqrt{2/\pi} \label{eq:risk_mae}
\end{align}
$$

Where the risk for the MAE comes from the [half-normal](https://en.wikipedia.org/wiki/Half-normal_distribution) distribution. The empirical estimate of the MAE and MSE is simply their sample average:

$$
\begin{align*}
\hat{\text{MSE}}(\theta) &= n^{-1} \sum_{i=1}^n [y_i - f_\theta(x_i)]^2 \\
\hat{\text{MAE}}(\theta) &= n^{-1} \sum_{i=1}^n |y_i - f_\theta(x_i)|
\end{align*}
$$

On a test set with a sufficiently large $n$, $\hat{\text{MSE}}(\theta) \to R_{MSE}(\theta)$. However, for fininte sample it is clear that $\hat{\text{MSE}}(\theta)$ is a random variable whose first moment will be centered around the risk.

```python
from sklearn.linear_model import LinearRegression
from arch.bootstrap import IIDBootstrap
from sklearn.metrics import mean_squared_error as MSE
from sklearn.metrics import mean_absolute_error as MAE

def dgp_yX(n,p,t0=1,sig2=1,theta0=None):
    X = np.random.randn(n,p)
    if theta0 is None:
        theta0 = np.repeat(t0,p) * np.sign(np.random.rand(p)-0.5)
    eta = X.dot(theta0)
    error = np.sqrt(sig2)*np.random.randn(n)
    y = eta + error
    return y, X, theta0

np.random.seed(seed)
n, p, t0 = 100, 20, 0.5
sig2 = t0**2*p
nsim = 250
holder = []
for ii in range(nsim):
    y, X, theta0 = dgp_yX(n, p, t0=t0, sig2=sig2)
    mdl = LinearRegression(fit_intercept=True).fit(y=y,X=X)
    l2_error = np.sum((theta0 - mdl.coef_)**2)
    ytest, Xtest, _ = dgp_yX(100*n, p, theta0=theta0, sig2=sig2)
    eta_test = mdl.predict(Xtest)
    # Calculate theoretical R(MSE), R(MAE)
    risk_mse, risk_mae = sig2 + l2_error, np.sqrt((sig2 + l2_error)*2/np.pi)
    hat_mse, hat_mae = MSE(ytest, eta_test), MAE(ytest, eta_test)
    tmp = pd.DataFrame({'risk_mse':risk_mse, 'hat_mse':hat_mse, 'risk_mae':risk_mae, 'hat_mae':hat_mae}, index=[ii])
    holder.append(tmp)
    del tmp
df_risk = pd.concat(holder).rename_axis('idx').reset_index().melt('idx',None,'tmp')
df_risk = df_risk.assign(tt=lambda x: x.tmp.str.split('_',2,True).iloc[:,0],
                         metric=lambda x: x.tmp.str.split('_',2,True).iloc[:,1]).drop(columns=['tmp'])
df_risk = df_risk.pivot_table('value',['idx','metric'],'tt').reset_index()

plotnine.options.figure_size = (8, 4)
gg_risk = (ggplot(df_risk,aes(x='hat',y='risk',color='metric')) + theme_bw() +
           geom_point() + labs(x='Empirical risk',y='Theoretical risk') +
           facet_wrap('~metric',labeller=labeller(metric={'mae':'MAE','mse':'MSE'}),scales='free') +
           theme(subplots_adjust={'wspace': 0.15}) + guides(color=False) +
           ggtitle('Figure 2A: Empirical and theoretical risk estimates') +
           geom_abline(slope=1,intercept=0,linetype='--'))
gg_risk
```

Figure 2A confirms that the empirical risk estimates are closely aligned with their theoretical counterparts. Once again, with a sufficient sample size, the scatter plot would show no variation outside the line going through the origin. In section (1), knowledge of the population standard deviation of the statistic ($\sigma$) was needed in order to calculate the test statistic ($s_2$). Because this quantity is unknown, the [bootstrap](https://en.wikipedia.org/wiki/Bootstrapping_(statistics)) can be used to estimate the variance of the performance metric of interest (e.g. MSE & MAE). If $\hat\sigma_{BS}$ is the empirical standard deviation of the bootstrap, then the population standard deviation can be estimated by multiplying it by the number of samples $n$.

In the simulation below, the accuracy of the bootstrap standard deviation will be compared to the true population standard deviation for the MSE. I am using the MSE rather than the MAE because the former can be characterized by a chi-square distribution:

$$
\begin{align*}
\frac{1}{\sqrt{\sigma^2 + \|\theta_0 - \theta \|_2^2}}\sum_{i=1}^n e_i^2 &\sim \chi^2_n  \\
Var(\chi^2) &= 2n \\
Var(\hat{\text{MSE}}(\theta)) &= \frac{\sigma^2_{MSE}}{n} = \frac{2[\sigma_u^2+\|\theta_0 - \theta \|_2^2]^2}{n}
\end{align*}
$$

The quality of the bootstrapped variance can be compared to its true second moment (that is unknown in practice):

$$
\begin{align}
\sigma^2_{MSE} &\approx n\cdot \hat\sigma^2_{BS} \label{eq:sig2_bs}
\end{align}
$$

Figure 2B below shows that the bootstrapped variance is almost an unbiased estimator of the true variance.

```python
n_bs = 1000
nsim = 1000
np.random.seed(seed)
ntrain = 100
ntest = 1000

holder = np.zeros([nsim,2])
for ii in range(nsim):
    y, X, theta0 = dgp_yX(ntrain, p, t0=t0, sig2=sig2)
    mdl = LinearRegression(fit_intercept=True).fit(y=y,X=X)
    l2_error = np.sum((theta0 - mdl.coef_)**2)
    sig2e = l2_error + sig2
    ytest, Xtest, _ = dgp_yX(ntest, p, theta0=theta0, sig2=sig2)
    eta_test = mdl.predict(Xtest)
    eta_boot = pd.Series(eta_test).sample(ntest*n_bs,replace=True)
    y_boot = ytest[eta_boot.index].reshape([ntest,n_bs])
    eta_boot = eta_boot.values.reshape([ntest,n_bs])
    res_boot = y_boot - eta_boot
    mse_boot = MSE(y_boot, eta_boot,multioutput='raw_values')
    sig2_mse = mse_boot.var()*len(ytest)
    sig2_gt = 2*sig2e**2
    holder[ii] = [sig2_mse, sig2_gt]
dat_sig2 = pd.DataFrame(holder,columns=['hat','gt'])
dat_sig2 = np.sqrt(dat_sig2)

xx, xm = np.ceil(dat_sig2.max().max()), np.floor(dat_sig2.min().min())
plotnine.options.figure_size = (4.5, 4)
gg_sig2 = (ggplot(dat_sig2, aes(x='gt',y='hat')) + theme_bw() +
           geom_point(alpha=0.5,size=1) +
           labs(y='Estimated standard deviation',x='True standard deviation') +
           ggtitle('Figure 2B: Bootstrap standard deviation as an approximation') +
           geom_abline(slope=1,intercept=0,color='blue') +
           scale_x_continuous(limits=[xm, xx]) + scale_y_continuous(limits=[xm, xx]))
gg_sig2
```

## (3) Applied example

At this point we are ready to run a simulation for the MSE & MAE by modifying the procedure outlined in section (1). Instead of estimating the true mean of the Gaussian, the risk of a linear regression model's MSE & MAE will be estimated. The pipeline is as follows:

1. Learn $f_\theta$ on an independent training dataset
2. Calculate MSE & MAE on an independent test set
3. Use the test set to obtain the bootstrap variance of the MSE or MAE: $\hat\sigma^2_{1,BS}$
4. Get an upper-estimate of performance for the null: $\hat{\text{MSE}}_0 = \hat{\text{MSE}}_1 + k \cdot \hat\sigma_{1,BS}$
5. Set the null hypothesis: $H_0: \text{MSE} \geq \hat{\text{MSE}}_0$
6. Find the sample size needed to obtain 80% power and its associated critical value $t_\alpha$ using \eqref{eq:quantile}
7. Estimate model performance on the prospective test set
8. Reject the null if $(\hat{\text{MSE}}_2-\hat{\text{MSE}}_0)/\hat\sigma_{2,BS} < t_\alpha$

In the code block below I am using a [studentized bootstrap](https://www.textbook.ds100.org/ch/18/hyp_studentized.html) to estimate the standard error on both the testing and prospective validation sets. A one-sided confidence interval (CI) of $k \cdot \hat\sigma_{BS}$ is an approximation on the CI at the $\Phi(k)$ level: $z_{\Phi(-k)}\cdot \sqrt{\sigma^2 / n}$. Since the estimate of $\hat\sigma_{BS}$ can be biased downwards for smaller sample sizes and skewed distributions, the studentized bootstrap can "correct" for this. Specifically, the bootstrapped statistic is mapped to a t-score:

$$
\begin{align*}
t^*_b = \frac{\hat{\text{MSE}^*_b} - \hat{\text{MSE}}}{\hat\sigma{_b^*}}
\end{align*}
$$

This approach requires re-bootstrapping a given bootstrapped sample and estimating its standard error $\hat\sigma_b^*$. Though this is computationally intensive, it helps give close to exact nominal coverage levels, and, as I show, can be easily vectorized with the `sample` attribute in `pandas` classes and using 3-D arrays in `numpy`. The upper bound of the interval is: $\hat{\text{MSE}} - q_{\alpha} \hat\sigma_{BS}$, where $q_\alpha$ is the empirical quantile of the $t^*_b$ distribution. For example if $k=1.5$, but $q_{\alpha}=-1.6$, then the standard errors are "too small", so we can adjust by rescaling $\hat\sigma_{BS} \gets \hat\sigma_{BS}\cdot (-q_{\alpha}/k)$. The simulations below will target a sample size needed to obtain 80% power, a type-I error rate of 5%, a $k$ of 1.5, and use a training and test set size of 150. The `power_find` function estimates that 399 samples will be needed in the prospective dataset to reject the null at these rates.

```python
def boot_mse_mae(eta, resp, k, n_bs=1000, n_student=250):
    nn = len(eta)
    mse_hat, mae_hat = MSE(resp, eta), MAE(resp, eta)
    eta_boot = pd.Series(eta).sample(nn*n_bs,replace=True)
    y_boot = pd.DataFrame(resp[eta_boot.index].reshape([nn,n_bs]))
    eta_boot = pd.DataFrame(eta_boot.values.reshape([nn,n_bs]))
    mse_boot = MSE(y_boot, eta_boot, multioutput='raw_values')
    mae_boot = MAE(y_boot, eta_boot, multioutput='raw_values')
    # Run studentized bootstrap for adjustment
    # Recall in the studentized bootstrap the 1-a quantile is used for the lowerbound and the a-quantile for the upper
    tmp1 = eta_boot.sample(frac=n_student,replace=True,axis=0)
    tmp2 = y_boot.iloc[tmp1.index]
    tmp1 = tmp1.values.reshape([n_student]+list(eta_boot.shape))
    tmp2 = tmp2.values.reshape([n_student]+list(y_boot.shape))
    sig_student_mse = np.mean((tmp2 - tmp1)**2,axis=1).std(0)
    sig_student_mae = np.mean(np.abs(tmp2 - tmp1),axis=1).std(0)
    t_mse = (mse_boot - mse_hat)/sig_student_mse
    t_mae = (mae_boot - mae_hat)/sig_student_mae
    k_adjust_mse = -np.quantile(t_mse,norm.cdf(-k))
    k_adjust_mae = -np.quantile(t_mae,norm.cdf(-k))
    # Get standard error estimates
    sig2_mse_n, sig2_mae_n = mse_boot.var(), mae_boot.var()
    # Scale by studentized-t factor
    sig2_mse_n, sig2_mae_n = sig2_mse_n*(k_adjust_mse/k)**2, sig2_mae_n*(k_adjust_mae/k)**2
    return sig2_mse_n, sig2_mae_n, k_adjust_mse, k_adjust_mae

def power_est(n2, k, n1, alpha):
    dist_true = cond_dist(k=k, n1=n1, n2=n2, null=True)
    dist_false = cond_dist(k=k, n1=n1, n2=n2, null=False)
    crit_value = dist_true.quantile(alpha)
    power = dist_false.cdf_x(crit_value)
    return power

def power_find(pp, k, n1, alpha):
    n2 = minimize_scalar(fun=lambda x: (power_est(x, k, n1, alpha)-pp)**2,method='brent').x
    return n2

ntrain, ntest = 150, 150
p, t0 = 20, 0.5
sig2 = t0**2*p / 2
nsim = 5000
k = 1.5
n_bs = 1000
n_student = 250
alpha = 0.05
power = 0.8

# --------------------------------- #
# (6) Find sample size needed for 80% power and critical value
nprosp = int(np.ceil(power_find(power, k=k, n1=ntest, alpha=alpha)))
# Critical value for prospective set
dist_prosp_true = cond_dist(k=k, n1=ntest, n2=nprosp, null=True)
crit_prosp = dist_prosp_true.quantile(alpha)

import os
if os.path.exists('df_sim.csv'):
    print('Not running')
else:
    np.random.seed(seed)

    stime = time()
    holder = []
    for ii in range(nsim):
        if (ii + 1) % 25 == 0:
            nsec, nleft = time() - stime, nsim - (ii+1)
            rate = (ii+1) / nsec
            tleft = nleft / rate
            print('Iteration (%i and %i), ETA: %i seconds' % (ii+1, nsim, tleft))
        # --------------------------------- #
        # (1) Learn f_theta on training set
        y, X, theta0 = dgp_yX(ntrain, p, t0=t0, sig2=sig2)
        mdl = LinearRegression(fit_intercept=True).fit(y=y,X=X)

        # --------------------------------- #
        # (2) Calculate the MSE and MAE on the test set
        ytest, Xtest, _ = dgp_yX(ntest, p, theta0=theta0, sig2=sig2)
        eta_test = mdl.predict(Xtest)
        mse_point, mae_point = MSE(ytest, eta_test), MAE(ytest, eta_test)

        # --------------------------------- #
        # (3) & (4) Calculate bootstrap variance and the population variance
        sig2_mse_n, sig2_mae_n, k_mse, k_mae = boot_mse_mae(eta=eta_test, resp=ytest, k=k, n_bs=n_bs, n_student=n_student)
        sig2_mse, sig2_mae = sig2_mse_n * ntest, sig2_mae_n * ntest # Population level estimate

        # --------------------------------- #
        # (5) Set upper bound on null
        mse_null = mse_point + k*np.sqrt(sig2_mse_n)
        mae_null = mae_point + k*np.sqrt(sig2_mae_n)
        del mse_point, mae_point

        # --------------------------------- #
        # (7) Generate prospective test set
        yprosp, Xprosp, _ = dgp_yX(nprosp, p, theta0=theta0, sig2=sig2)

        # --------------------------------- #
        # (8) Run inference using studentized bootstrap
        eta_prosp = mdl.predict(Xprosp)
        mse_prosp, mae_prosp = MSE(yprosp, eta_prosp), MAE(yprosp, eta_prosp)
        sig2_mse_n_prosp, sig2_mae_n_prosp, k_mse_prosp, k_mae_prosp = boot_mse_mae(eta=eta_prosp, resp=yprosp,
                                                                                k=k, n_bs=n_bs, n_student=n_student)
        z_mse, z_mae = (mse_prosp - mse_null) / np.sqrt(sig2_mse_n_prosp), (mae_prosp - mae_null) / np.sqrt(sig2_mae_n_prosp)
        #print('Z-score for MSE: %0.2f, MAE: %0.2f' % (z_mse, z_mae))

        # These are statistical quantities known only to the "simulator"
        l2_error = np.sum((theta0 - mdl.coef_)**2)
        mse_gt = l2_error + sig2
        mae_gt = np.sqrt(mse_gt * 2 / np.pi)
        sig2_mse_gt = 2*(l2_error + sig2)**2

        tmp = pd.DataFrame({'mse_gt':mse_gt, 'mse_null':mse_null, 'mae_gt':mae_gt, 'mae_null':mae_null,
                            'k_mse':k_mse, 'k_mae':k_mae, 'k_mse_prosp':k_mse_prosp, 'k_mae_prosp':k_mae_prosp,
                            'z_mse':z_mse, 'z_mae':z_mae,'crit':crit_prosp, 'nprosp':nprosp,
                            'sig2_prosp':sig2_mse_n_prosp*nprosp, 'sig2_test':sig2_mse_n*ntest,'sig2_gt':sig2_mse_gt},
                           index=[ii])
        holder.append(tmp)
    df_sim = pd.concat(holder).rename_axis('idx').reset_index()
    df_sim.to_csv('df_sim.csv',index=False)
# Combine results
df_sim = pd.read_csv('df_sim.csv')
dat_coverage = df_sim.melt('idx',['mse_gt','mse_null','mae_gt','mae_null'],'tmp')
dat_coverage = dat_coverage.assign(metric=lambda x: x.tmp.str.split('_',2,True).iloc[:,0],
              tt=lambda x: x.tmp.str.split('_',2,True).iloc[:,1]).drop(columns=['tmp'])
dat_coverage = dat_coverage.pivot_table('value',['idx','metric'],'tt').reset_index()
dat_coverage = dat_coverage.assign(null_is_false=lambda x: x['null'] > x['gt'])
dat_power = df_sim.melt(['idx','crit'],['z_mse','z_mae'],'metric')
dat_power = dat_power.assign(reject=lambda x: x.value < x.crit, metric=lambda x: x.metric.str.replace('z_',''))
dat_power = dat_power.merge(dat_coverage[['idx','metric','null_is_false']],'left',['idx','metric'])

plotnine.options.figure_size = (7, 5)
brks = list(np.concatenate((np.linspace(-13,crit_prosp,14),np.linspace(crit_prosp,6,9)[1:])))
dat_txt = dat_power.groupby(['metric','null_is_false']).reject.mean().reset_index()
dat_txt = dat_txt.assign(lbls=lambda x: (x.reject*100).map('{:,.1f}%'.format))
dat_txt = dat_txt.assign(x=lambda x: np.where(x.null_is_false==True,-8, -8),
                         y=lambda x: np.where(x.null_is_false==True,750, 90))

di_metric = {'mse':'MSE', 'mae':'MAE'}
di_null = {'False':'Null is True', 'True':'Null is False'}
gg_zdist = (ggplot(dat_power, aes(x='value',fill='reject')) + theme_bw() +
            geom_histogram(alpha=0.5,color='black',breaks=brks) +
            facet_grid('null_is_false~metric',scales='free',
                       labeller=labeller(metric=di_metric, null_is_false=di_null)) +
            labs(x='Z-score', y='Count (out of 5000)') + guides(fill=False,color=False) +
            ggtitle('Figure 3A: Distribution of second-stage z-scores\nVertical line shows critical value') +
            geom_vline(xintercept=crit_prosp,color='black') +
            geom_text(aes(x='x',y='y',label='lbls'),color="#00BFC4",data=dat_txt,size=10,inherit_aes=False))
gg_zdist
```

Figure 3A shows that the two-stage approach is extremely accurate! The simulated power frequency is between 80-81% and the type-I error between 3-5%, just as was expected.

```python
dat_sig2 = df_sim.melt(['idx','sig2_gt'],['sig2_prosp','sig2_test'],'msr').assign(msr=lambda x: x.msr.str.replace('sig2_',''))
di_msr = {'prosp':'Prospective', 'test':'Test', 'gt':'Ground-Truth'}

plotnine.options.figure_size = (6, 3)
gg_sig2 = (ggplot(dat_sig2,aes(x='np.sqrt(value)',y='np.sqrt(sig2_gt)',color='msr')) + theme_bw() +
           geom_point(size=1,alpha=0.5) + geom_abline(slope=1,intercept=0) +
           facet_wrap('~msr',labeller=labeller(msr=di_msr)) +
           labs(x='Estimated standard deviation',y='Actual standard deviation') +
           ggtitle('Figure 3B: Bootstrap variance quality for MSE') +
           guides(color=False) + theme(panel_spacing_x=0.15))
print(gg_sig2)

dat_k = df_sim.melt('idx',['k_mse','k_mae','k_mse_prosp','k_mae_prosp'],'tmp').assign(tmp=lambda x: x.tmp.str.replace('k_',''))
dat_k = dat_k.assign(metric=lambda x: np.where(x.tmp.str.contains('mse'),'mse','mae'),
             dset=lambda x: np.where(x.tmp.str.contains('prosp'),'prosp','test')).drop(columns='tmp')
dat_k = dat_k.assign(metric=lambda x: pd.Categorical(x.metric,['mse','mae']))

plotnine.options.figure_size = (7, 3)
gg_k = (ggplot(dat_k,aes(x='value',fill='dset')) + theme_bw() +
        geom_density(alpha=0.5) + geom_vline(xintercept=k) +
        facet_wrap('~metric',scales='free',labeller=labeller(metric={'mse':'MSE','mae':'MAE'})) +
        scale_fill_discrete(name='Data',labels=['Prospective','Test']) +
        theme(subplots_adjust={'wspace': 0.15},legend_position=(0.39,0.70)) +
        labs(x='q_alpha',y='Density') + ggtitle('Figure 3C: Distribution of studentized quantiles'))
print(gg_k)
```

Figure 3B shows that the estimate of the population variance for the MSE is reasonably close to the one obtained from the studentized bootstrap. The variance tends to be overestimated slightly, and Figure 3C explains shows that this is the case because the (negative) $\alpha$ quantile of the studentized bootstrapped statistics tends to be larger than the target of $k=1.5$. In other words, the bootstrap-standard error is usually adjusted upwards. Though not shown here, using a vanilla bootstrap approach will cause the type-II errors and proportion of true nulls to be slightly too large. In fact the empirical coverage of the null, after the studentized adjustment, is basically spot on. For a properly estimated variance, the null hypothesis should be true/false $\Phi(-k)$/$\Phi(k)$ percent of the time.

```python
print(np.round(dat_coverage.groupby(['metric']).null_is_false.mean().reset_index().assign(expectation=[norm.cdf(k),norm.cdf(k)]),3))
```

      metric  null_is_false  expectation
    0    mae          0.939        0.933
    1    mse          0.935        0.933

## (4) Conclusion

This post has shown how to construct a prospective trial to validate any ML regression model for any performance metric of interest. There are a few caveats. First, the statistic of interest needs to have a bootstrapped distribution that is reasonably "smooth". Discontinuities or extreme skewness will limit the quality of the estimate. Second, the distribution of the data for the test set and prospective trial needs to representative.

On a statistical level there are several, what I find to be, surprising conclusions that this analysis has shown:

1. The sum of a truncated normal and standard Gaussian can be re-written as a conditionally correlated bivariate normal whose density has a known closed-form solution, and whose CDF can be calculated from leveraging the CDF from a multivariate normal distribution \eqref{eq:cdf_X}.
2. The conditional distribution noted above is only a function of the ratio of the two sample sizes, rather than their absolute level, and $k$.
3. Sample-size calculations can be determined by specifying only three of the four terms: $n_2/n_1$, $k$, $\alpha$, or $1-\beta$ (the power).

What is remarkable about these three conclusions is that they are completely independent of the choice of ML model or performance metric. In other words, the sample size calculation used in section (3) would work just as well for a [Random Forest](https://en.wikipedia.org/wiki/Random_forest) predicting house prices and evaluated by its [Tweedie deviance](https://en.wikipedia.org/wiki/Tweedie_distribution#The_Tweedie_deviance) as it would for a [Gaussian Process Regression](https://en.wikipedia.org/wiki/Kriging) predicting patient volumes and evaluated using [R-squared](https://en.wikipedia.org/wiki/Coefficient_of_determination). This approach proves to be generalizable because the variance of the performance metric is the unknown quantity that is allowed to vary. In the binary classification case the variance of the performance metric was known because the variance of a binomial proportion is a function of its mean, whereas the threshold was the random variable.

The two-stage approach also means that the "posterior" distribution of outcomes on the prospective validation set can be defined by the following 2x2 table:

|       | $H_0$ is true | $H_0$ false     |
| -----------        |    -----:  |   ------ |
| Reject $H_0$      | $\alpha\cdot$$\Phi$($-k$)      | (1-$\beta$)$\Phi$($k$)   |
| Do not reject $H_0$   | (1-$\alpha$)$\Phi$($-k$)    | $\beta\cdot$$\Phi$($k$)      |

This gives researchers significant freedom to control the uncertainty for each of these outcome categories. Lastly, it should be noted that the choice of using $k$-standard deviations above the point estimate is for mathematical tractability, and probably not for actually applied use. In almost all applied use-cases, the upper-bound will be picked by subject matter experts and that value of $k$ backed-out from this choice, rather than the other way around. Though the ordering of this decision is essential for real-world applications it is immaterial to the mathematics and hence the simpler form is described.


# Raumluftqualität 2.0

## Zeitliche Entwicklung der CO_2-Konzentration in Räumen

In einem gut gelüfteten, leeren Raum wird sich zunächst genau so viel CO_2 befinden, wie in der Außenluft.

Wenn sich dann Personen in den Raum begeben und CO_2 freisetzen, wird die CO_2-Konzentration langsam zunehmen. Auf welchen Wert sie sich schließlich einstellt, hängt vom Außenluftvolumenstrom ab, mit dem der Raum belüftet wird.

Bei einem völlig unbelüfteten Raum wird das von den Personen produzierte CO_2 sich in der Raumluft immer stärker anreichern, wobei je Zeiteinheit die gleiche Menge an CO_2 freigesetzt wird.

### Beispiel:

In einem Raum von $15 \rm m^2$ Grundfläche bei $2.5 \rm m$ Geschosshöhe befinden sich 2 Personen, die je Person $30\,{\frac{\ell}{h}}$ CO_2 ausatmen. Die CO_2-Konzentration der Außenluft ist 400 ppM. Im Raum sollen 1200 ppM CO_2 zulässig sein.

Stellen Sie die zeitliche Entwicklung der CO_2-Konzentration in einem Diagramm dar.

Gegeben:

Raumvolumen: $V_{\rm ra} = 15 {\rm m^2}\cdot 2.5 {\rm m} = 37.5 {\rm m^3}$

CO_2-Produktion: $\dot V_{\rm sch} = 2\cdot 30 {\rm \dfrac{\ell}{h}} = 60\,000\, {\rm\dfrac{cm^3}{h}}$

Damit ergibt sich die Änderungsrate
$
  \dot k = \cfrac{\dot V_{\rm sch}}{V_{\rm ra}}
         = {\rm\dfrac{60\,000\,cm^3}{37.5\, m^3\cdot h}}
         = {\rm 1600 \dfrac{ppM}{h}}
$

Für das Schadstoffvolumen im Raum ergibt sich:

\begin{align}
  k(t)&= 400 {\rm ppM} + 1600\,{\rm\dfrac{ppM}{h}}\, t
\end{align}

Dies Ergebnis wird in den folgenden Zeilen in einem Diagramm dargestellt:

```python
import matplotlib.pyplot as plt
%config InlineBackend.figure_format = 'retina'

import pandas as pd
import numpy as np
```

```python
lt = np.linspace(0,120,13) # 10-min Schritte
df = pd.DataFrame(
    {
        't': lt,
        'k': 400 + 1600*lt/60 # 60min = 1h
    }
)

display(df.T)
ax=df.plot(x='t',y='k', label='$k = k(t)$')
ax.axhline(1200,c='r')
ax.grid()
ax.set(
    xlim=(0,120),xlabel='Zeit $t$ in $min$',
    ylabel='CO_2-Konzentration $k$ in $\mathrm{ppM}$'
);
```

Die zulässige CO_2-Konzentration wird bereits nach kurzer Zeit (etwa 30 min) erreicht. Nach etwa einer Stunde ist die Raumluftqualität inakzeptabel.

### Aufgabe

In einem Gebäude ($400 \rm m^2$ Grundfläche, $3.50 \rm m$ Raumhöhe) arbeiten 120 Personen körperlich mittelschwer belastet. Berechnen Sie unter der Voraussetzung, dass das Gebäude nicht belüftet wird, wie sich die CO_2-Konzentration im Raum entwickelt. Die CO_2-Konzentration der Außenluft beträgt 400 ppM.

Nach welcher Zeit wird die zulässige CO_2-Konzentration von 1200 ppM überschritten?

Stellen Sie den Vorgang in einem Diagramm dar.

```python
A_ra = 400 # m**2
h_ra = 3.5 # m
V_ra = A_ra*h_ra
V_ra
```

    1400.0

```python
n = 120 # Personen
dV_co2_person = 30e-3 # 30 l/h

dV_co2 = n*dV_co2_person
dV_co2
```

    3.5999999999999996

```python
k_0 = k_au = 400e-6 # 400 ppM
k_zul = 1200e-6 # 1200 ppM

lt = np.linspace(0,2) # Zeitintervall von vier Stunden

df = pd.DataFrame(
    {
        't': lt,
        'k': (k_0 + dV_co2/V_ra*lt)*1e6 # in ppM
    }
)

df.head().T
```

      0
      1
      2
      3
      4

      t
      0.0
      0.040816
      0.081633
      0.122449
      0.163265

      k
      400.0
      504.956268
      609.912536
      714.868805
      819.825073

```python
ax = df.plot(x='t',y='k',label='$k=k(t)$')

ax.set(
    xlim=(0,4),xlabel='Zeit $t$ in Stunden',
    ylim=(0,3500),ylabel='CO_2-Konzentration in ppM'
)

ax.axhline(k_zul*1e6,c='r')
ax.grid()
```

Nach einer Stunde (von Hand berechnet):

```python
(k_au + dV_co2/V_ra)*1e6
```

    2971.4285714285716


```python
%load_ext autoreload
%autoreload 2

import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import timedelta
from itertools import groupby, count

sys.path.append('../../src')

from utils.formatter import rename_columns, pivot_database, apply_sct_gain
from utils.functions import clean_folder, load_datasets_from_csv
from di.functions import append_di_column
from plot import show_serie
from paths import DB_DIR, FMTD_DB_DIR, FXD_DB_DIR
```

```python
clean_folder(FXD_DB_DIR, "*.csv")
```

```python
dts = load_datasets_from_csv(FMTD_DB_DIR).copy()
```

```python
# DHT11 humidity data is invalid
show_serie(dts['house']['H'])
```

```python
show_serie(dts['pump']['H'])
```

```python
# Searching the most correlate environment with the ambient where DHT11 with wrong data was
# "house" corresponds to the courtyard ambient, where the DHT11 was

a = dts['house']['T'].corr(dts['computer']['T'])
b = dts['house']['T'].corr(dts['external']['T'])
c = dts['house']['T'].corr(dts['fridge']['T'])

# The most correlated ambient found was "computer" ambient.
print(a, b, c)
```

    0.967366180459045 0.9328857343052038 0.8690161947744548

```python
## Using a simple Rule of Three, as as shown,
```

\begin{align}
\ H_h & = \frac{H_c}{T_c}\times T_h \
\end{align}

```python
# FIX DHT11 HUMIDITY DATA
data = pd.concat([dts['computer'][['T','H']] , dts['house']['T']], keys=['Quarto2','Pátio'], axis=1)
new_h = (data['Quarto2']['H']/data['Quarto2']['T'])*data['Pátio']['T']
```

```python
# Showing new humidity (blue) and old house humidity (orange)
fig, ax = plt.subplots(figsize=(15,4))
plt.plot(new_h)
plt.plot(data['Quarto2']['H'])
```

```python
# Checking Covariance and Correlation between "House" temperature and "New" humidity
corr = dts['house']['T'].corr(new_h)
cov = dts['house']['T'].cov(new_h)
print("Corr: {:.2f}, Cov: {:.2f}".format(corr, cov))
```

    Corr: -0.83, Cov: -15.61

```python
# Checking Covariance and Correlation between "House" temperature and "Computer" humidity
corr = dts['house']['T'].corr(dts['computer']['H'])
cov = dts['house']['T'].cov(dts['computer']['H'])
print("Corr: {:.2f}, Cov: {:.2f}".format(corr, cov))
```

    Corr: -0.89, Cov: -14.99

```python
# "New" humidity was chosed
dts['house']['H'] = new_h
dts['pump']['H'] = new_h
```

```python
#Due to wrong installation, the first data samples are inverted
show_serie(dts['fridge']['Pwr'])
```

```python
# Fixing
fg_pwr=dts['fridge']['Pwr']
dts['fridge']['Pwr'] = np.where(fg_pwr.index < '2021-02-16 15:50:00', abs(fg_pwr), fg_pwr)
```

```python
# Showing
show_serie(dts['fridge']['Pwr'])
```

```python
# Export data
for key in dts:
    dts[key].to_csv(FXD_DB_DIR+"/{}.csv".format(key), float_format='%.4f')
```