```python
# Libraries Sympy and Numpy
from sympy import*
import numpy as np

# To define automatic printing mode (Not needed anymore)
# init_printing()

# For priting with text
from IPython.display import display, Latex

# Plotting Libraries
from mpl_toolkits import mplot3d

import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import matplotlib.ticker as ticker

#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
#%matplotlib inline

# Use Latex in plots
plt.rcParams['text.usetex'] = False
```

# 1. IPT System Frequency Splitting Model

## Equations Derivation

```python
# Definition of the symbolic variables

# Resistances --> Coils and Load
R_1, R_2, R_L = symbols('R_1, R_2, R_L',
                       real = True)

# Inductance and Capacitance
C_1, C_2, L_1, L_2, M_i = symbols('C_1, C_2, L_1,L_2, M_i',
                                 real = True)

# Reactance
X_1, X_2 = symbols('X_1, X_2',
                       real = True)

# Angular frequency
w_e = symbols('w_e', real = True)

# Input Voltage and Current
V_1, I_1 = symbols('V_1, I_1', real = True)
```

```python
# Reactance in each mesh
#X_1 = (w_e*L_1) - (1/(w_e*C_1))
#X_2 = (w_e*L_2) - (1/(w_e*C_2))
```

- Equivalent Circuit's Matrix Representation: $[Z][I] = [V]$

```python
# Impedance Matriz [Z]
Z_m = Matrix([[R_1 + (I*X_1), I*w_e*M_i],
            [I*w_e*M_i, R_2 + R_L + (I*X_2)]])

# Voltage Vector
V_m = Matrix([[V_1], [0]])
```

```python
# Solve the System of Equations
I_m = Z_m.inv()*V_m
```

```python
# Input Current
result_I = "Input Current: \n $${} = {}$$".format(latex(I_1),
                                                 latex(I_m[0]))
display(Latex(result_I))

# Separate Numerator and denominator of I1
I_1_n, I_1_d = fraction(I_m[0])
```

Input Current:
 $$I_{1} = \frac{V_{1} \left(- i R_{2} - i R_{L} + X_{2}\right)}{- i M_{i}^{2} w_{e}^{2} - i R_{1} R_{2} - i R_{1} R_{L} + R_{1} X_{2} + R_{2} X_{1} + R_{L} X_{1} + i X_{1} X_{2}}$$

```python
# Operate separately each polynomial
display(factor(I_1_n))
display(factor(I_1_d + (I*(M_i**2)*(w_e**2))))
```

$\displaystyle - i V_{1} \left(R_{2} + R_{L} + i X_{2}\right)$

$\displaystyle - i \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)$

```python
# Reconstruct the factored result
I_1 = (factor(I_1_n))/(factor(I_1_d + (I*(M_i**2)*(w_e**2)))
                       - (I*(M_i**2)*(w_e**2)))
display(simplify(I_1))
```

$\displaystyle \frac{V_{1} \left(R_{2} + R_{L} + i X_{2}\right)}{M_{i}^{2} w_{e}^{2} + \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)}$

- Input Impedance Expression $Z_{in}$:

```python
Z_in = V_1*(1/simplify(I_1))
display(Z_in)
```

$\displaystyle \frac{M_{i}^{2} w_{e}^{2} + \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)}{R_{2} + R_{L} + i X_{2}}$

- Freq. Splitting occurs when $\Im(Z_{in}) = 0$

```python
# Assume both reactance are equal for simplicity
# Common reactance X_e
X_e = symbols('X_e', real = True)

# Substitute the variable
Z_in = Z_in.subs([(X_1, X_e), (X_2, X_e)])

# Imaginary part of Zin
simplify(im(Z_in))
```

$\displaystyle \frac{X_{e} \left(- M_{i}^{2} w_{e}^{2} - R_{1} R_{2} - R_{1} R_{L} + X_{e}^{2} + \left(R_{2} + R_{L}\right) \left(R_{1} + R_{2} + R_{L}\right)\right)}{X_{e}^{2} + \left(R_{2} + R_{L}\right)^{2}}$

```python
# Operate separately
factor(-(R_1*R_2) - (R_1*R_L) + ((R_2 + R_L)*(R_1 + R_2 + R_L)))
```

$\displaystyle \left(R_{2} + R_{L}\right)^{2}$

## a) 1st Condition to only have one non-trivial root: $\Delta < 0$

```python
# Angular Resonance frequency (central)
w_c = 2*np.pi*(800e3)  # 800 kHz

# Assume coils' self-inductance equal [H]
L_c = 53.49e-6

# Frequency-dependent parameter
beta_c = (w_c*L_c)**2
```

```python
# Formula for condition Delta
Delta_c = lambda R, beta, k: (((R**2) - (2*beta))**2) - (4*(beta**2)*(1 - (k**2)))

# Inductive Coupling Coefficient Range k_i
k_i = np.arange(0, 1 + 0.01, 0.01)
print(k_i.shape)
print(k_i[-1])

# Load Resistance [ohms]
R_L = np.arange(1., 1000. + 1., 1.)
print(R_L.shape)
print(R_L[-1])

# Create a mesh
k_i_m, R_L_m = np.meshgrid(k_i, R_L)

# Calculate the function Delta
f_delta = Delta_c(R_L_m, beta_c, k_i_m)
print(f_delta.shape)
```

    (101,)
    1.0
    (1000,)
    1000.0
    (1000, 101)

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# Set the plot --> size(width, height)
fig1 = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig1.suptitle('Frequency Splitting: $\Delta$ < 0', fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R^{\'}_{L}$) [$\Omega$]')
axs.set_zlabel('Function $\Delta$')

# Title
axs.set_title('3D Plot of $\Delta$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_i_m, R_L_m, f_delta, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 2)

# Plot the contour plot
cp = plt.contour(k_i_m, R_L_m, f_delta, cmap = cm.Spectral)

# Define the levels to label
#c_labels = [0.e0]
# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.1)

# Information
axs.set_title('Contour Plot of $\Delta$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R^{\'}_{L}$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig1.subplots_adjust(wspace=0.3)
plt.show()
```

```python
#fig1.savefig ("delta_ipt.eps", format = 'eps')
fig1.savefig ("delta_ipt.svg", format = "svg", dpi = 1200)
```

## b) 2nd Condition to only have one non-trivial root:
$(q>0) \wedge (\Delta>0) \wedge (-q + \sqrt{\Delta} \leq 0)$

- The condition $q>0$ implies $R_{L}> \sqrt{2}\omega L$

```python
# We use the previously defined w_c, L_c, and beta_c

# RL is re-defined to comply with the condition:

# Load Resistance [ohms]
R_L = np.arange(np.ceil(np.sqrt(2)*w_c*L_c),
                1000. + 1., 1.)   # Ceil function --> Up the inferior limit
print(R_L.shape)
print(f'Load Resistance interval: ({R_L[0]}, {R_L[-1]}]')
```

    (620,)
    Load Resistance interval: (381.0, 1000.0]

- With the previous $R_L$ values, determine:

 Plot of $\Gamma(R_L, k_i) = -q(R_L) + \sqrt{\Delta(R_L, k_i)} \leq 0$.

 Only $\Delta > 0$ are importat, so the rest will be nan.

```python
# Formula for condition Gamma
Gamma_c = lambda R, beta, k: -((R**2) - (2*beta)) + (np.sqrt(Delta_c(R, beta, k)))

# Create a new mesh
k_i_m, R_L_m = np.meshgrid(k_i, R_L)

print(k_i_m.shape)
print(k_i_m)
print(R_L_m.shape)
print(R_L_m)
print()

# Calculate the Gamma function
f_gamma= Gamma_c(R_L_m, beta_c, k_i_m)
print(f'With sqrt(-1): \n {f_gamma} \n')

# Since 1 is not part of the result
# Invalid values (nan) are set to 1
f_gamma = np.nan_to_num(f_gamma, nan = 1)
print(f'After setting to 1: \n {f_gamma} \n')
```

    (620, 101)
    [[0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     ...
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]]
    (620, 101)
    [[ 381.  381.  381. ...  381.  381.  381.]
     [ 382.  382.  382. ...  382.  382.  382.]
     [ 383.  383.  383. ...  383.  383.  383.]
     ...
     [ 998.  998.  998. ...  998.  998.  998.]
     [ 999.  999.  999. ...  999.  999.  999.]
     [1000. 1000. 1000. ... 1000. 1000. 1000.]]

    With sqrt(-1):
     [[            nan             nan             nan ...             nan
                  nan      0.        ]
     [            nan             nan             nan ...             nan
                  nan      0.        ]
     [            nan             nan             nan ...             nan
                  nan      0.        ]
     ...
     [-12365.74627493 -12364.50058911 -12360.76354272 ...   -486.26641676
        -244.32643007      0.        ]
     [-12336.38550525 -12335.14282056 -12331.41477751 ...   -485.12790536
        -243.75454267      0.        ]
     [-12307.13565211 -12305.89595691 -12302.17688225 ...   -483.99358204
        -243.18475785      0.        ]]

    After setting to 1:
     [[ 1.00000000e+00  1.00000000e+00  1.00000000e+00 ...  1.00000000e+00
       1.00000000e+00  0.00000000e+00]
     [ 1.00000000e+00  1.00000000e+00  1.00000000e+00 ...  1.00000000e+00
       1.00000000e+00  0.00000000e+00]
     [ 1.00000000e+00  1.00000000e+00  1.00000000e+00 ...  1.00000000e+00
       1.00000000e+00  0.00000000e+00]
     ...
     [-1.23657463e+04 -1.23645006e+04 -1.23607635e+04 ... -4.86266417e+02
      -2.44326430e+02  0.00000000e+00]
     [-1.23363855e+04 -1.23351428e+04 -1.23314148e+04 ... -4.85127905e+02
      -2.43754543e+02  0.00000000e+00]
     [-1.23071357e+04 -1.23058960e+04 -1.23021769e+04 ... -4.83993582e+02
      -2.43184758e+02  0.00000000e+00]]

    <ipython-input-13-0713b495cec5>:2: RuntimeWarning: invalid value encountered in sqrt
      Gamma_c = lambda R, beta, k: -((R**2) - (2*beta)) + (np.sqrt(Delta_c(R, beta, k)))

```python
# Index of an specific value
result = np.where(f_gamma == 1)
print('Tuple of arrays returned : ', result)

# zip the 2 arrays to get the exact coordinates
listOfCoordinates= list(zip(result[0], result[1]))

# iterate over the list of coordinates
#for cord in listOfCoordinates:
#    print(cord)

# Just the first and last coordinate
print(f'1st: {listOfCoordinates[0]},\nlast: {listOfCoordinates[-1]} \n')

# Minimum value in sq_de
#ind = np.unravel_index(np.argmin(sq_delta, axis=None), sq_delta.shape)
#print(ind)
#print(sq_delta[ind])
```

    Tuple of arrays returned :  (array([  0,   0,   0, ..., 156, 156, 156]), array([ 0,  1,  2, ...,  8,  9, 10]))
    1st: (0, 0),
    last: (156, 10)

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# Set the plot --> size(width, height)
fig2 = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig2.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig2.suptitle('Frequency Splitting: $(q>0) \wedge (\Delta>0) \wedge (-q + \sqrt{\Delta} \leq 0)$',
             fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R_{L}$) [$\Omega$]')
axs.set_zlabel('Function $\Gamma$')

# Title
axs.set_title('3D Plot of $\Gamma$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_i_m, R_L_m, f_gamma, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig2.add_subplot(1, 2, 2)

# Plot the contour plot

cp = plt.contour(k_i_m, R_L_m,f_gamma,
                 levels = [-4e4, -2e4, -1e4, 0],
                 cmap = cm.Spectral)

# Define the levels to label
#c_labels = [0.e0]
# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.1)

# Information
axs.set_title('Contour Plot of $\Gamma$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R_{L}$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig2.subplots_adjust(wspace=0.3)
plt.show()
```

### - Another way yo visualize only the regions:

```python
"""
Terms definitions for the IPT problem
"""

# Definition of term p
# Return the value of p term
def p_termi(L, k_i):
    return (L**2)*(1 - (k_i**2))

# Definition of term q
# Return the value of q term
def q_termi(R, L, C):
    return (R**2) - ((2*L)/(C))

# Definition of term r
# Return the value of r term
def r_termi(C):
    return (1/(C**(2)))

# Definition of condition Delta
def Delta_IPT(R, L, C, k_i):

    p_aux = p_termi(L, k_i)
    #print(f'p {p_aux}')

    q_aux = q_termi(R, L, C)
    #print(f'q {q_aux}')

    r_aux = r_termi(C)
    #print(f'r {r_aux}')

    # Delta condition
    delta = (q_aux**2) - (4*p_aux*r_aux)
    #print(f'delta {delta}')

    return delta
```

```python
# Assume coils' self-inductance equal [H]
L_c = 53.49e-6

C_r = 739.92e-12

# Inductive Coupling Coefficient Range k_i
k_i = np.arange(0, 1 + 0.01, 0.01)
print(k_i.shape)
print(k_i[-1])

# Load Resistance [ohms]
R_L = np.arange(1., 1000. + 1., 1.)
print(R_L.shape)
print(R_L[-1])

# Create a mesh
k_i_m, R_L_m = np.meshgrid(k_i, R_L)
```

    (101,)
    1.0
    (1000,)
    1000.0

```python
# Calculate the function gamma (Condition 2)
# Only check for the region, the value of
# the function is not important

f_gamma = ((q_termi(R_L_m, L_c, C_r) > 0)
           & (Delta_IPT(R_L_m, L_c, C_r, k_i_m) > 0)
           & (-q_termi(R_L_m, L_c, C_r)
              + np.sqrt(Delta_IPT(R_L_m, L_c, C_r, k_i_m)) <= 0)).astype(int)
print(f_gamma.shape)
print(f_gamma)
```

    (1000, 101)
    [[0 0 0 ... 0 0 0]
     [0 0 0 ... 0 0 0]
     [0 0 0 ... 0 0 0]
     ...
     [1 1 1 ... 1 1 1]
     [1 1 1 ... 1 1 1]
     [1 1 1 ... 1 1 1]]

    <ipython-input-25-ae9987a86952>:8: RuntimeWarning: invalid value encountered in sqrt
      + np.sqrt(Delta_IPT(R_L_m, L_c, C_r, k_i_m)) <= 0)).astype(int)

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# Set the plot --> size(width, height)
fig2 = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig2.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig2.suptitle('Frequency Splitting: $(q>0) \wedge (\Delta>0) \wedge (-q + \sqrt{\Delta} \leq 0)$',
             fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R^{\'}_{L}$) [$\Omega$]')
axs.set_zlabel('Function $\Gamma$')

# Title
axs.set_title('3D Plot of $\Gamma$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_i_m, R_L_m, f_gamma, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig2.add_subplot(1, 2, 2)

# Plot the contour plot

cp = plt.contourf(k_i_m, R_L_m,f_gamma,
                 levels = [-1, 0, 1],
                 cmap = cm.Spectral)

# Define the levels to label
#c_labels = [0.e0]
# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, orientation = 'horizontal', shrink=0.6, aspect=20, pad = 0.1)

# Information
axs.set_title('Contour Plot of $\Gamma$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R^{\'}_{L}$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig2.subplots_adjust(wspace=0.5)
plt.show()
```

```python
fig2.savefig ("gamma_ipt.svg", format = "svg", dpi = 1200)
```

## c) 3nd Condition to only have one non-trivial root:
$(q<0) \wedge (\Delta>0) \wedge (-q - \sqrt{\Delta} \leq 0)$

This is not a condition to consider since it will provide at least

one non-trivial root.

- The condition $q<0$ implies $R_L < \sqrt{2}\omega L$

```python
# We use the previously defined w_c, L_c, and beta_c

# RL is re-defined to comply with the condition:

# Load Resistance [ohms]
R_L = np.arange(1., (np.sqrt(2)*w_c*L_c), 1.)
print(R_L.shape)
print(f'Load Resistance interval: [{R_L[0]}, {R_L[-1]})')
```

    (380,)
    Load Resistance interval: [1.0, 380.0)

- With the previous $R_L$ values, determine:

 Plot of $\Psi(R_L, k_i) = -q(R_L) - \sqrt{\Delta(R_L, k_i)} \leq 0$.

 Only $\Delta > 0$ are importat, so the rest will be nan.

```python
# Formula for condition Psi
Psi_c = lambda R, beta, k: -((R**2) - (2*beta)) - (np.sqrt(Delta_c(R, beta, k)))

# Create a new mesh
k_i_m, R_L_m = np.meshgrid(k_i, R_L)

print(k_i_m.shape)
print(k_i_m)
print(R_L_m.shape)
print(R_L_m)
print()

# Calculate the Psi function
f_psi= Psi_c(R_L_m, beta_c, k_i_m)
print(f'With sqrt(-1): \n {f_psi} \n')

# Since 1 is not part of the result
# Invalid values (nan) are set to 1
f_psi = np.nan_to_num(f_psi, nan = 1.)
print(f'After setting to 1: \n {f_psi} \n')
```

    (380, 101)
    [[0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     ...
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]
     [0.   0.01 0.02 ... 0.98 0.99 1.  ]]
    (380, 101)
    [[  1.   1.   1. ...   1.   1.   1.]
     [  2.   2.   2. ...   2.   2.   2.]
     [  3.   3.   3. ...   3.   3.   3.]
     ...
     [378. 378. 378. ... 378. 378. 378.]
     [379. 379. 379. ... 379. 379. 379.]
     [380. 380. 380. ... 380. 380. 380.]]

    With sqrt(-1):
     [[            nan 143239.12214754 141740.019677   ...   2891.66489673
        1445.83234529      0.        ]
     [            nan 143611.91175624 141894.01763856 ...   2891.72612341
        1445.86264939      0.        ]
     [            nan             nan 142173.38925858 ...   2891.82817368
        1445.91315905      0.        ]
     ...
     [            nan             nan             nan ...             nan
                  nan      0.        ]
     [            nan             nan             nan ...             nan
                  nan      0.        ]
     [            nan             nan             nan ...             nan
                  nan      0.        ]]

    After setting to 1:
     [[1.00000000e+00 1.43239122e+05 1.41740020e+05 ... 2.89166490e+03
      1.44583235e+03 0.00000000e+00]
     [1.00000000e+00 1.43611912e+05 1.41894018e+05 ... 2.89172612e+03
      1.44586265e+03 0.00000000e+00]
     [1.00000000e+00 1.00000000e+00 1.42173389e+05 ... 2.89182817e+03
      1.44591316e+03 0.00000000e+00]
     ...
     [1.00000000e+00 1.00000000e+00 1.00000000e+00 ... 1.00000000e+00
      1.00000000e+00 0.00000000e+00]
     [1.00000000e+00 1.00000000e+00 1.00000000e+00 ... 1.00000000e+00
      1.00000000e+00 0.00000000e+00]
     [1.00000000e+00 1.00000000e+00 1.00000000e+00 ... 1.00000000e+00
      1.00000000e+00 0.00000000e+00]]

    <ipython-input-12-f4cf8d1a4a28>:2: RuntimeWarning: invalid value encountered in sqrt
      Psi_c = lambda R, beta, k: -((R**2) - (2*beta)) - (np.sqrt(Delta_c(R, beta, k)))

```python
# Index of an specific value
result = np.where(f_psi == 1)
print('Tuple of arrays returned : ', result)

# zip the 2 arrays to get the exact coordinates
listOfCoordinates= list(zip(result[0], result[1]))

# iterate over the list of coordinates
#for cord in listOfCoordinates:
#    print(cord)

# Just the first and last coordinate
print(f'1st: {listOfCoordinates[0]},\nlast: {listOfCoordinates[-1]} \n')

# Minimum value in sq_de
#ind = np.unravel_index(np.argmin(sq_delta, axis=None), sq_delta.shape)
#print(ind)
#print(sq_delta[ind])
```

    Tuple of arrays returned :  (array([  0,   1,   2, ..., 379, 379, 379]), array([ 0,  0,  0, ..., 97, 98, 99]))
    1st: (0, 0),
    last: (379, 99)

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

# Set the plot --> size(width, height)
fig = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig.suptitle('Frequency Splitting: $(q<0) \wedge (\Delta>0) \wedge (-q - \sqrt{\Delta} \leq 0)$',
             fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R_{L}$) [$\Omega$]')
axs.set_zlabel('Function $\Psi$')

# Title
axs.set_title('3D Plot of $\Psi$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_i_m, R_L_m, f_psi, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig.add_subplot(1, 2, 2)

# Plot the contour plot

cp = plt.contour(k_i_m, R_L_m,f_psi,
                 levels = [0, 2e4, 6e4, 1e5],
                 cmap = cm.Spectral)

# Define the levels to label
#c_labels = [-2e10, 0, 1e10]

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, format = fmt, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.1)

# Information
axs.set_title('Contour Plot of $\Psi$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Inductive Coupling Coefficient ($k_{i}$)')
axs.set_ylabel('Load Resistance ($R_{L}$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig.subplots_adjust(wspace=0.3)
plt.show()
```

# 2. Freq. Splitting Model for Active Compensation Network

## Equations Derivation

```python
# Definition of the symbolic variables

# Resistances --> Coils and Load
R_1, R_2, R_L = symbols('R_1, R_2, R_L',
                       real = True)

# Inductance and Capacitance
C_1, C_2, L_1, L_2, M_i = symbols('C_1, C_2, L_1,L_2, M_i',
                                 real = True)

# Reactance
X_1, X_2, X_m = symbols('X_1, X_2, X_m',
                       real = True)

# Angular frequency
w_e = symbols('w_e', real = True)

# Input Voltage and Current
V_1, I_1 = symbols('V_1, I_1', real = True)
```

- Equivalent Circuit's Matrix Representation: $[Z][I] = [V]$

```python
# Impedance Matriz [Z]
Z_m = Matrix([[R_1 + (I*X_1), I*X_m],
            [I*X_m, R_2 + R_L + (I*X_2)]])

# Voltage Vector
V_m = Matrix([[V_1], [0]])
```

```python
# Solve the System of Equations
I_m = Z_m.inv()*V_m
```

```python
# Input Current
result_I = "Input Current: \n $${} = {}$$".format(latex(I_1),
                                                 latex(I_m[0]))
display(Latex(result_I))

# Separate Numerator and denominator of I1
I_1_n, I_1_d = fraction(I_m[0])
```

Input Current:
 $$I_{1} = \frac{V_{1} \left(- i R_{2} - i R_{L} + X_{2}\right)}{- i R_{1} R_{2} - i R_{1} R_{L} + R_{1} X_{2} + R_{2} X_{1} + R_{L} X_{1} + i X_{1} X_{2} - i X_{m}^{2}}$$

```python
# Operate separately each polynomial
display(factor(I_1_n))
display(factor(I_1_d + (I*(X_m**2))))
```

$\displaystyle - i V_{1} \left(R_{2} + R_{L} + i X_{2}\right)$

$\displaystyle - i \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)$

```python
# Reconstruct the factored result
I_1 = (factor(I_1_n))/(factor(I_1_d + (I*(X_m**2)))
                       - (I*(X_m**2)))
display(simplify(I_1))
```

$\displaystyle \frac{V_{1} \left(R_{2} + R_{L} + i X_{2}\right)}{X_{m}^{2} + \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)}$

- Input Impedance Expression $Z_{in}$:

```python
Z_in = V_1*(1/simplify(I_1))
display(Z_in)
```

$\displaystyle \frac{X_{m}^{2} + \left(R_{1} + i X_{1}\right) \left(R_{2} + R_{L} + i X_{2}\right)}{R_{2} + R_{L} + i X_{2}}$

- Freq. Splitting occurs when $\Im(Z_{in}) = 0$

```python
# Assume both reactance are equal for simplicity
# Common reactance X_e
X_e = symbols('X_e', real = True)

# Substitute the variable
Z_in = Z_in.subs([(X_1, X_e), (X_2, X_e)])

# Imaginary part of Zin
simplify(im(Z_in))
```

$\displaystyle \frac{X_{e} \left(- R_{1} R_{2} - R_{1} R_{L} + X_{e}^{2} - X_{m}^{2} + \left(R_{2} + R_{L}\right) \left(R_{1} + R_{2} + R_{L}\right)\right)}{X_{e}^{2} + \left(R_{2} + R_{L}\right)^{2}}$

```python
# Operate separately
factor(-(R_1*R_2) - (R_1*R_L) + ((R_2 + R_L)*(R_1 + R_2 + R_L)))
```

$\displaystyle \left(R_{2} + R_{L}\right)^{2}$

- Analyze the simple non-trivial root $\chi = 0$

```python
# Definition of the symbolic variables

# Capacitances T-Model
C_ep, C_Mp = symbols('C_ep, C_Mp',
                       real = True)

# Capacitances Pi-Model
C_e, C_M = symbols('C_e, C_M',
                       real = True)

# Capacitive Coupling Coefficient
k_c = symbols('k_c', real = True)
```

```python
# Replace the k_c
C_M = k_c*C_e

# Equivalent Pi --> T Model
C_ep = C_e + (2*C_M)
C_Mp = (C_e/C_M)*(C_ep)
```

```python
# Operation for the main resonance frequency
# Defining the equivalent parallel capacitance
C_pp = ((1/C_ep) + (1/C_Mp))**(-1)
simplify(C_pp)
```

$\displaystyle \frac{C_{e} \left(2 k_{c} + 1\right)}{k_{c} + 1}$

```python
# C_Mp also appears in the formula
simplify(C_Mp)
```

$\displaystyle 2 C_{e} + \frac{C_{e}}{k_{c}}$

## a) 1st Condition to only have one non-trivial root: $\Delta < 0$

 $L$ and $k_i$ are fixed from the IPT Model.

 External capacitance $C_e$ is defined according to standard values.

 $0 < k_c < 1$

 Frequency dependence is visualized separetely.

```python
# Defining the Inductive Coupling Coefficient k_i
k_i = np.array([0.214])

# Assume coils' self-inductance equal [H]
L_c = 53.49e-6

# External Capacitance [F]
C_e = np.array([0.668e-9])
```

- Define methods to calculate the condition $\Delta$

```python
##################################
## Define Methods for Calculations
##################################

# Definition of Equivalent Parallel Capacitance C_pp
def C_ppri(C, k_c):
    return (C*(1 + (2*k_c)))/(1 + k_c)

# Definition of the Mutual Capacitance in T-Model (C_Mp)
def C_Mpri(C, k_c):
    return C*((1/k_c) + 2)

# Definition of term p
# Return the value of p term
def p_term(L, k_i):
    return (L**2)*(1 - (k_i**2))

# Definition of term q
# Return the value of q term
def q_term(R, L, C, k_c, k_i):
    return (R**2) + (2*L*((k_i/C_Mpri(C, k_c))
                          - (1/C_ppri(C, k_c))))

# Definition of term r
# Return the value of r term
def r_term(C, k_c):
    return (1/((C_ppri(C, k_c))**2)) - (1/((C_Mpri(C, k_c))**2))

# Definition of condition Delta
def Delta_c1(R, L, C, k_c, k_i):

    # Auxiliar variables
    C_Mp = C_Mpri(C, k_c)
    #print(f'C_Mp {C_Mp}')

    C_pp = C_ppri(C, k_c)
    #print(f'C_pp {C_pp}')

    p_aux = p_term(L, k_i)
    #print(f'p {p_aux}')

    q_aux = q_term(R, L, C, k_c, k_i)
    #print(f'q {q_aux}')

    r_aux = r_term(C, k_c)
    #print(f'r {r_aux}')

    # Delta condition
    delta = (q_aux**2) - (4*p_aux*r_aux)
    #print(f'delta {delta}')

    return delta
```

```python
# Capacitive Coupling Coefficient Range k_c
k_c = np.arange(0.01, 1 + 0.01, 0.01)
print(k_c.shape)
print(k_c)

# Load Resistance [ohms]
R_L = np.arange(1., 1000. + 1., 1.)
print(R_L.shape)
print(R_L[-1])

# Create a mesh
k_c_m, R_L_m = np.meshgrid(k_c, R_L)

# Calculate the function Delta
f_delta = Delta_c1(R_L_m, L_c, C_e, k_c_m, k_i)
print(f_delta.shape)
```

    (100,)
    [0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1  0.11 0.12 0.13 0.14
     0.15 0.16 0.17 0.18 0.19 0.2  0.21 0.22 0.23 0.24 0.25 0.26 0.27 0.28
     0.29 0.3  0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.4  0.41 0.42
     0.43 0.44 0.45 0.46 0.47 0.48 0.49 0.5  0.51 0.52 0.53 0.54 0.55 0.56
     0.57 0.58 0.59 0.6  0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69 0.7
     0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8  0.81 0.82 0.83 0.84
     0.85 0.86 0.87 0.88 0.89 0.9  0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98
     0.99 1.  ]
    (1000,)
    1000.0
    (1000, 100)

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# Set the plot --> size(width, height)
fig1 = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig1.suptitle('Frequency Splitting Active CN: $\Delta$ Function', fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Capacitive Coupling Coefficient ($k_c$)')
axs.set_ylabel('Load Resistance ($R_L$) [$\Omega$]')
axs.set_zlabel('Function $\Delta$')

# Title
axs.set_title('3D Plot of $\Delta$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_c_m, R_L_m, f_delta, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 2)

# Plot the contour plot
cp = plt.contour(k_c_m, R_L_m, f_delta,
                  levels = [-0.5e11, 0, 1.5e11, 3e11, 4.5e11],
                  cmap = cm.Spectral)

# Define the levels to label
#c_labels = [0.e0]
# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.1)

# Information
axs.set_title('Contour Plot of $\Delta$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Capacitive Coupling Coefficient ($k_c$)')
axs.set_ylabel('Load Resistance ($R_L$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig1.subplots_adjust(wspace=0.3)
plt.show()
```

```python
#fig1.savefig ("delta_ipt.eps", format = 'eps')
fig1.savefig ("delta_acn.svg", format = "svg", dpi = 1200)
```

- Plot the behavior of resonance frequency (central) w.r.t. $k_c$

$w_r = \sqrt{\frac{1}{L {C^{\prime}_{p}}}}$

```python
# Define central resonance frequency
def f_operation(L, C, k_c):
    freq = (1/(2*np.pi))*np.sqrt((1)/(L*C_ppri(C, k_c)))
    return freq
```

```python
# Calculate the resonance frequency
f_res = (1e-3)*f_operation(L_c, C_e, k_c)
print(f_res.shape)
```

    (100,)

```python
#interactive plotting in separate window
%matplotlib qt

#normal charts inside notebooks
#%matplotlib inline

# Set the plot --> size(width, height)
fig2 = plt.figure(figsize=(10,10))
ax = fig2.gca()

# Axis labels
ax.set_xlabel('Capacitive Coupling Coefficient $k_{c}$')
ax.set_ylabel('Resonance Frequency ($f_{r}$) [kHz]')

# Title
ax.set_title('System\'s Resonance Frequency', fontsize = 25)

# Plot the surface
lines = ax.plot(k_c, f_res, 'g', linewidth = 2.5)

ax.xaxis.grid(True)
ax.yaxis.grid(True)

# Axis limits
ax.set_xlim([0, 1])

# Legend
#plt.legend(('Maxwell'), loc = 'upper right')

plt.show()

```

```python
fig2.savefig ("fresonant_acn.svg", format = "svg", dpi = 1200)
```

## b) 2nd Condition to only have one non-trivial root:
$(q>0) \wedge (\Delta>0) \wedge (-q + \sqrt{\Delta} \leq 0)$

```python
# Defining the Inductive Coupling Coefficient k_i
k_i = np.array([0.214])

# Assume coils' self-inductance equal [H]
L_c = np.array([53.49e-6])

# External Capacitance [F]
C_e = np.array([0.66e-9])
```

```python
# Capacitive Coupling Coefficient Range k_c
k_c = np.arange(0.01, 1 + 0.01, 0.01)
print(k_c.shape)
print(k_c)

# Load Resistance [ohms]
R_L = np.arange(1., 1000. + 1., 1.)
print(R_L.shape)
print(R_L[-1])

# Create a mesh
k_c_m, R_L_m = np.meshgrid(k_c, R_L)
```

    (100,)
    [0.01 0.02 0.03 0.04 0.05 0.06 0.07 0.08 0.09 0.1  0.11 0.12 0.13 0.14
     0.15 0.16 0.17 0.18 0.19 0.2  0.21 0.22 0.23 0.24 0.25 0.26 0.27 0.28
     0.29 0.3  0.31 0.32 0.33 0.34 0.35 0.36 0.37 0.38 0.39 0.4  0.41 0.42
     0.43 0.44 0.45 0.46 0.47 0.48 0.49 0.5  0.51 0.52 0.53 0.54 0.55 0.56
     0.57 0.58 0.59 0.6  0.61 0.62 0.63 0.64 0.65 0.66 0.67 0.68 0.69 0.7
     0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79 0.8  0.81 0.82 0.83 0.84
     0.85 0.86 0.87 0.88 0.89 0.9  0.91 0.92 0.93 0.94 0.95 0.96 0.97 0.98
     0.99 1.  ]
    (1000,)
    1000.0

```python
# Calculate the function gamma (Condition 2)
# Only check for the region, the value of
# the function is not important

f_gamma = ((q_term(R_L_m, L_c, C_e, k_c_m, k_i) > 0)
           & (Delta_c1(R_L_m, L_c, C_e, k_c_m, k_i) > 0)
           & (-q_term(R_L_m, L_c, C_e, k_c_m, k_i)
              + np.sqrt(Delta_c1(R_L_m, L_c, C_e, k_c_m, k_i)) <= 0)).astype(int)
print(f_gamma.shape)
print(f_gamma)
```

    (1000, 100)
    [[0 0 0 ... 0 0 0]
     [0 0 0 ... 0 0 0]
     [0 0 0 ... 0 0 0]
     ...
     [1 1 1 ... 1 1 1]
     [1 1 1 ... 1 1 1]
     [1 1 1 ... 1 1 1]]

    <ipython-input-17-fd1090a33310>:8: RuntimeWarning: invalid value encountered in sqrt
      + np.sqrt(Delta_c1(R_L_m, L_c, C_e, k_c_m, k_i)) <= 0)).astype(int)

```python
# Index of an specific value
result = np.where(np.isnan(f_gamma))
print('Tuple of arrays returned : ', result)

# zip the 2 arrays to get the exact coordinates
listOfCoordinates= list(zip(result[0], result[1]))

# iterate over the list of coordinates
#for cord in listOfCoordinates:
#    print(cord)

# Just the first and last coordinate
print(f'1st: {listOfCoordinates[0]},\nlast: {listOfCoordinates[-1]} \n')

# Minimum value in sq_de
#ind = np.unravel_index(np.argmin(sq_delta, axis=None), sq_delta.shape)
#print(ind)
#print(sq_delta[ind])
```

```python
#interactive plotting in separate window
#%matplotlib qt

#normal charts inside notebooks
%matplotlib inline

# Set the plot --> size(width, height)
fig1 = plt.figure(figsize=(15,10))

################
# 1st Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 1, projection = '3d')

# Complete figure information
fig1.suptitle('Frequency Splitting: $(q>0) \wedge (\Delta>0) \wedge (-q + \sqrt{\Delta} \leq 0)$',
             fontsize = 20)

# For a simple 3D plot
#ax = fig.gca(projection = '3d')

# Axis labels
axs.set_xlabel('Capacitive Coupling Coefficient ($k_c$)')
axs.set_ylabel('Load Resistance ($R_{L}$) [$\Omega$]')
axs.set_zlabel('Function $\Gamma$')

# Title
axs.set_title('3D Plot of $\Gamma$ Function', fontsize = 15)

# Plot the surface
surf = axs.plot_surface(k_c_m, R_L_m, f_gamma, cmap = cm.coolwarm,
                      linewidth = 0, antialiased = False)

# Figure size (single plot)
#plt.rcParams["figure.figsize"] = (30,30)

# Add a color bar which maps values to colors.
plt.colorbar(surf, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.05)

################
# 2nd Subplot
################

# Set the subplot for the two graphs
axs = fig1.add_subplot(1, 2, 2)

# Plot the contour plot
cp = plt.contourf(k_c_m, R_L_m, f_gamma,
                  levels = [-1, 0, 1],
                 cmap = cm.Spectral)

# Define the levels to label
#c_labels = [0.e0]
# To specify scientific notation
fmt = ticker.LogFormatterSciNotation()

axs.clabel(cp, inline = 1, colors = 'k', fmt = fmt, fontsize = 10)

# Color bar
plt.colorbar(cp, orientation = 'horizontal', shrink=0.6,
             aspect=20, pad = 0.09)

# Information
axs.set_title('Contour Plot of $\Gamma$ Function', fontsize = 15)

# Axis labels
axs.set_xlabel('Capacitive Coupling Coefficient ($k_c$)')
axs.set_ylabel('Load Resistance ($R_L$) [$\Omega$]')

axs.xaxis.grid(True)
axs.yaxis.grid(True)

# Distance between subplots
fig1.subplots_adjust(wspace=0.3)
plt.show()
```

```python
fig1.savefig ("fresonant_acn.svg", format = "svg", dpi = 1200)
```

```python
# 3. $\pi$-Model and $T$-Model equivalent equations

- Equivalent capacitive model for 4 plates
```

## $\pi$-Model:

```python
# Define the symbolic variables

# Capacitance
c_1, c_2, c_M = symbols('c_1 c_2 c_M', real = True)

# Angular frequency
w = symbols('w', real = True)
```

```python
# Common denominator
com_D = (1/(w*I))*(1/c_1 + 1/c_2 + 1/c_M)
```

```python
# T-model equivalent c'_1
Zp_1 = ((1/(w*I))**2)*((1/c_1)*(1/c_M))/com_D
simplify(Zp_1)
```

```python
# T-model equivalent c'_2
Zp_2 = ((1/(w*I))**2)*((1/c_2)*(1/c_M))/com_D
simplify(Zp_2)
```

```python
# T-model equivalent c'_M
Zp_M = ((1/(w*I))**2)*((1/c_1)*(1/c_2))/com_D
simplify(Zp_M)
```

## $T$-Model:

```python
# Define the symbolic variables

# Capacitance
cp_1, cp_2, cp_M = symbols('cp_1 cp_2 cp_M', real = True)
```

```python
# Common numerator
com_N = ((1/(w*I))**2)*((1/(cp_1*cp_M)) + (1/(cp_1*cp_2)) + (1/(cp_M*cp_2)))
```

```python
# Pi-model equivalent c_1
Z_1 = (com_N)/(1/(w*cp_2*I))
simplify(Z_1)
```

```python
# Pi-model equivalent c_2
Z_2 = (com_N)/(1/(w*cp_1*I))
simplify(Z_2)
```

```python
# Pi-model equivalent c_M
Z_M = (com_N)/(1/(w*cp_M*I))
simplify(Z_M)
```

# 4. Latex Formulas

$\Im(Z_{in}) = \frac{\chi [(R^{'}_{L} + R_{2})^{2} + \chi^{2} - (\omega M)^{2}]}{(R^{'}_{L} + R_{2})^{2} + \chi^{2}} = 0$

$\chi = \omega L - \frac{1}{\omega C}$

$k_{i} = \frac{M}{L} $

$R^{'}_{L} \gg R_{2} \Rightarrow (R^{'}_{L} + R_{2}) \approx R^{'}_{L}$

$f(\omega^{2}) = \left[ L^{2}(1 - k_{i}^{2}) \right] (\omega^{2})^{2} + \left[ R^{'2}_{L} - \frac{2L}{C} \right] \omega^{2} + \frac{1}{C^{2}}$

$\omega = \sqrt{\frac{-q \pm \sqrt{\Delta}}{2p}}$

$\Delta =  q^{2} - 4pr$

$-q \pm \sqrt{\Delta} \leq 0$

$\Im(Z_{in}) = \frac{\chi [(R^{'}_{L} + R_{2})^{2} + \chi^{2} - \chi_{m}^{2}]}{(R^{'}_{L} + R_{2})^{2} + \chi^{2}} = 0$

$\chi = \omega L - \frac{1}{\omega C^{'}_{p}}$

$\chi_{m} = \omega M - \frac{1}{\omega C^{'}_{M}}$

$\frac{1}{C^{'}_{p}} = \frac{1}{C^{'}} + \frac{1}{C^{'}_{M}} = \frac{C + C_{M}}{C(C + 2C_{M})}$

$f(\omega^{2}) = \left[ L^{2}(1 - k_{i}^{2}) \right] (\omega^{2})^{2} + \left[ R^{'2}_{L} + 2L \left(\frac{k_{i}}{C^{'}_{M}} - \frac{1}{C^{'}_{p}} \right) \right] \omega^{2} + \left( \frac{1}{C^{'2}_{p}} - \frac{1}{C^{'2}_{M}} \right)$

$k_{c} = \frac{C_{M}}{C} $

```python

```


# Drag Minimization of a Supersonic Body of Revolution Using Surrogate-Based Optimization

In this problem we want to find the three-dimensional shape with the lowest supersonic wave drag for a fixed length and fixed maximum diameter.  This problem has a known solution called the [Sears-Haack body](http://en.wikipedia.org/wiki/Sears–Haack_body) (see below).  This solution is useful, for example, in supersonic aircraft design.  To design an aircraft with low wave drag, it should have an area distribution close to that of the Sears-Haack body (it's more complicated than that, but that's the basic idea).

For this exercise, we assume that the solution is unknown, and use surrogate-based optimization to find the best design we can.  We parameterize the body of revolution as shown below, with six radii to optimize (the body has fore-aft symmetry, and the aft half is not shown). Points are clustered towards the nose where the shape changes rapidly.  The interior 6 points are bookended by a point at the nose with 0 radius so that the body is closed, and a point with a fixed radius of 5 so that the diameter of the body is constant (all dimensions are unitless in this problem).  An Akima spline is fit between the 8 points to define the full shape.  The length of the body is 100.

My colleague Dev Rajnarayan has provided the code that will be used to analyze the wave drag of the supersonic body as a function of the six radii.  You will not have access to the source code, and the output has some "discretization noise", both of which simulate common real-life scenarios.  Think of this code as a computational fluid dynamics (CFD) simulation.  CFD is typically very timing consuming to run (the code in this exercise is quick) and unless you have access to the source to add analytic gradients, or gradients are already provided, then optimizing with CFD in the loop is usually too costly.  Because of a limited time budget we want to find a good solution with as few function calls as possible.  We may also want to explore variations in our design after finding the optimal solution.  These requirements make the problem a good candidate for a surrogate model.  We will construct a global polynomial surrogate model of the simulation and will optimize the surrogate rather than the actual function.

The following write-up will guide you through the steps of the analysis, but you'll need to fill in some of the  details yourself.  This guide uses Matlab because the external code is written in Java and it's easier to interface with for this one-off analysis.  Everywhere where you need to complete the code is marked with a %TODO.

First, clear everything and load the [provided jar files](http://flowlab.groups.et.byu.net/me575/hw/supersonic-files.zip) (the external code).

```matlab
clear; close all; clc; clear java;

% Add jar files to java classpath
javaaddpath({strcat(pwd,'/me575hw6.jar'), strcat(pwd,'/colt.jar')});
wavedrag = @WaveDragAS.compute;  % rename function call for convenience

% syntax: drag = wavedrag(x)  % where x is a vector of length six
```

Let's define some reasonable bounds on the design variables:

```matlab
% Lower bound
lb = [0.05, 0.15, 0.65, 1.75, 3.00, 4.25];

% Upper bound
ub = [0.25, 0.40, 0.85, 2.25, 3.25, 4.75];
```

We need to sample our function to construct the initial surrogate, and will use Latin Hypercube Sampling (LHS). Matlab has a function called `lhsdesign` which can do this for us. We will start with a really small number of function calls (20) in our 6-dimensional space, but you can adjust that number as you like. The lhsdesign call will return numbers between 0 and 1, and so we need to scale those so that the number span lb to ub.

$$x = lb+(ub−lb)x_{lhs}$$

We also need to evaluate our wave drag coefficient function at each of the sample points. Note
that LHS is based on random number generation so you will get different sample points every time (unless you set the random number generator seed).

```matlab
% Construct LHS Sample (use lhsdesign)
M = 20;  % number of samples
N = 6;  % number of design variables

% TODO

% in a loop, create the actual x using the equation above,
% and evaluate wavedrag(x) at each x, and save the result a vector called f.

% TODO
% f =
```

We now need to decide what type of surrogate to construct.  For this exercise we choose a polynomial model, and more specifically a quadratic model. (To do a better job we should do parameter studies where we vary the order of the model, try other surrogate models, and use cross-validation to select the model, but we will skip all of that to keep things simpler in this exercise.) There are 6 dimensions so our quadratic model will have 28 unknowns that we need to estimate.  The constant $c$ provides 1 term, the linear term $l$ has 6 terms, and the quadratic term Q has $n(n+1)/2 = 21$ terms.

$$\text{drag} \approx c + l^T x + x^T Q x$$

Note that there are only 28 terms because we don't want to double count terms like $x_1 x_2$ and $x_2 x_1$, which are actually the same.  If we combine all the unknowns from $c$, $l$, and $Q$ into one unknown vector $w$, we can rewrite that same equation as:

\begin{align}
  \hat{f} &= w_1 + w_2 x_1 + w_3 x_2 + \ldots + w_8 x_1^2 + w_9 x_1 x_2 + w_{10} x_1 x_3 + \ldots + w_{28} x_6^2\\
  \hat{f} &= w^T x_{exp}
\end{align}

where $\hat{f}$ is our estimated value for $f$.  The expanded x vector $x_{exp}$ is something we are going to need a few times so I've provided a function called expandQuad.m to do that for you.

syntax: `x_exp = expandQuad(x)`

We can now use our samples to perform a maximum likelihood estimate of $w$.  We will form a linear system

\begin{equation}
\left[\begin{matrix}
f_1 \\ f_2\\ \vdots\\ f_M
\end{matrix}\right]
= \left[\begin{matrix}
[1 & x_1 & x_2 & \ldots & x_1^2 & x_1 x_2 &  x_1 x_3 & \ldots & x_6^2]_{\textrm{at sample 1}}\\
[1 & x_1 & x_2 & \ldots & x_1^2 & x_1 x_2 &  x_1 x_3 & \ldots & x_6^2]_{\textrm{at sample 2}}\\
\vdots\\
[1 & x_1 & x_2 & \ldots & x_1^2 & x_1 x_2 &  x_1 x_3 & \ldots & x_6^2]_{\textrm{at sample M}}\\
\end{matrix}
\right]
\left[\begin{matrix}
w_1 \\ w_2\\ w_3 \\\vdots\\ w_{28}
\end{matrix}\right]
\end{equation}

Which can be expressed as $f = \Phi w$ where $\Phi$ is

\begin{equation}
\Phi = \left[\begin{matrix}
{x_{exp}}_1\\
{x_{exp}}_2\\
\vdots\\
{x_{exp}}_M\\
\end{matrix}\right]
\end{equation}

```matlab
% Create PHI (in a loop)
PHI = zeros(M, 28);
% TODO
```

We can now estimate the w terms through a separate optimization process. In this case the optimization is easy because our parametrization is linear in w. The equation $f = \Phi w$ does not have a unique solution, instead we are interested in a least-squares approximate solution (minimizing the sum of the squared errors between prediction and actual function values).  We now have $\Phi$ and $f$ so we can use a least squares solution to find $w$.

```matlab
% Compute w from a least squares solution of Phi w = f
% TODO
```

With an estimate for $w$, we have created a surrogate model that can estimate the wave drag coefficient at any point (accuracy at any point is another question).  The estimate of the drag at any point $x$ using our surrogate quadratic model would simply be:

\begin{equation}
  \widehat{C_D} = x_{exp}^T w
\end{equation}

or in code:

`CDhat = expandQuad(x)*w;`

With our surrogate model constructed we can now perform an optimization using the surrogate model to find a minimum drag design (according to the surrogate).

```matlab
% optimization or surrogate model
J = @(x) expandQuad(x)*w;
x0 = [0.100,  0.198,  0.7,  1.800,  3.200,  4.550];  % starting point
[xopt, fopt] = fmincon(J, x0, [], [], [], [], lb, ub);
```

The resulting point will probably not be particularly accurate.  This is because we used an extremely small number of samples (20) to construct our initial surrogate in a 6-dimensional space, so we can't hope for high accuracy everywhere except near the sample points.  One strategy to improve this, is to sample more points initially to create a better global model.  This would be a good approach if wanted to explore global changes to the designs.  However, in this case we don't necessarily care about having an accurate model everywhere, but instead care about having an accurate model near the optimum.  We intentionally kept the initial sampling small because we are going to add our "optimum" solution to our sample, recreate the surrogate with the new point added, and then reoptimize using the new surrogate.  This should improve our surrogate near the actual optimum, and if we iterate a few times we should get a pretty good solution.  (Note: If we are not careful we could add points to our surrogate that are really close to points that our already there, which can cause numerical issues with our least squares solutions---but we won't worry about that in this example.)

Fortunately, we also know exactly how far our surrogate is off by evaluating the actual function at the point in question.  Let's run our optimization until the optimal point has a predicted drag coefficient within 5% (arbitrarily chosen) of the actual drag coefficient.

$$ \frac{f(\hat{x}^*) - \hat{f}(\hat{x}^*)}{f(\hat{x}^*)} < 0.05$$

where $x^*$ is the actual (unknown) optimal point, $\hat{x}^*$ is our esitmated optimal point, $\hat{f}$ is our surrogate model, and $f$ is the actual model.

We will see that even though the difference between our predicted function value $\hat{f}(\hat{x}^*)$ and the actual function value $f(\hat{x}^*)$ may be off by as much as 5%, the difference between the actual function value $f(\hat{x}^*)$ and the actual optimum $f^* = f(x^*)$ (which is what we actually care about) is smaller.  In addition to our error stopping criteria, we will limit the loop to 20 function calls at the most (generally the loop breaks after about 10).

```matlab
% starting design
x0 = [0.100,  0.198,  0.7,  1.800,  3.200,  4.550];

for i = 1:20

    % perform optimization using surrogate
    % TODO

    % compute the error in our surrogate prediction and decide whether or not to break
    % TODO

    % if not, add another row to sample data (add a row to PHI and a row to F)
    % be sure to use the actual function value at xopt
    % and not not the f from fmincon which is based on the surrogate
    % TODO

    % re-estimate the surrogate parameters (w)
    % TODO

end
```

We can use the optimal point of the previous solution as the starting point of the next iteration, but evaluating the surrogate is ridiculously cheap (just a vector-vector multiply) so it really doesn’t matter. We now have a pretty good solution to the minimum wave drag body of revolution and it should take around 30 function calls. By comparison, optimizing with the actual function takes around 100 function calls for this same starting point (depending on which algorithm you use), and the result is generally not that much better:

```matlab
J = @(x) wavedrag(x);
[xopt, fopt, exitflag, output] = fmincon(J, x0, [], [], [], [], lb, ub);
disp(output.funcCount);

% solution error
CD_sol = wavedrag(xopt);
fprintf('Optimization Error = %3.3g%%\n', (factual - CD_sol)/CD_sol*100);  % factual comes from the SBO
```

The error in using surrogate-based optimization (SBO) versus a full optimization in this problem is generally less than 1% (will vary randomly because of LHS), but requires only about 1/3 as many function calls.

We can also make some comparisons to the theoretical minimum because it is a known solution for this problem.  It is important to note that even optimizing with the actual function will not reach the theoretical minimum because of the discretization error (gets within 1.63\%).  The theoretical minimum wave drag for a fixed length and diameter is:

\begin{equation}
  {C_D}_{min} = \left(\frac{\pi d}{l}\right)^2
\end{equation}

We can print out our error as compared to the theoretical minimum:

```matlab
% theoretical error
d = 10; l = 100;
CD_min = (pi*d/l)^2;
fprintf('Theoretical Error = %3.3g%%\n', (factual - CD_min)/CD_min*100);
```

Finally, we can plot the difference in shapes between our SBO solution and the theoretical solution.  The difference should be extremely minimal.

```matlab
% --------------- Plot geometry ------------------------
% SBO solution
x = l*[0.0, 0.005, 0.01, 0.025, 0.1, 0.2, 0.35, 0.5, 0.65, 0.8, 0.9, 0.975, 0.99, 0.995, 1.0];
r = [0 xopt 5 fliplr(xopt) 0];  % this must come from your SBO solution

% theoretical optimal solution - Sears Haack
xSH = .01:.01:1;
rSH = d/2*sqrt(sqrt(1-xSH.^2)-xSH.^2.*log((1+sqrt(1-xSH.^2))./xSH));

xSH = l/2*[-fliplr(xSH) xSH] + l/2;
rSH = [fliplr(rSH) rSH];

figure; hold on;
h1 = plot(x,r,'b');
plot(x,-r,'b');
h2 = plot(xSH,rSH,'r--');
plot(xSH,-rSH,'r--');
axis equal;
legend([h1, h2], {'SBO','Sears-Haack'});
% ----------------------------------------------------------
```


Working with Gaussians
=======

### Preliminaries

- Goal
  - Review of processing of Gaussian distributions in linear systems
- Materials
  - Mandatory
    - These lecture notes
  - Optional
    - Bishop pp. 85-93
    - [MacKay - 2006 - The Humble Gaussian Distribution](./files/Mackay-2006-The-humble-Gaussian-distribution.pdf) (highly recommended!)

### Sums and Transformations of Gaussian Variables

- The Gaussian distribution
$$
\mathcal{N}(x|\mu,\Sigma) = |2 \pi \Sigma |^{-\frac{1}{2}} \,\mathrm{exp}\left\{-\frac{1}{2}(x-\mu)^T \Sigma^{-1} (x-\mu) \right\}
$$
for variable $x$ is completely specified by its mean $\mu$ and variance $\Sigma$.
  - $\Lambda = \Sigma^{-1}$ is called the **precision matrix**.

- A **linear transformation** $z=Ax+b$ of a Gaussian variable $\mathcal{N}(x|\mu,\Sigma)$ is Gaussian distributed as

$$
p(z) = \mathcal{N} \left(z \,|\, A\mu+b, A\Sigma A^T \right) \tag{SRG-4a}
$$

- The **sum of two independent Gaussian variables** is also Gaussian distributed. Specifically, if $x \sim \mathcal{N} \left(x|\mu_x, \Sigma_x \right)$ and $y \sim \mathcal{N} \left(y|\mu_y, \Sigma_y \right)$, then the PDF for $z=x+y$ is given by
$$\begin{align}
p(z) &= \mathcal{N}(x\,|\,\mu_x,\Sigma_x) \ast \mathcal{N}(y\,|\,\mu_y,\Sigma_y) \notag\\
  &= \mathcal{N} \left(z\,|\,\mu_x+\mu_y, \Sigma_x +\Sigma_y \right) \tag{SRG-8}
\end{align}$$
  - [Exercise]: Show that Eq.SRG-8 is really a special case of Eq.SRG-4a.
  - The sum of two Gaussian _distributions_ is NOT a Gaussian distribution. Why not?

### Example: Gaussian Signals in a Linear System

- [Q.]: Given independent variables
$x \sim \mathcal{N}(\mu_x,\sigma_y)$ and $y \sim \mathcal{N}(\mu_y,\sigma_y)$, what is the PDF for $z = A\cdot(x -y) + b$ ?

- [A.]: $z$ is also Gaussian with
$$
p_z(z) = \mathcal{N}(z|A(\mu_x-\mu_y)+b, \, A(\sigma_x \mathbf{+} \sigma_y)A^T)
$$

- Think about the role of the Gaussian distribution for stochastic linear systems in relation to what sinusoidals mean for deterministic linear system analysis.

### Example: Bayesian Estimation of a Constant

- [Question] Estimate a constant $\theta$ from one 'noisy' measurement $x$ about that constant. Assume the following model specification (the tilde $\sim$ means: 'is distributed as'):

$$\begin{align*}
x &= \theta + \epsilon \\
\theta &\sim \mathcal{N}(\mu_\theta,\sigma_\theta^2) \\
\epsilon &\sim \mathcal{N}(0,\sigma^2_{\epsilon})
\end{align*}$$

[Answer]

- **1. Model specification**
Note that you can rewrite these specifications in probabilistic notation as follows:
$$\begin{align}
    p(x|\theta) &=\mathcal{N}(x|\theta,\sigma^2_{\epsilon}) \tag{likelihood}\\
    p(\theta) &=\mathcal{N}(\theta|\mu_\theta,\sigma_\theta^2) \tag{prior}
\end{align}$$

- **2. Inference** for the posterior PDF $p(\theta|x)$
$$\begin{align*}
p(\theta|x)  &= \frac{p(x|\theta)p(\theta)}{p(x)} = \frac{p(x|\theta)p(\theta)} { \int p(x|\theta)p(\theta) \, \mathrm{d}\theta } \notag \\
    &= \frac{1}{C} \,\mathcal{N}(x|\theta,\sigma^2_{\epsilon})\, \mathcal{N}(\theta|\mu_\theta,\sigma_\theta^2) \notag \\
    &= \frac{1}{C_1} \mathrm{exp} \left\{ -\frac{(x-\theta)^2}{2\sigma^2_{\epsilon}} - \frac{(\theta-\mu_\theta)^2}{2\sigma_\theta^2} \right\} \notag \\
    &= \frac{1}{C_1} \mathrm{exp} \left\{ \theta^2\left( -\frac{1}{2\sigma^2_{\epsilon}} - \frac{1}{2\sigma_\theta^2} \right) + \theta \left( \frac{x}{\sigma^2_{\epsilon}} + \frac{\mu_\theta}{\sigma_\theta^2} \right) +  C_2 \right\} \notag \\
    &= \frac{1}{C_1} \mathrm{exp} \left\{ -\frac{\sigma_\theta^2 + \sigma^2_{\epsilon}}{2\sigma_\theta^2 \sigma^2_{\epsilon}} \left( \theta - \frac{x\sigma_\theta^2 + \mu_\theta \sigma^2_{\epsilon}}{\sigma_\theta^2 + \sigma^2_{\epsilon}} \right)^2 + C_3  \right\}
\end{align*}$$
which we recognize as a Gaussian distribution.

  - This computational 'trick' for multiplying two Gaussians is called **completing the square**. The procedure makes use of the equality $$ax^2+bx+c_1 = a\left(x+\frac{b}{2a}\right)^2+c_2$$

- Hence, it follows that the posterior for $\theta$ is

$$\begin{equation*}
    p(\theta|x) = \mathcal{N} (\theta |\, \mu_{\theta|x}, \sigma_{\theta|x}^2)
\end{equation*}$$

where

$$\begin{align*}
  \frac{1}{\sigma_{\theta|x}^2}  &= \frac{\sigma^2_{\epsilon} + \sigma_\theta^2}{\sigma^2_{\epsilon}\sigma_\theta^2} = \frac{1}{\sigma_\theta^2} + \frac{1}{\sigma^2_{\epsilon}}\\
  \mu_{\theta|x}   &= \sigma_{\theta|x}^2 \, \left( \frac{1}{\sigma^2_{\epsilon}}x + \frac{1}{\sigma_\theta^2} \mu_\theta \right)
\end{align*}$$

- So, multiplication of two Gaussian distributions yields another (unnormalized) Gaussian with
  - posterior precision equals **sum of prior precisions**
  - posterior precision-weighted mean equals **sum of prior precision-weighted means**
- (This is worth remembering)

### CODE EXAMPLE

Let's plot the exact product of two Gaussian PDFs as well as the normalized product according to the above derivation.

```julia
using PyPlot, Distributions
d1 = Normal(0, 1) # μ=0, σ^2=1
d2 = Normal(3, 2) # μ=3, σ^2=4

# Calculate the parameters of the product d1*d2
s2_prod = (d1.σ^-2 + d2.σ^-2)^-1
m_prod = s2_prod * ((d1.σ^-2)*d1.μ + (d2.σ^-2)*d2.μ)
d_prod = Normal(m_prod, sqrt(s2_prod)) # Note that we neglect the normalization constant.

# Plot stuff
x = range(-4, stop=8, length=100)
plot(x, pdf.(d1,x), "k")
plot(x, pdf.(d2,x), "b")
plot(x, pdf.(d1,x) .* pdf.(d2,x), "r-") # Plot the exact product
plot(x, pdf.(d_prod,x), "r:")          # Plot the normalized Gaussian product
legend([L"\mathcal{N}(0,1)",
        L"\mathcal{N}(3,4)",
        L"\mathcal{N}(0,1) \mathcal{N}(3,4)",
        L"Z^{-1} \mathcal{N}(0,1) \mathcal{N}(3,4)"]);
```

The solid and dotted red curves are identical up to a scaling factor $Z$.

### Multivariate Gaussian Multiplication

- In general, the multiplication of two multi-variate Gaussians yields an (unnormalized) Gaussian, see [SRG-6]:
$$\begin{equation*}
\mathcal{N}(x|\mu_a,\Sigma_a) \cdot \mathcal{N}(x|\mu_b,\Sigma_b) = \alpha \cdot \mathcal{N}(x|\mu_c,\Sigma_c)
\end{equation*}$$
where
$$\begin{align*}
\Sigma_c^{-1} &= \Sigma_a^{-1} + \Sigma_b^{-1} \\
\Sigma_c^{-1} \mu_c &= \Sigma_a^{-1}\mu_a + \Sigma_b^{-1}\mu_b
\end{align*}$$
and normalization constant $\alpha = \mathcal{N}(\mu_a|\, \mu_b, \Sigma_a + \Sigma_b)$.

- If we define the **precision** as $\Lambda \equiv \Sigma^{-1}$, then we see that **precisions add** and **precision-weighted means add** too.

- As we just saw, great application to Bayesian inference!

$$\begin{equation*}
\underbrace{\text{Gaussian}}_{\text{posterior}}
 \propto \underbrace{\text{Gaussian}}_{\text{likelihood}} \times \underbrace{\text{Gaussian}}_{\text{prior}}
\end{equation*}$$

### Conditioning and Marginalization of a Gaussian

- Let $z = \begin{bmatrix} x \\ y \end{bmatrix}$ be jointly normal distributed as

$$\begin{align*}
p(z) &= \mathcal{N}(z | \mu, \Sigma)
  =\mathcal{N} \left( \begin{bmatrix} x \\ y \end{bmatrix} \left| \begin{bmatrix} \mu_x \\ \mu_y \end{bmatrix},
  \begin{bmatrix} \Sigma_x & \Sigma_{xy} \\ \Sigma_{yx} & \Sigma_y \end{bmatrix} \right. \right)
\end{align*}$$

- Since covariance matrices are by definition symmetric, it follows that $\Sigma_x$ and $\Sigma_y$ are symmetric and $\Sigma_{xy} = \Sigma_{yx}^T$.

- Let's factorize $p(x,y)$ into $p(y|x)\, p(x)$ through conditioning and marginalization (proof in Bishop pp.87-89)

  - **Conditioning**

$$\begin{align*}
p(y|x) &= p(x,y)/p(x) \\
 &= \mathcal{N}\left(y|\mu_y + \Sigma_{yx}\Sigma_x^{-1}(x-\mu_x),\, \Sigma_y - \Sigma_{yx}\Sigma_x^{-1}\Sigma_{xy} \right)
\end{align*}$$

  - **Marginalization**

$$
p(x) = \int p(x,y)\,\mathrm{d}y = \mathcal{N}\left( x|\mu_x, \Sigma_x \right)
$$

- Useful for applications to Bayesian inference in jointly Gaussian systems.

#### CODE EXAMPLE

Plot of the joint, marginal, and conditional distributions.

```julia
using PyPlot, Distributions

μ = [1.0; 2.0]
Σ = [0.3 0.7;
     0.7 2.0]
joint = MvNormal(μ,Σ)
marginal_x = Normal(μ[1], sqrt(Σ[1,1]))

#Plot p(x,y)
subplot(221)
x_range = y_range = range(-2,stop=5,length=1000)
joint_pdf = [ pdf(joint, [x_range[i];y_range[j]]) for  j=1:length(y_range), i=1:length(x_range)]
imshow(joint_pdf, origin="lower", extent=[x_range[1], x_range[end], y_range[1], y_range[end]])
grid(); xlabel("x"); ylabel("y"); title("p(x,y)"); tight_layout()

# Plot p(x)
subplot(222)
plot(range(-2,stop=5,length=1000), pdf.(marginal_x, range(-2,stop=5,length=1000)))
grid(); xlabel("x"); ylabel("p(x)"); title("Marginal distribution p(x)"); tight_layout()

# Plot p(y|x)
x = 0.1
conditional_y_m = μ[2]+Σ[2,1]*inv(Σ[1,1])*(x-μ[1])
conditional_y_s2 = Σ[2,2] - Σ[2,1]*inv(Σ[1,1])*Σ[1,2]
conditional_y = Normal(conditional_y_m, sqrt.(conditional_y_s2))
subplot(223)
plot(range(-2,stop=5,length=1000), pdf.(conditional_y, range(-2,stop=5,length=1000)))
grid(); xlabel("y"); ylabel("p(y|x)"); title("Conditional distribution p(y|x)"); tight_layout()
```

As is clear from the plots, the conditional distribution is a renormalized slice from the joint distribution.

### Example: Conditioning of Gaussian

- Consider (again) the system

$$\begin{align*}
x &= \theta + \epsilon \\
\theta &\sim \mathcal{N}(\theta|\mu_\theta,\sigma_\theta^2) \\
\epsilon &\sim \mathcal{N}(\epsilon|0,\sigma^2_{\epsilon})
\end{align*}$$

- This system is equivalent to (derive this!)

$$
p(x,\theta|\,\mu,\sigma) = \mathcal{N} \left( \begin{bmatrix} x\\
  \theta  \end{bmatrix}
  \left| \begin{bmatrix} \mu_\theta\\
  \mu_\theta\end{bmatrix},
         \begin{bmatrix} \sigma_\theta^2+\sigma_{\epsilon}^2  & \sigma_\theta^2\\
         \sigma_\theta^2 &\sigma_\theta^2
  \end{bmatrix}
  \right. \right)
$$

- Direct substitution of the rule for Gaussian conditioning leads to (derive this yourself!)
$$\begin{align*}
p(\theta|x) &= \mathcal{N} \left( \theta\,|\,\mu_{\theta|x}, \sigma_{\theta|x}^2 \right)\,, \quad
\text{with} \\
K &= \frac{\sigma_\theta^2}{\sigma_\theta^2+\sigma_{\epsilon}^2} \qquad \text{($K$ is called: Kalman gain)}\\
\mu_{\theta|x} &= \mu_\theta + K \cdot (x-\mu_\theta)\\
\sigma_{\theta|x}^2 &= \left( 1-K \right) \sigma_\theta^2
\end{align*}$$

$\longrightarrow$ Moral: For jointly Gaussian systems, we can do inference simply in one step by using the formulas for conditioning and marginalization.

### Application: Recursive Bayesian Estimation

Now consider the signal $x_t=\theta+\epsilon_t$, where $D_t= \left\{x_1,\ldots,x_t\right\}$ is observed _sequentially_ (over time).

[Question]
- Derive a recursive algorithm for $p(\theta|D_t)$, i.e., an update rule for (posterior) $p(\theta|D_t)$ based on (prior) $p(\theta|D_{t-1})$ and (new observation) $x_t$.

[Answer]

- Let's define the estimate after $t$ observations (i.e., our solution) as $p(\theta|D_t) = \mathcal{N}(\theta\,|\,\mu_t,\sigma_t^2)$.

- **Model specification**. We define the joint distribution for $\theta$ and $x_t$, given background $D_{t-1}$, by
$$\begin{align*} p(x_t,\theta \,|\, D_{t-1}) &= p(x_t|\theta) \, p(\theta|D_{t-1}) \\
  &= \underbrace{\mathcal{N}(x_t\,|\, \theta,\sigma^2_{\epsilon})}_{\text{likelihood}} \, \underbrace{\mathcal{N}(\theta\,|\,\mu_{t-1},\sigma_{t-1}^2)}_{\text{prior}}
\end{align*}$$

- **Inference**. Use Bayes rule,
$$\begin{align*}
p(\theta|D_t) &\propto p(x_t|\theta) \, p(\theta|D_{t-1}) \\
 &= \mathcal{N}(x_t|\theta,\sigma^2_{\epsilon}) \, \mathcal{N}(\theta\,|\,\mu_{t-1},\sigma_{t-1}^2) \\
 &= \mathcal{N}(\theta|x_t,\sigma^2_{\epsilon}) \, \mathcal{N}(\theta\,|\,\mu_{t-1},\sigma_{t-1}^2) \\
 &= \mathcal{N}(\theta|\mu_t,\sigma_t^2)
\end{align*}$$
with
$$\begin{align*}
K_t &= \frac{\sigma_{t-1}^2}{\sigma_{t-1}^2+\sigma_{\epsilon}^2} \qquad \text{(Kalman gain)}\\
\mu_t &= \mu_{t-1} + K_t \cdot (x_t-\mu_{t-1})\\
\sigma_t^2 &= \left( 1-K_t \right) \sigma_{t-1}^2
\end{align*}$$
(as before, we used the formulas for conditioning in a multivariate Gaussian system).

- This linear _sequential_ estimator of mean and variance in Gaussian observations is called a **Kalman Filter**.

- Note that the uncertainty about $\theta$ decreases over time (since $0<(1-K_t)<1$). This makes sense: since we assume that the statistics of the system do not change (stationarity), each new sample provides new information.

- Recursive Bayesian estimation is the basis for **adaptive signal processing** algorithms such as Least Mean Squares (LMS) and Recursive Least Squares (RLS).

#### CODE EXAMPLE

Let's implement the Kalman filter described above. We'll use it to recursively estimate the value of $\theta$ based on noisy observations.

```julia
using PyPlot

N = 50                         # Number of observations
θ = 2.0                        # True value of the variable we want to estimate
σ_ϵ2 = 0.25                    # Observation noise variance
x = sqrt(σ_ϵ2) * randn(N) .+ θ # Generate N noisy observations of θ

t = 0
μ = fill!(Vector{Float64}(undef,N), NaN)    # Means of p(θ|D) over time
σ_μ2 = fill!(Vector{Float64}(undef,N), NaN) # Variances of p(θ|D) over time

function performKalmanStep()
    # Perform a Kalman filter step, update t, μ, σ_μ2
    global t += 1
    if t>1 # Use posterior from prev. step as prior
        K = σ_μ2[t-1] / (σ_ϵ2 + σ_μ2[t-1]) # Kalman gain
        μ[t] = μ[t-1] + K*(x[t] - μ[t-1])  # Update mean using (1)
        σ_μ2[t] = σ_μ2[t-1] * (1.0-K)      # Update variance using (2)
    elseif t==1 # Use prior
        # Prior p(θ) = N(0,1000)
        K = 1000.0 / (σ_ϵ2 + 1000.0) # Kalman gain
        μ[t] = 0 + K*(x[t] - 0)      # Update mean using (1)
        σ_μ2[t] = 1000 * (1.0-K)     # Update variance using (2)
    end
end

while t<N
    performKalmanStep()
end

# Plot the 'true' value of θ, noisy observations x, and the recursively updated posterior p(θ|D)
t = collect(1:N)
plot(t, θ*ones(N), "k--")
plot(t, x, "rx")
plot(t, μ, "b-")
fill_between(t, μ-sqrt.(σ_μ2), μ+sqrt.(σ_μ2), color="b", alpha=0.3)
legend([L"\theta", L"x[t]", L"\mu[t]"])
xlim((1, N)); xlabel(L"t"); grid()
```

The shaded area represents 2 standard deviations of posterior $p(\theta|D)$. The variance of the posterior is guaranteed to decrease monotonically for the standard Kalman filter.

### Product of Normally Distributed Variables

- (We've seen that) the sum of two Gausssian distributed variables is also Gaussian distributed.

- Has the _product_ of two Gaussian distributed variables also a Gaussian distribution?

- **No**! In general this is a difficult computation. As an example, let's compute $p(z)$ for $Z=XY$ for the special case that $X\sim \mathcal{N}(0,1)$ and $Y\sim \mathcal{N}(0,1)$.
$$\begin{align*}
p(z) &= \int_{X,Y} p(z|x,y)\,p(x,y)\,\mathrm{d}x\mathrm{d}y \\
  &= \frac{1}{2 \pi}\int  \delta(z-xy) \, e^{-(x^2+y^2)/2} \, \mathrm{d}x\mathrm{d}y \\
  &=  \frac{1}{\pi} \int_0^\infty \frac{1}{x} e^{-(x^2+z^2/x^2)/2} \, \mathrm{d}x \\
  &= \frac{1}{\pi} \mathrm{K}_0( \lvert z\rvert )\,.
\end{align*}$$
where  $\mathrm{K}_n(z)$ is a [modified Bessel function of the second kind](http://mathworld.wolfram.com/ModifiedBesselFunctionoftheSecondKind.html).

#### CODE EXAMPLE

We plot $p(Z)$ to give an idea of what this distribution looks like.

```julia
using PyPlot, Distributions, SpecialFunctions
X = Normal(0,1)
pdf_product_std_normals(z::Vector) = (besselk.(0, abs.(z))./π)
range1 = collect(range(-4,stop=4,length=100))
plot(range1, pdf.(X, range1))
plot(range1, pdf_product_std_normals(range1))
legend([L"p(X)=p(Y)=\mathcal{N}(0,1)", L"p(Z)"]); grid()
```

### Review Gaussians

The success of Gaussian distributions in probabilistic modeling is large due to the following properties:

- A linear transformation of a Gaussian distributed variable is also Gaussian distributed

- The convolution of two Gaussian functions is another Gaussian function (use in sum of 2 variables)

- The product of two Gaussian functions is another Gaussian function (use in Bayes rule).

- Conditioning and marginalization of multivariate Gaussian distributions produce Gaussians again (use in working with observations and when doing Bayesian predictions)

- The Gaussian PDF has higher entropy than any other with the same variance. (Not discussed in this course).

- Any smooth function with single rounded maximum, if raised to higher and higher powers, goes into a Gaussian function. (Not discussed).

### Some Useful Matrix Calculus

Aside from working with Gaussians, it will be helpful for the next lessons to be familiar with some matrix calculus. We shortly recapitulate used formulas here.

- We define the **gradient** of a scalar function $f(A)$ w.r.t. an $n \times k$ matrix $A$ as
$$
\nabla_A f \triangleq
    \begin{bmatrix}
\frac{\partial{f}}{\partial a_{11}} & \frac{\partial{f}}{\partial a_{12}} & \cdots & \frac{\partial{f}}{\partial a_{1k}}\\
\frac{\partial{f}}{\partial a_{21}} & \frac{\partial{f}}{\partial a_{22}} & \cdots & \frac{\partial{f}}{\partial a_{2k}}\\
\vdots & \vdots & \cdots & \vdots\\
\frac{\partial{f}}{\partial a_{n1}} & \frac{\partial{f}}{\partial a_{n2}} & \cdots & \frac{\partial{f}}{\partial a_{nk}}
    \end{bmatrix}
$$

- The following formulas are useful (see Bishop App.-C)
$$\begin{align*}
|A^{-1}|&=|A|^{-1} \tag{B-C.4} \\
\nabla_A \log |A| &= (A^{T})^{-1} = (A^{-1})^T \tag{B-C.28} \\
\mathrm{Tr}[ABC]&= \mathrm{Tr}[CAB] = \mathrm{Tr}[BCA] \tag{B-C.9} \\
\nabla_A \mathrm{Tr}[AB] &=\nabla_A \mathrm{Tr}[BA]= B^T \tag{B-C.25} \\
\nabla_A \mathrm{Tr}[ABA^T] &= A(B+B^T)  \tag{B-C.27}\\
 \nabla_x x^TAx &= (A+A^T)x \tag{from B-C.27}\\
\nabla_X a^TXb &= \nabla_X \mathrm{Tr}[ba^TX] = ab^T \notag
\end{align*}$$

### What's Next?

- We discussed how Bayesian probability theory provides an integrated framework for making predictions based on observed data.

- The process involves model specification (your main task!), inference and actual model-based prediction.

- The latter two tasks are only difficult because of computational issues.

- Maximum likelihood was introduced as a computationally simpler approximation to the Bayesian approach.

- In particular under some linear Gaussian assumptions, a few interesting models can be designed.

- The rest of this course (part-1) concerns introduction to these Linear Gaussian models.

```julia
open("../../styles/aipstyle.html") do f
    display("text/html", read(f, String))
end
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>

```julia

```


# 01 - Polynomial Linear Regression (One Variable)
Prepared by Jan Christian Blaise Cruz

DLSU Machine Learning Group

```python
# First, some necessary imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import load_boston

%matplotlib inline
np.random.seed(42)
```

## The Data
We're mainly working with the Boston Housing dataset (Harrison & Rubinfield, 1993) for this problem. Scikit-learn has a convience function that loads the dataset so we'll use that.

```python
d = load_boston()
print(d.DESCR) # Print the dataset description
```

    Boston House Prices dataset
    ===========================

    Notes
    ------
    Data Set Characteristics:

        :Number of Instances: 506

        :Number of Attributes: 13 numeric/categorical predictive

        :Median Value (attribute 14) is usually the target

        :Attribute Information (in order):
            - CRIM     per capita crime rate by town
            - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.
            - INDUS    proportion of non-retail business acres per town
            - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)
            - NOX      nitric oxides concentration (parts per 10 million)
            - RM       average number of rooms per dwelling
            - AGE      proportion of owner-occupied units built prior to 1940
            - DIS      weighted distances to five Boston employment centres
            - RAD      index of accessibility to radial highways
            - TAX      full-value property-tax rate per $10,000
            - PTRATIO  pupil-teacher ratio by town
            - B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town
            - LSTAT    % lower status of the population
            - MEDV     Median value of owner-occupied homes in $1000's

        :Missing Attribute Values: None

        :Creator: Harrison, D. and Rubinfeld, D.L.

    This is a copy of UCI ML housing dataset.
    http://archive.ics.uci.edu/ml/datasets/Housing

    This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.

    The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic
    prices and the demand for clean air', J. Environ. Economics & Management,
    vol.5, 81-102, 1978.   Used in Belsley, Kuh & Welsch, 'Regression diagnostics
    ...', Wiley, 1980.   N.B. Various transformations are used in the table on
    pages 244-261 of the latter.

    The Boston house-price data has been used in many machine learning papers that address regression
    problems.

    **References**

       - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.
       - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.
       - many more! (see http://archive.ics.uci.edu/ml/datasets/Housing)

We can extract the data like so.

```python
X, y = d.data, d.target
```

And get the shapes.

Notice that we have a matrix of examples $X \in \mathbb{R}^{m \times d}$ and a vector of targets $y \in \mathbb{R}^m$ where $m$ is the number of training examples and $d$ is the "dimensionality" or the number of features.

```python
print(X.shape, y.shape)
```

    (506, 13) (506,)

For now, we're only interested in the property tax feature of the dataset, which is in index 8. We'll take that out. Since the resulting numpy array is a 1d vector, we'll add a new axis (new dimension).

```python
x = X[:, 9]
print(x.shape, y.shape)

# Add new dimension
x = x[..., np.newaxis]
print(x.shape)
```

    (506,) (506,)
    (506, 1)

We'll write a convenience function to plot our data.

```python
def plot_dist(X, y):
    data = np.append(X[:, 0][..., np.newaxis], y[..., np.newaxis], axis=1)
    data = data[data[:, 0].argsort()]
    plt.scatter(data[:, 0], data[:, 1], c='r')
```

And let's check our data.

```python
plot_dist(x, y)
```

## Linear Regression
Now we'll use a linear regression model to solve our task.

Recall our hypothesis for linear regression:

\begin{align}
h_{\theta}(x) = \theta_0x_0 + \theta_1x_1 + ... + \theta_nx_n
\end{align}

Which we can simply write as a summation:

\begin{align}
h_{\theta}(x) = \sum_{i=0}^{m}{\theta_{i}x_{i}}
\end{align}

Which can be simplified in vector notation:

\begin{align}
h_{\theta}(x) = X \theta
\end{align}

Given $X\in\mathbb{R}^{m\times d}$ and our theta parameter vector $\theta\in\mathbb{R}^{d\times 1}$, our ```model``` function will output a 2d array of predictions (specifically $\hat{y}\in\mathbb{R}^{m\times 1}$). Since our actual target vector is a 1d array ($y\in\mathbb{R}^m$), we'll index the first column on our output to reduce the dimensions.

```python
def model(X, theta):
    return X.dot(theta)[:, 0]
```

We'll have to first add a bias term to our data manually, represented by an array of ones.

```python
bias = np.ones([x.shape[0], 1], dtype=np.int32)
print(bias.shape, x.shape)
```

    (506, 1) (506, 1)

We append this to the data.

```python
x = np.append(x, bias, axis=1)
```

To get the shape:

```python
print(x.shape)
```

    (506, 2)

Our feature matrix now looks like this.

```python
x
```

    array([[296.,   1.],
           [242.,   1.],
           [242.,   1.],
           ...,
           [273.,   1.],
           [273.,   1.],
           [273.,   1.]])

We'll randomly initialize our theta vector $\theta\in\mathbb{R}^{d\times 1}$.

```python
features = x.shape[1]
print("Number of features: {}".format(features))

theta = np.random.rand(features, 1)
print(theta)
```

    Number of features: 2
    [[0.37454012]
     [0.95071431]]

And get our initial predictions.

```python
yhat = model(x, theta)
```

Verify the shapes.

```python
print(yhat.shape, y.shape)
```

    (506,) (506,)

Next, we'll need a measure of error.

Recall our Mean Squared Error (MSE) loss function:

\begin{align}
J(\theta) = \frac{1}{2m}\sum_{i=0}^{m}(h_\theta(x^{(i)}) - y^{(i)})^2
\end{align}

Which in vector notation is simply:

\begin{align}
J(\theta) = \frac{1}{2m}(X\theta - y)^T(X\theta - y)
\end{align}

```python
def loss_function(yhat, y):
    m = y.size
    J = 1 / (2 * m) * np.sum(np.square(yhat - y))
    return J
```

Let's test it.

```python
loss_function(yhat, y)
```

    10924.413868645808

Now we code up gradient descent. The update rule for gradient descent is:

\begin{align}
\theta := \theta - \alpha \frac{\partial}{\partial \theta}J(\theta)
\end{align}

Computing for the partial derivative yields this update rule:

\begin{align}
\theta := \theta - \alpha \frac{1}{m}\sum^{m}_{i = 0}(h_{\theta}(x^{(i)})-y^{(i)})\cdot x^{(i)}
\end{align}

Or in vector notation:

\begin{align}
\theta := \theta - \alpha \frac{1}{m} X^{T}(\hat{y}-y)
\end{align}

Do remember that the update rule has to be applied for all theta parameters consecutively. Since we have a bias feature, we have two theta parameters.

First, we code up the derivative of our loss function for convenience.

```python
def loss_derivative(X, y, theta):
    yhat = model(X, theta)
    diff = (yhat - y)[..., np.newaxis]
    return X.T.dot(diff)
```

We can test it.

```python
loss_derivative(x, y, theta)
```

    array([[32865637.63732845],
           [   66447.46470911]])

Then we code up gradient descent itself.

```python
def gradient_descent(X, y, theta, alpha=0.01):
    m = y.size
    yhat = model(X, theta)
    loss_p = loss_derivative(X, y, theta)
    new_theta = theta - alpha * (1 / m) * loss_p
    return new_theta
```

## Training
Next up, we train our model using gradient descent and observe how the model changes and learns.

Let's train for 10K epochs with a learning rate of 1e-5

```python
epochs = 10000
alpha = 0.00001
theta = np.random.rand(features, 1)

for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 1000 == 0 or e == 1:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))
```

    Loss at Epoch      1: 46502.4447
    Loss at Epoch   1000: 108.9010
    Loss at Epoch   2000: 108.6805
    Loss at Epoch   3000: 108.4606
    Loss at Epoch   4000: 108.2413
    Loss at Epoch   5000: 108.0227
    Loss at Epoch   6000: 107.8048
    Loss at Epoch   7000: 107.5874
    Loss at Epoch   8000: 107.3707
    Loss at Epoch   9000: 107.1546
    Loss at Epoch  10000: 106.9392

Let's write a convenience function for plotting the line.

```python
def plot_line(X, theta):
    data = np.append(X[:, 0][..., np.newaxis], yhat[..., np.newaxis], axis=1)
    data = data[data[:, 0].argsort()]
    plt.plot(data[:, 0], data[:, 1])
```

And plot it in conjunction with our data.

```python
plot_line(x, theta)
plot_dist(x, y)
```

That's not the line we're looking for.

Let's train for real this time, for 500K epochs with the same settings.

```python
theta = np.random.rand(features, 1)
alpha = 0.00001
epochs = 500000
```

And use gradient descent.

```python
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 50000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

plot_line(x, theta)
plot_dist(x, y)
```

That's a better line.

Let's train more.

```python
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 50000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

plot_line(x, theta)
plot_dist(x, y)
```

Getting better! One last 500K.

```python
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 50000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

plot_line(x, theta)
plot_dist(x, y)
```

And now we have a good fitting model for our data.

## Polynomial Linear Regression
In this section, we'll use polynomials of higher degree as our hypothesis function. Since the Boston dataset doesn't have an easily-visible feature that has a higher-degree trend, let's generate our own data.

```python
np.random.seed(422)
x = np.arange(-10, 10) + np.random.normal(-1, 1)
y = x ** 2 + x + np.random.normal(-8, 8, 20)

x = x[..., np.newaxis]
print(x.shape, y.shape)
```

    (20, 1) (20,)

And plot it.

```python
ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
```

This time, it makes more sense to use a second-degree hypothesis, specifically of the form:

\begin{align}
h_{\theta}(x) = \theta_01 + \theta_1x + \theta_2x^2
\end{align}

In machine learning, it is common to denote $\theta_0 = 1$. Do note that we'll still code this up using vector notation, so we'll have to manually add the square and the bias term to our features.

```python
x2 = np.square(x)
bias = np.ones([x.shape[0], 1], dtype=np.int32)
x = np.append(x, x2, axis=1)
x = np.append(x, bias, axis=1)
```

Our feature matrix now looks like this.

```python
x
```

    array([[-1.16896251e+01,  1.36647335e+02,  1.00000000e+00],
           [-1.06896251e+01,  1.14268085e+02,  1.00000000e+00],
           [-9.68962510e+00,  9.38888345e+01,  1.00000000e+00],
           [-8.68962510e+00,  7.55095844e+01,  1.00000000e+00],
           [-7.68962510e+00,  5.91303342e+01,  1.00000000e+00],
           [-6.68962510e+00,  4.47510840e+01,  1.00000000e+00],
           [-5.68962510e+00,  3.23718338e+01,  1.00000000e+00],
           [-4.68962510e+00,  2.19925836e+01,  1.00000000e+00],
           [-3.68962510e+00,  1.36133334e+01,  1.00000000e+00],
           [-2.68962510e+00,  7.23408317e+00,  1.00000000e+00],
           [-1.68962510e+00,  2.85483297e+00,  1.00000000e+00],
           [-6.89625098e-01,  4.75582776e-01,  1.00000000e+00],
           [ 3.10374902e-01,  9.63325795e-02,  1.00000000e+00],
           [ 1.31037490e+00,  1.71708238e+00,  1.00000000e+00],
           [ 2.31037490e+00,  5.33783219e+00,  1.00000000e+00],
           [ 3.31037490e+00,  1.09585820e+01,  1.00000000e+00],
           [ 4.31037490e+00,  1.85793318e+01,  1.00000000e+00],
           [ 5.31037490e+00,  2.82000816e+01,  1.00000000e+00],
           [ 6.31037490e+00,  3.98208314e+01,  1.00000000e+00],
           [ 7.31037490e+00,  5.34415812e+01,  1.00000000e+00]])

With a shape that we can anticipate.

```python
x.shape
```

    (20, 3)

Let's try out a setup.

```python
features = x.shape[1]
theta = np.random.rand(features, 1)
alpha = 0.00001
epochs = 10000
```

Plotting the initial guess doesn't give a pretty accurate line.

```python
yhat = model(x, theta)

ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
ax.plot(x[:, 0], yhat)
```

After 10K epochs:

```python
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 1000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
ax.plot(x[:, 0], yhat)
```

That's a better line!

Let's see what we get if we train it more.

```python
epochs = 500000
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 50000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
ax.plot(x[:, 0], yhat)
```

It now captures the trough at the $[-3.0, 1.0]$ range.

Train some more.

```python
epochs = 500000
for e in range(1, epochs + 1):
    yhat = model(x, theta)
    loss = loss_function(yhat, y)
    theta = gradient_descent(x, y, theta, alpha=alpha)

    if e % 50000 == 0:
        print("Loss at Epoch {:6}: {:.4f}".format(e, loss))

ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
ax.plot(x[:, 0], yhat)
```

Looking at the loss, we can see that the model isn't improving anymore. This is a signal that the model has achieved "convergence."

We can (in this case) verify that we have achieved the solution by using the analytical (closed form) solution:

\begin{align}
\theta = (X^Ty)(X^TX)^{-1}
\end{align}

```python
theta_analytical = x.T.dot(y).dot(np.linalg.inv(x.T.dot(x)))[..., np.newaxis]
yhat = model(x, theta)
loss = loss_function(yhat, y)
print("Analytical loss: {:.4f}".format(loss))

ax = plt.axes()
ax.set_xlim(np.min(x[:, 0]), np.max(x[:, 0]))
ax.set_ylim(np.min(y), np.max(y))
ax.plot(x[:, 0], y)
ax.plot(x[:, 0], yhat)
```

Checking at the two solutions, we can see that the two methods arrive at similar theta parameters.

```python
print(theta)
print(theta_analytical)
```

    [[ 0.60209095]
     [ 0.95523157]
     [-8.26084417]]
    [[ 0.60457283]
     [ 0.95626765]
     [-8.32658598]]

```python

```

notebooks/01 Polynomial Linear Regression.ipynb