# L08: Interpolation - extrapolation
# Setup notebook

```python
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import pandas as pd
import scipy as sp
from scipy import interpolate
from mpl_toolkits.mplot3d import Axes3D
import sympy
```

# Linear Interpolation

$\frac{y-y_0}{x-x_0} = \frac{y_1-y_0}{x_1-x_0}$

$y = y_0 + (y_1 - y_0)\frac{x-x_0}{x_1-x_0}$

$y=\frac{x_1-x}{x_1-x_0}y_0+\frac{x-x_0}{x_1-x_0}y_1$

```python
x = np.linspace(1,100,10)
y = np.log(x)
df = pd.DataFrame(np.c_[x,y],columns = ['x','y'])
df.head()
```

      x
      y

      0
      1.0
      0.000000

      1
      12.0
      2.484907

      2
      23.0
      3.135494

      3
      34.0
      3.526361

      4
      45.0
      3.806662

```python
df.plot.scatter('x','y')
plt.show()
```

## Based on the given data, use linear interpolation to find x at y = 17.68

## Method 1: Manual $y = y_0 + (y_1 - y_0)\frac{x-x_0}{x_1-x_0}$

### Step 1: find the iterval that cover the point of interest

Method 1A: find the use `.argmin`

```python
#find the index of the minimum distance value
#then find the index that is on the left side
val = 22
idx = np.abs(x-val).argmin()
if x[idx] > val:
    idx = idx - 1
idx
```

    1

Method 1b: use `np.searchsorted`

```python
val = 22
#np.searchsorted give the location that
#if value is inserted before this index
#the value is perserved.
#Array has to be sorted already
idx2 = np.searchsorted(x, val) - 1
idx2
```

    1

```python
x
```

    array([   1.,   12.,   23.,   34.,   45.,   56.,   67.,   78.,   89.,  100.])

```python
y
```

    array([ 0.        ,  2.48490665,  3.13549422,  3.52636052,  3.80666249,
            4.02535169,  4.20469262,  4.35670883,  4.48863637,  4.60517019])

```python
y_ans = y[idx] + (y[idx+1] - y[idx])*(val - x[idx]) / (x[idx+1] - x[idx])
y_ans
```

    3.0763498917344996

```python
mpl.rc('font', **{'size': 15})
ax = df.plot.scatter('x','y')
ax.plot(val,y_ans, 'ro', label = 'point of interest')
ax.legend(loc=4)
plt.show()
```

# Method 2: Automatic: `numpy.interp`
Format: `np.interp(val,x,y)` val can be numpy array

```python
np.interp(val,x,y)
```

    3.0763498917345

```python
ans = np.interp(list(range(100)),x,y)
```

```python
mpl.rc('font', **{'size': 15})
ax = df.plot.scatter('x','y')
ax.plot(list(range(100)),ans, 'ro', label = 'point of interest',markersize = 1)
ax.legend(loc=4)
plt.show()
```

# Linear Extrapolation

$y = y_0 + (y_1 - y_0)\frac{x-x_0}{x_1-x_0}$

```python
x = np.linspace(0.1,2,20)
y = np.log(x)
df = pd.DataFrame(np.c_[x,y],columns = ['x','y'])
df.plot.scatter('x','y')
plt.show()
```

```python
val = 22
idx = np.searchsorted(x, val) - 1
y_ans = y[idx] + (y[idx-1] - y[idx])*(val - x[idx]) / (x[idx-1] - x[idx])
#Notice: We know that we are going to extrapolate to the right-hand side
#Therefore, we use idx-1 and idx (change idx+1 of the previous case to idx-1)
```

```python
ax = df.plot.scatter('x','y',figsize = (12,8))
ax.plot(val,y_ans, 'ro', label = 'linear extrapolation')
ax.plot([x[idx],val],[y[idx],y_ans],'g-')
ax.plot(np.linspace(0,22,40), np.log(np.linspace(0.1,22,40)),label='actual log curve')
ax.grid()
ax.legend(loc=4)
plt.show()
```

```python
x
```

    array([ 0.1,  0.2,  0.3,  0.4,  0.5,  0.6,  0.7,  0.8,  0.9,  1. ,  1.1,
            1.2,  1.3,  1.4,  1.5,  1.6,  1.7,  1.8,  1.9,  2. ])

```python
y
```

    array([ -2.30258509e+00,  -1.60943791e+00,  -1.20397280e+00,
            -9.16290732e-01,  -6.93147181e-01,  -5.10825624e-01,
            -3.56674944e-01,  -2.23143551e-01,  -1.05360516e-01,
            -1.11022302e-16,   9.53101798e-02,   1.82321557e-01,
             2.62364264e-01,   3.36472237e-01,   4.05465108e-01,
             4.70003629e-01,   5.30628251e-01,   5.87786665e-01,
             6.41853886e-01,   6.93147181e-01])

```python
Lin_inter = interpolate.interp1d(x,y,kind='linear',fill_value='extrapolate')
```

```python
y_sp = Lin_inter(val)
ax = df.plot.scatter('x','y',figsize = (12,8))
ax.plot(val,y_sp, 'ro', label = 'SciPy\nlinear extrapolation')
ax.plot([x[idx],val],[y[idx],y_sp],'g-')
ax.plot(np.linspace(0,22,40), np.log(np.linspace(0.1,22,40)),label='actual log curve')
ax.grid()
ax.legend(loc=4)
plt.show()
```

# Linear Interpolation for 2-D Data

### Step 0: Generate the data and store in a file

```python
T = np.linspace(50,100,7) #T is for temperature in F
P = np.linspace(100,5000, 11) #P is for pressure in psia
TT, PP = np.meshgrid(T,P)
```

```python
T
```

    array([  50.        ,   58.33333333,   66.66666667,   75.        ,
             83.33333333,   91.66666667,  100.        ])

```python
P
```

    array([  100.,   590.,  1080.,  1570.,  2060.,  2550.,  3040.,  3530.,
            4020.,  4510.,  5000.])

```python
TT
#TT changes as column changes
```

    array([[  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ],
           [  50.        ,   58.33333333,   66.66666667,   75.        ,
              83.33333333,   91.66666667,  100.        ]])

```python
PP
#PP changes as row changes
```

    array([[  100.,   100.,   100.,   100.,   100.,   100.,   100.],
           [  590.,   590.,   590.,   590.,   590.,   590.,   590.],
           [ 1080.,  1080.,  1080.,  1080.,  1080.,  1080.,  1080.],
           [ 1570.,  1570.,  1570.,  1570.,  1570.,  1570.,  1570.],
           [ 2060.,  2060.,  2060.,  2060.,  2060.,  2060.,  2060.],
           [ 2550.,  2550.,  2550.,  2550.,  2550.,  2550.,  2550.],
           [ 3040.,  3040.,  3040.,  3040.,  3040.,  3040.,  3040.],
           [ 3530.,  3530.,  3530.,  3530.,  3530.,  3530.,  3530.],
           [ 4020.,  4020.,  4020.,  4020.,  4020.,  4020.,  4020.],
           [ 4510.,  4510.,  4510.,  4510.,  4510.,  4510.,  4510.],
           [ 5000.,  5000.,  5000.,  5000.,  5000.,  5000.,  5000.]])

```python
Mu = ((4e3*np.exp(-1e-3*TT**2)) -PP*5e-2+300)*0.5
# Mu = (1e3*np.exp(-0.00007 * ((TT-32)/1.8 + 273.15)**2)-(5e-4*PP))*20+100
```

```python
Mu
```

    array([[ 311.66999725,  214.06139014,  170.98725691,  154.71312627,
             149.42795145,  147.94848712,  147.59079986],
           [ 299.41999725,  201.81139014,  158.73725691,  142.46312627,
             137.17795145,  135.69848712,  135.34079986],
           [ 287.16999725,  189.56139014,  146.48725691,  130.21312627,
             124.92795145,  123.44848712,  123.09079986],
           [ 274.91999725,  177.31139014,  134.23725691,  117.96312627,
             112.67795145,  111.19848712,  110.84079986],
           [ 262.66999725,  165.06139014,  121.98725691,  105.71312627,
             100.42795145,   98.94848712,   98.59079986],
           [ 250.41999725,  152.81139014,  109.73725691,   93.46312627,
              88.17795145,   86.69848712,   86.34079986],
           [ 238.16999725,  140.56139014,   97.48725691,   81.21312627,
              75.92795145,   74.44848712,   74.09079986],
           [ 225.91999725,  128.31139014,   85.23725691,   68.96312627,
              63.67795145,   62.19848712,   61.84079986],
           [ 213.66999725,  116.06139014,   72.98725691,   56.71312627,
              51.42795145,   49.94848712,   49.59079986],
           [ 201.41999725,  103.81139014,   60.73725691,   44.46312627,
              39.17795145,   37.69848712,   37.34079986],
           [ 189.16999725,   91.56139014,   48.48725691,   32.21312627,
              26.92795145,   25.44848712,   25.09079986]])

```python
plt.figure()
plt.pcolor(TT,PP,Mu,cmap='jet')
plt.colorbar()
plt.show()
```

```python
fig = plt.figure(figsize=(6,4))
ax = Axes3D(fig)
ax.plot_surface(TT,PP,Mu,cmap='jet')
ax.set_xlabel('\nT')
ax.set_ylabel('\nP')
ax.set_zlabel('\nF')
# ax.view_init(30,30)
plt.show()
```

```python
fig = plt.figure(figsize=(6,4))
ax = Axes3D(fig)
ax.plot_surface(TT,PP,Mu,cmap='jet')
ax.set_xlabel('\nT')
ax.set_ylabel('\nP')
ax.set_zlabel('\nF')
ax.view_init(30,30)
plt.show()
```

```python
TT.shape
```

    (11, 7)

```python
T.shape
```

    (7,)

```python
P.shape
```

    (11,)

```python
Mu.shape
```

    (11, 7)

### Creating a Table
$\begin{bmatrix}
    & T_1 & T_2 & \cdots & T_n \\
P_1 & \mu_{11} & \mu_{12} & \cdots & \mu_{1n}\\
P_2 & \vdots & \vdots & \vdots & \vdots \\
\vdots & \vdots & \vdots & \vdots & \vdots\\
P_m & \mu_{m1} & \mu_{m2} & \cdots & \mu_{mn}
\end{bmatrix}$

 and save it to a file (.csv)

```python
df = pd.DataFrame(columns=T,index=P,data=Mu)
df
```

      50.0
      58.3333333333
      66.6666666667
      75.0
      83.3333333333
      91.6666666667
      100.0

      100.0
      311.669997
      214.06139
      170.987257
      154.713126
      149.427951
      147.948487
      147.5908

      590.0
      299.419997
      201.81139
      158.737257
      142.463126
      137.177951
      135.698487
      135.3408

      1080.0
      287.169997
      189.56139
      146.487257
      130.213126
      124.927951
      123.448487
      123.0908

      1570.0
      274.919997
      177.31139
      134.237257
      117.963126
      112.677951
      111.198487
      110.8408

      2060.0
      262.669997
      165.06139
      121.987257
      105.713126
      100.427951
      98.948487
      98.5908

      2550.0
      250.419997
      152.81139
      109.737257
      93.463126
      88.177951
      86.698487
      86.3408

      3040.0
      238.169997
      140.56139
      97.487257
      81.213126
      75.927951
      74.448487
      74.0908

      3530.0
      225.919997
      128.31139
      85.237257
      68.963126
      63.677951
      62.198487
      61.8408

      4020.0
      213.669997
      116.06139
      72.987257
      56.713126
      51.427951
      49.948487
      49.5908

      4510.0
      201.419997
      103.81139
      60.737257
      44.463126
      39.177951
      37.698487
      37.3408

      5000.0
      189.169997
      91.56139
      48.487257
      32.213126
      26.927951
      25.448487
      25.0908

```python
df.to_csv('df_plain.csv')
```

```python
T_first_level = ['Temperature'] * T.shape[0]
P_first_level = ['Pressure'] * P.shape[0]
df_multi_index = pd.DataFrame(columns=[T_first_level,T]
                              ,index=[P_first_level,P],data=Mu)
df_multi_index
```

      Temperature

      50.000000
      58.333333
      66.666667
      75.000000
      83.333333
      91.666667
      100.000000

      Pressure
      100.0
      311.669997
      214.06139
      170.987257
      154.713126
      149.427951
      147.948487
      147.5908

      590.0
      299.419997
      201.81139
      158.737257
      142.463126
      137.177951
      135.698487
      135.3408

      1080.0
      287.169997
      189.56139
      146.487257
      130.213126
      124.927951
      123.448487
      123.0908

      1570.0
      274.919997
      177.31139
      134.237257
      117.963126
      112.677951
      111.198487
      110.8408

      2060.0
      262.669997
      165.06139
      121.987257
      105.713126
      100.427951
      98.948487
      98.5908

      2550.0
      250.419997
      152.81139
      109.737257
      93.463126
      88.177951
      86.698487
      86.3408

      3040.0
      238.169997
      140.56139
      97.487257
      81.213126
      75.927951
      74.448487
      74.0908

      3530.0
      225.919997
      128.31139
      85.237257
      68.963126
      63.677951
      62.198487
      61.8408

      4020.0
      213.669997
      116.06139
      72.987257
      56.713126
      51.427951
      49.948487
      49.5908

      4510.0
      201.419997
      103.81139
      60.737257
      44.463126
      39.177951
      37.698487
      37.3408

      5000.0
      189.169997
      91.56139
      48.487257
      32.213126
      26.927951
      25.448487
      25.0908

```python
df_multi_index.to_csv('df_multi_index.csv')
```

### Step 1: Load data from file

### Working with regular dataframe (no multi-level indexing)

```python
df_plain_in = pd.read_csv('df_plain.csv',index_col = [0])
df_plain_in
```

      50.0
      58.333333333333336
      66.66666666666667
      75.0
      83.33333333333334
      91.66666666666667
      100.0

      100.0
      311.669997
      214.06139
      170.987257
      154.713126
      149.427951
      147.948487
      147.5908

      590.0
      299.419997
      201.81139
      158.737257
      142.463126
      137.177951
      135.698487
      135.3408

      1080.0
      287.169997
      189.56139
      146.487257
      130.213126
      124.927951
      123.448487
      123.0908

      1570.0
      274.919997
      177.31139
      134.237257
      117.963126
      112.677951
      111.198487
      110.8408

      2060.0
      262.669997
      165.06139
      121.987257
      105.713126
      100.427951
      98.948487
      98.5908

      2550.0
      250.419997
      152.81139
      109.737257
      93.463126
      88.177951
      86.698487
      86.3408

      3040.0
      238.169997
      140.56139
      97.487257
      81.213126
      75.927951
      74.448487
      74.0908

      3530.0
      225.919997
      128.31139
      85.237257
      68.963126
      63.677951
      62.198487
      61.8408

      4020.0
      213.669997
      116.06139
      72.987257
      56.713126
      51.427951
      49.948487
      49.5908

      4510.0
      201.419997
      103.81139
      60.737257
      44.463126
      39.177951
      37.698487
      37.3408

      5000.0
      189.169997
      91.56139
      48.487257
      32.213126
      26.927951
      25.448487
      25.0908

```python
#extracting index into numpy array
np.array(df_plain_in.index)
```

    array([  100.,   590.,  1080.,  1570.,  2060.,  2550.,  3040.,  3530.,
            4020.,  4510.,  5000.])

```python
#extracting column name into numpy array
#need 'float64', DataFrame automatically treat
#column name as string
np.array(df_plain_in.columns,dtype='float64')
```

    array([  50.        ,   58.33333333,   66.66666667,   75.        ,
             83.33333333,   91.66666667,  100.        ])

### Working with multi-level indexing dataframe

```python
df_mul_in = pd.read_csv('df_multi_index.csv',header=[0,1],index_col=[0,1])
# index_col is to tell the column where we have rows' name
# header is to tell the row where we have columns' name
df_mul_in
```

      Temperature

      50.0
      58.3333333333
      66.6666666667
      75.0
      83.3333333333
      91.6666666667
      100.0

      Pressure
      100.0
      311.669997
      214.06139
      170.987257
      154.713126
      149.427951
      147.948487
      147.5908

      590.0
      299.419997
      201.81139
      158.737257
      142.463126
      137.177951
      135.698487
      135.3408

      1080.0
      287.169997
      189.56139
      146.487257
      130.213126
      124.927951
      123.448487
      123.0908

      1570.0
      274.919997
      177.31139
      134.237257
      117.963126
      112.677951
      111.198487
      110.8408

      2060.0
      262.669997
      165.06139
      121.987257
      105.713126
      100.427951
      98.948487
      98.5908

      2550.0
      250.419997
      152.81139
      109.737257
      93.463126
      88.177951
      86.698487
      86.3408

      3040.0
      238.169997
      140.56139
      97.487257
      81.213126
      75.927951
      74.448487
      74.0908

      3530.0
      225.919997
      128.31139
      85.237257
      68.963126
      63.677951
      62.198487
      61.8408

      4020.0
      213.669997
      116.06139
      72.987257
      56.713126
      51.427951
      49.948487
      49.5908

      4510.0
      201.419997
      103.81139
      60.737257
      44.463126
      39.177951
      37.698487
      37.3408

      5000.0
      189.169997
      91.56139
      48.487257
      32.213126
      26.927951
      25.448487
      25.0908

```python
df_mul_in.columns
```

    MultiIndex(levels=[['Temperature'], ['100.0', '50.0', '58.3333333333', '66.6666666667', '75.0', '83.3333333333', '91.6666666667']],
               labels=[[0, 0, 0, 0, 0, 0, 0], [1, 2, 3, 4, 5, 6, 0]])

```python
df_mul_in.columns.get_level_values(1)
```

    Index(['50.0', '58.3333333333', '66.6666666667', '75.0', '83.3333333333',
           '91.6666666667', '100.0'],
          dtype='object')

```python
T_in = np.array(df_mul_in.columns.get_level_values(1),dtype = 'float64')
#Note that without dtype = 'float64' it is just string
T_in
```

    array([  50.        ,   58.33333333,   66.66666667,   75.        ,
             83.33333333,   91.66666667,  100.        ])

```python
Mu_in = df_mul_in.values
df_mul_in.values.astype('float64')
#without .astype('float64') it work too
#use .astype('float64') in case where you get string
```

    array([[ 311.66999725,  214.06139014,  170.98725691,  154.71312627,
             149.42795145,  147.94848712,  147.59079986],
           [ 299.41999725,  201.81139014,  158.73725691,  142.46312627,
             137.17795145,  135.69848712,  135.34079986],
           [ 287.16999725,  189.56139014,  146.48725691,  130.21312627,
             124.92795145,  123.44848712,  123.09079986],
           [ 274.91999725,  177.31139014,  134.23725691,  117.96312627,
             112.67795145,  111.19848712,  110.84079986],
           [ 262.66999725,  165.06139014,  121.98725691,  105.71312627,
             100.42795145,   98.94848712,   98.59079986],
           [ 250.41999725,  152.81139014,  109.73725691,   93.46312627,
              88.17795145,   86.69848712,   86.34079986],
           [ 238.16999725,  140.56139014,   97.48725691,   81.21312627,
              75.92795145,   74.44848712,   74.09079986],
           [ 225.91999725,  128.31139014,   85.23725691,   68.96312627,
              63.67795145,   62.19848712,   61.84079986],
           [ 213.66999725,  116.06139014,   72.98725691,   56.71312627,
              51.42795145,   49.94848712,   49.59079986],
           [ 201.41999725,  103.81139014,   60.73725691,   44.46312627,
              39.17795145,   37.69848712,   37.34079986],
           [ 189.16999725,   91.56139014,   48.48725691,   32.21312627,
              26.92795145,   25.44848712,   25.09079986]])

```python
Mu_in
```

    array([[ 311.66999725,  214.06139014,  170.98725691,  154.71312627,
             149.42795145,  147.94848712,  147.59079986],
           [ 299.41999725,  201.81139014,  158.73725691,  142.46312627,
             137.17795145,  135.69848712,  135.34079986],
           [ 287.16999725,  189.56139014,  146.48725691,  130.21312627,
             124.92795145,  123.44848712,  123.09079986],
           [ 274.91999725,  177.31139014,  134.23725691,  117.96312627,
             112.67795145,  111.19848712,  110.84079986],
           [ 262.66999725,  165.06139014,  121.98725691,  105.71312627,
             100.42795145,   98.94848712,   98.59079986],
           [ 250.41999725,  152.81139014,  109.73725691,   93.46312627,
              88.17795145,   86.69848712,   86.34079986],
           [ 238.16999725,  140.56139014,   97.48725691,   81.21312627,
              75.92795145,   74.44848712,   74.09079986],
           [ 225.91999725,  128.31139014,   85.23725691,   68.96312627,
              63.67795145,   62.19848712,   61.84079986],
           [ 213.66999725,  116.06139014,   72.98725691,   56.71312627,
              51.42795145,   49.94848712,   49.59079986],
           [ 201.41999725,  103.81139014,   60.73725691,   44.46312627,
              39.17795145,   37.69848712,   37.34079986],
           [ 189.16999725,   91.56139014,   48.48725691,   32.21312627,
              26.92795145,   25.44848712,   25.09079986]])

```python
df_mul_in.index.get_level_values(1)
```

    Float64Index([ 100.0,  590.0, 1080.0, 1570.0, 2060.0, 2550.0, 3040.0, 3530.0,
                  4020.0, 4510.0, 5000.0],
                 dtype='float64')

```python
P_in = np.array(df_mul_in.index.get_level_values(1))
P_in
```

    array([  100.,   590.,  1080.,  1570.,  2060.,  2550.,  3040.,  3530.,
            4020.,  4510.,  5000.])

### Find Mu at T = 70 & P = 3000 from a data file with scipy

```python
P_in.shape
```

    (11,)

```python
T_in.shape
```

    (7,)

```python
Mu_in.shape
```

    (11, 7)

```python
model = interpolate.interp2d(x = T_in, y = P_in, z = Mu_in)
#Usage:  x = [0,1,2];  y = [0,3]; z = [[1,2,3], [4,5,6]]
# x is for column, y is for row of z (mimic graph x,y)
# interpolate.interp2d(x, y, z, kind='linear',
#     copy=True, bounds_error=False, fill_value=None)
```

```python
val = (70,3000)
z_pred = model(*val)
z_pred
```

    array([ 91.97760466])

```python
z_pred
```

    array([ 91.97760466])

### Find Mu at T = 70 & P = 3000 from a data file manually
Step 0: need to find x and y that surround point(x,y) of interest first

```python
id_x = np.searchsorted(T_in,val[0]) - 1
id_y = np.searchsorted(P_in,val[1]) - 1
print(id_x,id_y)
```

    2 5

```python
T_in[2]
```

    66.666666666699996

```python
T_in[3]
```

    75.0

```python
P_in[5]
```

    2550.0

```python
P_in[6]
```

    3040.0

```python
Mu_in[5:7,2:4]
#first index is for P because,
# P is the index in pandas DataFrame
# while T is the column in pandas DataFrame
```

    array([[ 109.73725691,   93.46312627],
           [  97.48725691,   81.21312627]])

What we are going to do is equivalent to

$\begin{bmatrix}
 & \text{T_in[2] = 66.6666} & \text{x_target = 70} & \text{T_in[3] = 75.0}\\
\text{P_in[5] = 2550.0} & 109.73725691 & 103.22760465727725 & 93.46312627\\
\text{y_target = 3000} &  & 91.977604657277254 &\\
\text{P_in[6] = 3040.0} & 97.48725691 & 90.977604657277254 & 81.21312627
\end{bmatrix}
$

Step 1: find the z value at x = x_target and y = [y1, y2]

y1 and y2 are y are from the given information

```python
z_y1 = (Mu_in[id_y,id_x] + (Mu_in[id_y,id_x+1] - Mu_in[id_y,id_x])*
            (val[0] - T_in[id_x]) / (T_in[id_x+1] - T_in[id_x])  )
#index id_y for all Mu_in
z_y1
```

    103.22760465727725

```python
z_y2 = (Mu_in[id_y+1,id_x] + (Mu_in[id_y+1,id_x+1] - Mu_in[id_y+1,id_x])*
            (val[0] - T_in[id_x]) / (T_in[id_x+1] - T_in[id_x])  )
#index id_y+1 for all Mu_in
z_y2
```

    90.977604657277254

Step 3: use y1 and y2 to find z value at y_target

```python
z_ans = (z_y1 + (z_y2 - z_y1)*
            (val[1] - P_in[id_y]) / (P_in[id_y+1] - P_in[id_y])  )
z_ans
```

    91.977604657277254

```python
#comparing the result from manual method and scipy method
z_ans == z_pred[0]
```

    False

```python
z_ans - z_pred[0]
```

    -2.8421709430404007e-14

### Very close!

# Simple Cubic Spline (just a curve between 2 dots)
Interpolate by using piecewise third-order polynomial

$q = (1-t)y_1 + t y_2 + t(1-t)(a(1-t)+bt)$

$t=\frac{x-x_1}{x_2-x_1}$

$a = k_1(x_2-x_1)-(y_2 - y_1)$

$b = -k_2(x_2 - x_1) + (y_2 - y_1)$

###  $k$ is the slope
###  $q$ is the function value
### Required given data are
    q(x1) = y1
    q(x2) = y2
    q'(x1) = k1
    q'(x2) = k2

 Derivation https://en.wikipedia.org/wiki/Spline_interpolation

### Problem statement: $\text{Find the curve between two dots given location and slope at those points}$

$x1 = 0$ $\qquad\quad$ $x2 = 1$

$y1 = 0$ $\qquad\quad$ $y2 = 0$

$f'(x1) = 1$ $\qquad$ $f'(x2) = 1$

```python
x1 = 0
x2 = 1
y1 = 0
y2 = 0
k1 = 1
k2 = 1
x = np.linspace(-1,2,100)
t = (x-x1)/(x2-x1)
a = k1*(x2 - x1) - (y2 - y1)
b = -k2*(x2 - x1) + (y2 - y1)
q = (1 - t)*y1 + t*y2 + t*(1 - t)*(a*(1 - t) + b*t)
```

```python
plt.plot(t,q)
plt.plot([x1,x2],[y1,y2],'ob')
plt.show()
```

### Recheck the slope at x1 and x2 to see if it match with k1 & k2 or not

```python
np.searchsorted(t,1)
```

    66

```python
t[66]
```

    1.0

```python
np.searchsorted(t,0)
```

    33

```python
t[33]
```

    0.0

```python
#Check the slope by using np.gradient
#np.gradient take list of y and x, then
#calculate slope by central finite difference
#for point in the middle
```

```python
np.gradient(q,t)[66]
```

    1.0018365472910926

```python
np.gradient(q,t)[33]
```

    1.0018365472910926

```python
# Manually checking slope
```

```python
m1 = (q[66]-q[65])/(t[66]-t[65])
m2 = (q[67]-q[66])/(t[67]-t[66])
m = 0.5*(m1+m2)
m
```

    1.0018365472910928

```python
#our 2nd finite difference central scheme match with what numpy gave
```

### Simple cubic spline for various slope condition

```python
def simple_cubic(x,x1,x2,y1,y2,k1,k2):
    t = (x-x1)/(x2-x1)
    a = k1*(x2 - x1) - (y2 - y1)
    b = -k2*(x2 - x1) + (y2 - y1)
    q = (1 - t)*y1 + t*y2 + t*(1 - t)*(a*(1 - t) + b*t)
    return q

plt.figure()

x1 = 0
x2 = 1
y1 = 0
y2 = 0
plt.plot([x1,x2],[y1,y2],'ob')
for k2 in range(-2,3):
    XX = np.linspace(-0.5,1.5,100)
    q = simple_cubic(XX, x1 = 0,
                    x2 = 1,
                    y1 = 0,
                    y2 = 0,
                    k1 = 1,
                    k2 = k2)
    plt.plot(XX,q)
plt.xlim(-0.2,1.2)
plt.ylim(-0.6,0.6)
plt.grid()
plt.show()
```

# Full Cubic Spline Interpolation (for multiple points): Manual Approach
## Use Natural boundary condition: $q''(x_0) = q''(x_n) = 0$

 Piecewise cubic equation will be used to connect points

Properties of cubic spline output:

 1: q(x) is a piecewise continuous function

 2: q'(x) is a piecewise continuous function

 3: q''(x) is a piecewise continuous function

 4: q''(x0) = q''(xn) = 0

## Calculation steop

 Step 0: Specify x and y coordinate of each point

  Step 1: Calculate the slope at each point (to be explained shortly)

  Step 2: Create equation of each interval from the obtained slope

  Step 3: Use equation from step 2 to do interpolation

## Step 0: Generating input: $input = (x,y)_1, (x,y)_2, ..., (x,y)_n$

```python
x = np.linspace(1,10,20)
y = np.sin(5*x)+x
plt.plot(x,y,'bo',label = 'Given points')
x_true = np.linspace(1,10,200)
y_true = np.sin(5*x_true)+x_true
plt.plot(x_true,y_true,'--',label = 'True answers')
plt.legend(loc=4)
plt.show()
```

## Step 1: Calculate slope (k) at each point

Slope at each point is from solving system of linear equations. The derivation of the formula for system of linear equation for cubic spline interpolation is not covered. It can be found in https://en.wikipedia.org/wiki/Spline_interpolation

In short, it is in the form of a tri-diagonal matrix

$B_{0}k_{0} + C_{0} k_{1} = D_{0} \qquad\qquad\quad\:$ For i = 0

$A_{i}k_{i-1} + B_{i}k_{i} + C_{i} k_{i+1} = D_{i} \qquad$ For i = 1 to n-1

$A_{n}k_{n-1} + B_{n}k_{n}= D_{n} \qquad\qquad\:\:$ For i = 0

or

$\begin{bmatrix}
B_0  & C_0  \\
A_1 & B_1 & C_1 \\
 & \ddots & \ddots & \ddots \\
 &  & \ddots & \ddots & \ddots \\
 &  &  &  A_{n-1} & B_{n-1} & C_{n-1}\\
 &  &  &  & A_n & B_n
\end{bmatrix}
\begin{bmatrix}
k_0\\
k_1\\
k_2\\
\vdots\\
k_{n-1}\\
k_{n}
\end{bmatrix}
=
\begin{bmatrix}
D_0\\
D_1\\
D_2\\
\vdots\\
D_{n-1}\\
D_{n}
\end{bmatrix}
$

The definition of each equations are as follows:

$\begin{align*}
2(x_1-x_0)\:k_0 + (x_1-x_0)\:k_1 & = 3 (y_1-y_0) \\
\frac {k_{i-1}}{x_i-x_{i-1}} + \left(\frac {1}{x_i-x_{i-1}}+ \frac {1}{{x_{i+1}-x_i}}\right) 2k_i+ \frac {k_{i+1}}{{x_{i+1}-x_i}} & =
   3 \left(\frac {y_i - y_{i-1}}{{(x_i-x_{i-1})}^2}+\frac {y_{i+1} - y_i}{{(x_{i+1}-x_i)}^2}\right)\\
(x_n-x_{n-1})\:k_{n-1} + 2(x_n-x_{n-1})\:k_n & = 3 (y_n-y_{n-1})
\end{align*}$

$\begin{align*}A_i & = \frac {1}{x_i-x_{i-1}}\\
B_i & = 2 \left(\frac {1}{x_i-x_{i-1}}+ \frac {1}{{x_{i+1}-x_i}}\right)\\
C_i & = \frac {1}{{x_{i+1}-x_i}}\\
D_i & = 3 \left(\frac {y_i - y_{i-1}}{{(x_i-x_{i-1})}^2}+\frac {y_{i+1} - y_i}{{(x_{i+1}-x_i)}^2}\right)
\end{align*}$

```python
#From the previous example on how to create a sparse tri-diagonal matrix, we have
test = sp.sparse.diags(diagonals=[[10,20,30,40,50],[1,2,3,4],[0.1,0.2,0.3,0.4]],
                       offsets=[0,-1,1])
test.todense()
```

    matrix([[ 10. ,   0.1,   0. ,   0. ,   0. ],
            [  1. ,  20. ,   0.2,   0. ,   0. ],
            [  0. ,   2. ,  30. ,   0.3,   0. ],
            [  0. ,   0. ,   3. ,  40. ,   0.4],
            [  0. ,   0. ,   0. ,   4. ,  50. ]])

```python
B0 = 2*(x[1]-x[0])
C0 = x[1]-x[0]
D0 = 3*(y[1]-y[0])
Ai = [1.0/(x[i]-x[i-1]) for i in range(1,len(x)-1)]
Bi = [2 * (1.0/(x[i]-x[i-1]) + 1/(x[i+1]-x[i])) for i in range(1,len(x)-1)]
Ci = [1.0/(x[i+1]-x[i]) for i in range(1,len(x)-1)]
Di = [3*((y[i]-y[i-1])/(x[i]-x[i-1])**2 + (y[i+1]-y[i])/(x[i+1]-x[i])**2)
      for i in range(1,len(x)-1)]
n = len(x) - 1 # the index is from 0 to n-1, not to n
An = x[n]-x[n-1]
Bn = 2*(x[n]-x[n-1])
Dn = 3*(y[n]-y[n-1])
AA = Ai+[An] # + operation for list is concatenation (similar to .append)
BB = [B0] + Bi + [Bn]
CC = [C0] + Ci
DD = [D0] + Di + [Dn]
CSpline = sp.sparse.diags(diagonals=[BB,AA,CC], offsets=[0,-1,1])
#Solve AA*KK = DD, solve for KK or vector of slope
KK = sp.sparse.linalg.spsolve(CSpline.tocsc(),DD)
#sp.sparse.linalg.spsolve accept the sparse matrix in the csc and csr formats
#not the diagonal format
```

```python
KK.shape
```

    (20,)

## Step 2 + 3: Use simple cubic function to generate interpolation line

```python
#i is for starting index of an interval
x_tot = np.array([])
q_tot = np.array([])
for i in range(KK.shape[0]-1):
    xx = np.linspace(x[i],x[i+1],51)
    qq = simple_cubic(xx,x[i],x[i+1],y[i],y[i+1],KK[i],KK[i+1])
    x_tot = np.r_[x_tot,xx] # np.r_[] is to append numpy arrays together
    q_tot = np.r_[q_tot,qq]
plt.figure(figsize=(14,8))
plt.plot(x_tot,q_tot,label = 'Cubic Spline Answer')
plt.plot(x,y,'o', label = 'Given points')
plt.plot(x_true,y_true,'r-',linewidth = 0.5, label = 'Points Generator Function')
plt.legend(loc=4)
plt.xlabel('x')
plt.ylabel('y')
plt.show()

plt.figure(figsize=(14,8))
plt.plot(x,y,'o', label = 'Given points')
plt.title("Cannot tell the function\n from the given points alone")
plt.xlabel('x')
plt.ylabel('y')
plt.show()
```

# Full Cubic Spline with automatic function (Scipy)

```python
model = sp.interpolate.CubicSpline(x,y,bc_type='natural')
```

```python
y_pred = model(np.linspace(1,10,100))
plt.plot(np.linspace(1,10,100),y_pred, label='Scipy')
plt.title('Scipy versus Manual')
plt.plot(x_tot,q_tot,label = 'Manual',linewidth=5,alpha = 0.5)
plt.xlabel('x')
plt.ylabel('y')
plt.legend()
plt.show()
```

# Slope / Integration result from `scipy.interpolation.CubicSpline`

```python
y_slope_pred = model(np.linspace(1,10,100),nu=1)
plt.plot(np.linspace(1,10,100),y_pred, label='value')
plt.plot(np.linspace(1,10,100),y_slope_pred, label='slope')
plt.xlabel('x')
plt.ylabel('y')
plt.legend(bbox_to_anchor=(1., 0.8))
plt.grid()
plt.show()
```

```python
y_pred = model(np.linspace(1,10,1000))
y_slope_pred = model(np.linspace(1,10,1000),nu=1)
y_slope2_pred = model(np.linspace(1,10,1000),nu=2)
plt.plot(np.linspace(1,10,1000),y_pred, label='value')
plt.plot(np.linspace(1,10,1000),y_slope_pred, label='slope')
plt.plot(np.linspace(1,10,1000),y_slope2_pred, label='$\\frac{d^2x}{dy^2}$')
plt.xlabel('x')
plt.ylabel('y')
plt.legend(bbox_to_anchor=(1., 0.8))
plt.grid()
plt.show()
```

## Derivative comparison: Cubic spline vs Analytical Solution

```python
xsym = sympy.symbols('x')
diff_result = sympy.diff(xsym + sympy.sin(5*xsym), xsym)
sympy.init_printing(use_unicode=True)
diff_result
```

```python
dif_fn = sympy.lambdify('x', diff_result)
x_diff = np.linspace(1,10,101)
y_analy = np.array(list(map(dif_fn,x_diff)))
y_cubic = model(x_diff,nu=1)
plt.plot(x_diff,y_analy, label = 'Analytical slope value')
plt.plot(x_diff,y_cubic, label = 'C-spline slope value')
plt.title('Slope comparison')
plt.xlabel('x')
plt.ylabel('dy/dx')
plt.legend(bbox_to_anchor = (1,0.8))
plt.show()
```

```python
(y_cubic - y_analy).mean()
```

```python
(y_cubic - y_analy).max()
```

```python
(y_cubic - y_analy).std()
```

```python
rela_er = (y_cubic - y_analy) / y_analy * 100
print('Relative error in slope calculation is about {:.3f}% on average'.format(rela_er.mean()))
print('with the median of {:.3f}%'.format(np.median(rela_er)))
```

    Relative error in slope calculation is about -10.144% on average
    with the median of -13.276%

```python
plt.hist((y_cubic - y_analy) / y_analy * 100)
plt.ylabel('Frequency')
plt.xlabel('Relative error [%]')
plt.show()
```

## Integration result comparison
## Analytical versus approximation from cubic spline

```python
model.integrate(1,5)
```

    array(11.946398953852404)

```python
xsym = sympy.symbols('x')
result = sympy.integrate(xsym + sympy.sin(5*xsym), (xsym,1,5))
result
```

```python
result.evalf()
```

```python
ysym = sympy.symbols('y')
int_result = sympy.integrate(xsym + sympy.sin(5*xsym), (xsym,1,ysym))
sm_func = sympy.lambdify('y',int_result)

x_inte = np.linspace(1,10,101)
y_inte = [model.integrate(1,i) for i in x_inte]
y_sm = [sm_func(i) for i in x_inte]
plt.plot(x_inte, y_inte,linewidth = 3, label = 'integrating cubic spline')
plt.plot(x_inte, y_sm, label = 'analytical solution of generating function')
plt.ylabel('integration result')
plt.xlabel('x')
plt.legend(bbox_to_anchor = (1,0.8))
plt.show()
```

```python
sp.linalg.norm(np.array(y_sm) - np.array(y_inte))
```

```python
(np.array(y_sm) - np.array(y_inte)).mean()
```

```python
((np.array(y_sm) - np.array(y_inte))/np.array(y_sm).max()*100).mean()
```

```python
#on average, the error is about -0.076
```

```python
plt.plot(x_inte,np.array(y_inte) - np.array(y_sm))
plt.title('integration from actual vs \napproximation by cubic spline')
plt.ylabel('error from integration')
plt.xlabel('x')
plt.show()
```

```python
# Sometimes cubic spline over-estimate
# sometimes under-estimate
```

```python
plt.figure(figsize=(12,6))
IntErr = ((np.array(y_inte)-np.array(y_sm))/np.array(y_sm)) * 100
IntErr = IntErr[~np.isnan(IntErr)]
plt.hist(IntErr,alpha = 0.5)
plt.ylabel('frequency')
plt.xlabel('relative integration error')
plt.show()
print('average relative error = {:.3f}%'.format(IntErr.mean()))
```


 Text provided under a Creative Commons Attribution license, CC-BY. All code is made available under the FSF-approved MIT license. (c) Kyle T. Mandli

```python
%matplotlib inline
import numpy
import matplotlib.pyplot as plt
```

# Root Finding and Optimization

**GOAL:** Find where $f(x) = 0$.

### Example:  Future Time Annuity

When can I retire?

$$ A = \frac{P}{(r / m)} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n} - 1 \right ] $$

$A$ total value after $n$ years

$P$ is payment amount per compounding period

$m$ number of compounding periods per year

$r$ annual interest rate

$n$ number of years to retirement

If I want to retire in 20 years what does the annual interest rate $r$ need to be?

Set $P = \frac{\$18,000}{12} = \$1500, \quad m=12, \quad n=20$.

```python
def total_value(P, m, r, n):
    """Total value of portfolio given parameters

    Based on following formula:

    A = \frac{P}{(r / m)} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n}
                - 1 \right ]

    :Input:
     - *P* (float) - Payment amount per compounding period
     - *m* (int) - number of compounding periods per year
     - *r* (float) - annual interest rate
     - *n* (float) - number of years to retirement

     :Returns:
     (float) - total value of portfolio

    """

    return P / (r / float(m)) * ( (1.0 + r / float(m))**(float(m) * n)
                                 - 1.0)

P = 1500.0
m = 12
n = 20.0

r = numpy.linspace(0.05, 0.1, 100)
goal = 1e6

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, total_value(P, m, r, n))
axes.plot(r, numpy.ones(r.shape) * goal, 'r--')
axes.set_xlabel("r (interest rate)")
axes.set_ylabel("A (total value)")
axes.set_title("When can I retire?")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))
plt.show()
```

## Fixed Point Iteration

How do we go about solving this?

Could try to solve at least partially for $r$:

$$ A = \frac{P}{(r / m)} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n} - 1 \right ] ~~~~ \Rightarrow ~~~~~$$

$$ r = \frac{P \cdot m}{A} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n} - 1 \right ] ~~~~ \Rightarrow ~~~~~$$

$$ r = g(r)$$
or
$$ g(r) - r = 0$$

```python
def g(P, m, r, n, A):
    """Reformulated minimization problem

    Based on following formula:

    g(r) = \frac{P \cdot m}{A} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n} - 1 \right ]

    :Input:
     - *P* (float) - Payment amount per compounding period
     - *m* (int) - number of compounding periods per year
     - *r* (float) - annual interest rate
     - *n* (float) - number of years to retirement
     - *A* (float) - total value after $n$ years

     :Returns:
     (float) - value of g(r)

    """

    return P * m / A * ( (1.0 + r / float(m))**(float(m) * n)
                                 - 1.0)

P = 1500.0
m = 12
n = 20.0

r = numpy.linspace(0.00, 0.1, 100)
goal = 1e6

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, g(P, m, r, n, goal))
axes.plot(r, r, 'r--')
axes.set_xlabel("r (interest rate)")
axes.set_ylabel("$g(r)$")
axes.set_title("When can I retire?")
axes.set_ylim([0, 0.12])
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))
plt.show()
```

Guess at $r_0$ and check to see what direction we need to go...

1. $r_0 = 0.0800, \quad g(r_0) - r_0 = -0.009317550125425428$
1. $r_1 = 0.0850, \quad g(r_1) - r_1 = -0.00505763375972$
1. $r_2 = 0.0875, \quad g(r_2) - r_2 = -0.00257275331014$

A bit tedious, we can also make this algorithmic:
```python
r_values = numpy.linspace(0.08, 0.09, 10)
for r in r_values:
    print("r = ", r, "g(r) =", g(P, m, r, n, goal))
    print("Difference = ", numpy.abs(g(P, m, r, n, goal) - r))
    r = g(P, m, r, n, goal)
```

```python
r_values = numpy.linspace(0.08, 0.09, 11)
for r in r_values:
    print("r = ", r, "g(r) =", g(P, m, r, n, goal))
    print("Difference = ", numpy.abs(g(P, m, r, n, goal) - r))
    r = g(P, m, r, n, goal)
```

### Example 2:

Let $f(x) = x - e^{-x}$, solve $f(x) = 0$

Equivalent to $x = e^{-x}$ or $x = g(x)$ where $g(x) = e^{-x}$

```python
x = numpy.linspace(0.2, 1.0, 100)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, numpy.exp(-x), 'r')
axes.plot(x, x, 'b')
axes.set_xlabel("x")
axes.set_ylabel("f(x)")

x = 0.4
for steps in range(3):
    print("x = ", x, "Residual = ", numpy.abs(numpy.exp(-x) - x))
    x = numpy.exp(-x)
    axes.plot(x, numpy.exp(-x),'kx')
    axes.text(x, numpy.exp(-x), steps+1, fontsize="15")

plt.show()
```

### Example 3:

Let $f(x) = \ln x + x$ and solve $f(x) = 0$ or $x = -\ln x$.

Note that this problem is equivalent to $x = e^{-x}$.

```python
x = numpy.linspace(0.1, 1.0, 100)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, -numpy.log(x), 'r')
axes.plot(x, x, 'b')
axes.set_xlabel("x")
axes.set_ylabel("f(x)")
axes.set_ylim([0, 1.5])

x = 0.5
for steps in range(3):
    print("x = ", x, "Residual = ", numpy.abs(numpy.log(x) + x))
    x = -numpy.log(x)
    axes.plot(x, -numpy.log(x),'kx')
    axes.text(x, -numpy.log(x), steps+1, fontsize="15")

plt.show()
```

These are equivalent problems!  Something is awry...

## Analysis of Fixed Point Iteration

Existence and uniqueness of fixed point problems

*Existence:*

Assume $g \in C[a, b]$, if the range of the mapping $y = g(x)$ satisfies $y \in [a, b] \quad \forall \quad x \in [a, b]$ then $g$ has a fixed point in $[a, b]$.

```python
x = numpy.linspace(0.0, 1.0, 100)

# Plot function and intercept
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, numpy.exp(-x), 'r')
axes.plot(x, x, 'b')
axes.set_xlabel("x")
axes.set_ylabel("f(x)")

# Plot domain and range
axes.plot(numpy.ones(x.shape) * 0.4, x, '--k')
axes.plot(numpy.ones(x.shape) * 0.8, x, '--k')
axes.plot(x, numpy.ones(x.shape) * numpy.exp(-0.4), '--k')
axes.plot(x, numpy.ones(x.shape) * numpy.exp(-0.8), '--k')

axes.set_xlim((0.0, 1.0))
axes.set_ylim((0.0, 1.0))

plt.show()
```

```python
x = numpy.linspace(0.1, 1.0, 100)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, -numpy.log(x), 'r')
axes.plot(x, x, 'b')
axes.set_xlabel("x")
axes.set_ylabel("f(x)")
axes.set_xlim([0.1, 1.0])
axes.set_ylim([0.1, 1.0])

# Plot domain and range
axes.plot(numpy.ones(x.shape) * 0.4, x, '--k')
axes.plot(numpy.ones(x.shape) * 0.8, x, '--k')
axes.plot(x, numpy.ones(x.shape) * -numpy.log(0.4), '--k')
axes.plot(x, numpy.ones(x.shape) * -numpy.log(0.8), '--k')

plt.show()
```

```python
r = numpy.linspace(0.06, 0.1, 100)
goal = 1e6

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, g(P, m, r, n, goal))
axes.plot(r, r, 'r--')
axes.set_xlabel("r")
axes.set_ylabel("$g(r)$")
axes.set_xlim([0.06, 0.1])
axes.set_ylim([g(P, m, 0.06, n, goal), g(P, m, 0.1, n, goal)])
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

axes.plot([0.08, 0.08], [g(P, m, 0.06, n, goal), g(P, m, 0.1, n, goal)], '--k')
axes.plot([0.09, 0.09], [g(P, m, 0.06, n, goal), g(P, m, 0.1, n, goal)], '--k')
axes.plot(r, numpy.ones(r.shape) * g(P, m, 0.08, n, goal), '--k')
axes.plot(r, numpy.ones(r.shape) * g(P, m, 0.09, n, goal), '--k')

plt.show()
```

*Uniqueness:*

Additionally, suppose $g'(x)$ is defined on $x \in [a, b]$ and $\exists K < 1$ such that

$$
    |g'(x)| \leq K < 1 \quad \forall \quad x \in (a,b)
$$

then $g$ has a unique fixed point $P \in [a,b]$

```python
x = numpy.linspace(0.4, 0.8, 100)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, numpy.abs(-numpy.exp(-x)), 'r')
axes.plot(x, numpy.ones(x.shape), 'k--')
axes.set_xlabel("x")
axes.set_ylabel("f(x)")
axes.set_ylim((0.0, 1.1))
plt.show()
```

*Asymptotic convergence*: Behavior of fixed point iterations

$$x_{k+1} = g(x_k)$$

Assume that $\exists ~ x^\ast$ s.t. $x^\ast = g(x^\ast)$ (i.e. $x^\ast$ is the fixed point), then define

$$
    x_k = x^\ast + e_k \quad \quad x_{k+1} = x^\ast + e_{k+1}
$$
and
$$
    x^\ast + e_{k+1} = g(x^\ast + e_k)
$$

Taylor expand the function $g$ about $x^\ast$:

$$
    g(x) = g(x^\ast) + g'(x^\ast) (x - x^\ast) + \frac{g''(x^\ast)}{2!} (x - x^\ast)^2 + \mathcal{O}((x - x^\ast)^3)
$$

Evaluate this series at $x_k = x^\ast + e_k$ to find

$$
    g(x^\ast + e_k) = g(x^\ast) + g'(x^\ast) e_k + \frac{g''(x^\ast) e_k^2}{2} + \mathcal{O}(e_k^3)
$$

therefore from our definition from before that $x^\ast + e_{k+1} = g(x^\ast + e_k)$ we have

$$
    x^\ast + e_{k+1} = g(x^\ast) + g'(x^\ast) e_k + \frac{g''(x^\ast) e_k^2}{2} + \mathcal{O}(e_k^3)
$$

Note that because $x^* = g(x^*)$ these terms cancel leaving

$$e_{k+1} = g'(x^*) e_k + \frac{g''(x^*) e_k^2}{2}$$

So if $|g'(x^*)| \leq K < 1$ we can conclude that

$$|e_{k+1}| = K |e_k|$$

which shows convergence.  Also note that $K$ is related to $|g'(x^*)|$.

### Convergence of iterative schemes

Given any iterative scheme where

$$|e_{k+1}| = C |e_k|^n$$

If $C < 1$ and:
 - $n=1$ then the scheme is **linearly convergent**
 - $n=2$ then the scheme is **quadratically convergent**
 - $n > 1$ the scheme can also be called **superlinearly convergent**

If $C > 1$ then the scheme is **divergent**

### Examples Revisited
$g(x) = e^{-x}$ with $x^* \approx 0.56$

   $$|g'(x^*)| = |-e^{-x^*}| \approx 0.56$$

$g(x) = - \ln x \quad \text{with} \quad x^* \approx 0.56$

   $$|g'(x^*)| = \frac{1}{|x^*|} \approx 1.79$$

$$
    r = g(r) = \frac{P \cdot m}{A} \left[ \left(1 + \frac{r}{m} \right)^{m \cdot n} - 1 \right ]
$$

```python
import sympy
r, P, m, A, n = sympy.symbols('r P m A n')
g = P * m / A * ((1 + r /m)**(m * n) - 1)
g_prime = g.diff(r)
r_star = 0.08985602484084668
print("g'(r) = ", g_prime)
print("g'(r*) = ", g_prime.subs({P: 1500.0, m: 12, n:20, A: 1e6, r: r_star}))

f = sympy.lambdify(r, g_prime.subs({P: 1500.0, m: 12, n:20, A: 1e6}))
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
r = numpy.linspace(-0.01, 0.1, 100)
axes.plot(r, f(r))
axes.plot(r, numpy.ones(r.shape), 'k--')
axes.plot(r_star, f(r_star), 'ro')
axes.plot(0.0, f(0.0), 'ro')
axes.set_xlim((-0.01, 0.1))
axes.set_xlabel("$r$")
axes.set_ylabel("$g'(r)$")
```

## Better ways for root-finding/optimization

If $x^*$ is a fixed point of $g(x)$ then $x^*$ is also a *root* of $f(x^*) = g(x^*) - x^*$ s.t. $f(x^*) = 0$.

For instance:

$$f(r) = r - \frac{m P}{A} \left [ \left (1 + \frac{r}{m} \right)^{m n} - 1 \right ] =0 $$

or

$$f(r) = A - \frac{m P}{r} \left [ \left (1 + \frac{r}{m} \right)^{m n} - 1 \right ] =0 $$

## Classical Methods
 - Bisection (linear convergence)
 - Newton's Method (quadratic convergence)
 - Secant Method (super-linear)

## Combined Methods
 - RootSafe (Newton + Bisection)
 - Brent's Method (Secant + Bisection)

### Bracketing and Bisection

A **bracket** is an interval $[a,b]$ that contains exactly one zero or minima/maxima of interest.

In the case of a zero the bracket should satisfy
$$
    \text{sign}(f(a)) \neq \text{sign}(f(b)).
$$

In the case of minima or maxima we need
$$
    \text{sign}(f'(a)) \neq \text{sign}(f'(b))
$$

**Theorem**:

Let
$$
    f(x) \in C[a,b] \quad \text{and} \quad \text{sign}(f(a)) \neq \text{sign}(f(b))
$$

then there exists a number
$$
    c \in (a,b) \quad \text{s.t.} \quad f(c) = 0.
$$
(proof uses intermediate value theorem)

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.1, 100)
f = lambda r, A, m, P, n: A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r, A, m, P, n), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')
axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

a = 0.075
b = 0.095
axes.plot(a, f(a, A, m, P, n), 'ko')
axes.plot([a, a], [0.0, f(a, A, m, P, n)], 'k--')
axes.plot(b, f(b, A, m, P, n), 'ko')
axes.plot([b, b], [f(b, A, m, P, n), 0.0], 'k--')

plt.show()
```

Basic bracketing algorithms shrink the bracket while ensuring that the root/extrema remains within the bracket.

What ways could we "shrink" the bracket so that the end points converge to the root/extrema?

#### Bisection Algorithm

Given a bracket $[a,b]$ and a function $f(x)$ -
1. Initialize with bracket
2. Iterate
   1. Cut bracket in half and check to see where the zero is
   2. Set bracket to new bracket based on what direction we went

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.11, 100)
f = lambda r, A=A, m=m, P=P, n=n: A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)

# Initialize bracket
a = 0.07
b = 0.10

# Setup figure to plot convergence
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r, A, m, P, n), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')
axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
# axes.set_xlim([0.085, 0.091])
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))
axes.plot(a, f(a, A, m, P, n), 'ko')
axes.plot([a, a], [0.0, f(a, A, m, P, n)], 'k--')
axes.plot(b, f(b, A, m, P, n), 'ko')
axes.plot([b, b], [f(b, A, m, P, n), 0.0], 'k--')

# Algorithm parameters
TOLERANCE = 1e-4
MAX_STEPS = 5

# Initialize loop
delta_x = b - a
c = a + delta_x / 2.0
f_a = f(a)
f_b = f(b)
f_c = f(c)

# Loop until we reach the TOLERANCE or we take MAX_STEPS
for step in range(1, MAX_STEPS + 1):

    # Plot iteration
    axes.plot(c, f_c,'kx')
    axes.text(c, f_c, str(step + 1), fontsize="15")

    # Check tolerance - Could also check the size of delta_x
    # We check this first as we have already initialized the values
    # in c and f_c
    if numpy.abs(f_c) < TOLERANCE:
        break

    if numpy.sign(f_a) != numpy.sign(f_c):
        b = c
        f_b = f_c
    else:
        a = c
        f_a = f_c
    delta_x = b - a
    c = a + delta_x / 2.0
    f_c = f(c)

if step == MAX_STEPS:
    print("Reached maximum number of steps!")
else:
    print("Success!")
    print("  x* = %s" % c)
    print("  f(x*) = %s" % f(c))
    print("  number of steps = %s" % step)
```

#### Convergence of Bisection

Generally have
$$
    |e_{k+1}| = C |e_k|^n
$$
where we need $C < 1$ and $n > 0$.

Letting $\Delta x_k$ be the width of the $k$th bracket we can then estimate the error with
$$
    e_k \approx \Delta x_k
$$
and therefore
$$
    e_{k+1} \approx \frac{1}{2} \Delta x_k.
$$
Due to the relationship then between $x_k$ and $e_k$ we then know
$$
    |e_{k+1}| = \frac{1}{2} |e_k|
$$
so therefore the method is linearly convergent.

### Newton's Method (Newton-Raphson)
 - Given a bracket, bisection is guaranteed to converge linearly to a root
 - However bisection uses almost no information about $f(x)$ beyond its sign at a point

**Basic Idea**: Given $f(x)$ and $f'(x)$ use a linear approximation to $f(x)$ "locally" and use the x-intercept of the resulting line to predict where $x^*$ might be.

Given current location $x_k$, we have $f(x_k)$ and $f'(x_k)$ and form a line through the point $(x_k, f(x_k))$:

Form equation for the line:

$$y = f'(x_k) x + b$$

Solve for the y-intercept value $b$

$$f(x_k) = f'(x_k) x_k + b$$

$$b = f(x_k) - f'(x_k) x_k$$

and simplify.

$$y = f'(x_k) x + f(x_k) - f'(x_k) x_k$$

$$y = f'(x_k) (x - x_k) + f(x_k)$$

Now find the intersection of our line and the x-axis (i.e. when $y = 0$) and use the resulting value of $x$ to set $x_{k+1}$

$$
    0 = f'(x_k) (x_{k+1}-x_k) + f(x_k)
$$

$$
    x_{k+1} = x_k-\frac{f(x_k)}{f'(x_k)}
$$

An alternative method of derivation for Newton-Raphson (and more in line with our methods) uses Taylor series.  Expand the function $f(x)$ in a Taylor series about the current Newton-Raphson iteration $x_k$:

$$
    f(x) = f(x_k) + f'(x_k) (x - x_k) + \frac{f''(x_k)}{2!} (x - x_k)^2 + \mathcal{O}((x-x_k)^3)
$$

Let $\delta_k$ be the update to the $x_{k+1}$ iteration such that
$$
    x_{k+1} = x_k + \Delta x_k
$$
and evaluate our expression for $f(x)$ at $x_{k+1}$:
$$
    f(x_{k+1}) = f(x_k) + f'(x_k) \Delta x_k + \frac{f''(x_k)}{2!} \Delta x_k^2 + \mathcal{O}(\Delta x_k^3)
$$

Now assume that $x_{k+1} = x^\ast$, if this is the case the above simplifies to
$$
    0 = f(x_k) + f'(x_k) \Delta x_k + \frac{f''(x_k)}{2!} \Delta x_k^2 + \mathcal{O}(\Delta x_k^3)
$$
and dropping the higher order terms leads to
$$
    \Delta x_k = - \frac{f(x_k)}{f'(x_k)}
$$
assuming that $f \in \mathbb R$ leading to the update
$$
    x_{k+1} = x_k - \frac{f(x_k)}{f'(x_k)}.
$$

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.11, 100)
f = lambda r, A=A, m=m, P=P, n=n: \
        A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)
f_prime = lambda r, A=A, m=m, P=P, n=n: \
            -P*m*n*(1.0 + r/m)**(m*n)/(r*(1.0 + r/m)) \
                + P*m*((1.0 + r/m)**(m*n) - 1.0)/r**2

# Initial guess
x_k = 0.06

# Setup figure to plot convergence
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')

# Plot x_k point
axes.plot([x_k, x_k], [0.0, f(x_k)], 'k--')
axes.plot(x_k, f(x_k), 'ko')
axes.text(x_k, -5e4, "$x_k$", fontsize=16)
axes.plot(x_k, 0.0, 'xk')
axes.text(x_k, f(x_k) + 2e4, "$f(x_k)$", fontsize=16)
axes.plot(r, f_prime(x_k) * (r - x_k) + f(x_k), 'k')

# Plot x_{k+1} point
x_k = x_k - f(x_k) / f_prime(x_k)
axes.plot([x_k, x_k], [0.0, f(x_k)], 'k--')
axes.plot(x_k, f(x_k), 'ko')
axes.text(x_k, 1e4, "$x_{k+1}$", fontsize=16)
axes.plot(x_k, 0.0, 'xk')
axes.text(0.0873, f(x_k) - 2e4, "$f(x_{k+1})$", fontsize=16)

axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
axes.set_title("Newton-Raphson Steps")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

plt.show()
```

What does the alogrithm look like for Newton-Raphson?

### Algorithm

1. Initialize $x_k$
1. Begin loop
  1. Compute $f(x_k)$ and $f'(x_k)$
  1. Use these to compute new $x_{k+1}$
  1. Check stopping criteria

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.11, 100)
f = lambda r, A=A, m=m, P=P, n=n: \
        A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)
f_prime = lambda r, A=A, m=m, P=P, n=n: \
            -P*m*n*(1.0 + r/m)**(m*n)/(r*(1.0 + r/m)) \
                + P*m*((1.0 + r/m)**(m*n) - 1.0)/r**2

# Algorithm parameters
MAX_STEPS = 200
TOLERANCE = 1e-4

# Initial guess
x_k = 0.06

# Setup figure to plot convergence
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')

for n in range(1, MAX_STEPS + 1):
    axes.plot(x_k, f(x_k),'kx')
    axes.text(x_k, f(x_k), str(n), fontsize="15")
    x_k = x_k - f(x_k) / f_prime(x_k)
    if numpy.abs(f(x_k)) < TOLERANCE:
        break

if n == MAX_STEPS:
    print("Reached maximum number of steps!")
else:
    print("Success!")
    print("  x* = %s" % x_k)
    print("  f(x*) = %s" % f(x_k))
    print("  number of steps = %s" % n)

axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
axes.set_title("Newton-Raphson Steps")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

plt.show()
```

### Example:

$$f(x) = x - e^{-x}$$

$$f'(x) = 1 + e^{-x}$$

$$x_{k+1} = x_k - \frac{f(x_k)}{f'(x_k)} = x_k - \frac{x_k - e^{-x_k}}{1 + e^{-x_k}}$$

### Asymptotic Convergence of Newton's Method

For a simple root (non-multiplicative) - Let $g(x) = x - \frac{f(x)}{f'(x)}$, then

$$x_{k+1} = g(x_k)$$

Definitions of errors and iteration:

$$x_{k+1} = x^* + e_{k+1} \quad \quad x_k = x^* + e_k$$

General Taylor expansion:

$$
    x^* + e_{k+1} = g(x^* + e_k) = g(x^*) + g'(x^*) e_k + \frac{g''(x^*) e_k^2}{2!} + \mathcal{O}(e_k^3)
$$

Note that as before $x^*$ and $g(x^*)$ cancel:

$$e_{k+1} = g'(x^*) e_k + \frac{g''(x^*) e_k^2}{2!} + \ldots$$

What about $g'(x^*)$ though?

$$\begin{aligned}
    g(x) &= x - \frac{f(x)}{f'(x)} \\
    g'(x) & = 1 - \frac{f'(x)}{f'(x)} + \frac{f(x) f''(x)}{(f'(x))^2} = \frac{f(x) f''(x)}{(f'(x))^2}
\end{aligned}$$

which evaluated at $x = x^*$ becomes

$$
    g'(x^*) = \frac{f(x^*)f''(x^*)}{f'(x^*)^2} = 0
$$

since $f(x^\ast) = 0$ by definition (assuming $f''(x^\ast)$ and $f'(x^\ast)$ are appropriately behaved).

Back to our expansion we have again

$$
    e_{k+1} = g'(x^*) e_k + \frac{g''(x^*) e_k^2}{2!} + \ldots
$$

which simplifies to

$$
    e_{k+1} = \frac{g''(x^*) e_k^2}{2!} + \ldots
$$

$$
    e_{k+1} = \frac{g''(x^*) e_k^2}{2!} + \ldots
$$
leads to
$$
    |e_{k+1}| = \left | \frac{g''(x^*)}{2!} \right | |e_k|^2
$$

Newton's method is therefore quadratically convergent where the the constant is controlled by the second derivative.

For a multiple root (e.g. $f(x) = (x-1)^2$) the case is not particularly rosey unfortunately.  Why might this be?

### Example:
$f(x) = \sin (2 \pi x)$

$$x_{k+1} = x_k - \frac{\sin (2 \pi x)}{2 \pi \cos (2 \pi x)}= x_k - \frac{1}{2 \pi} \tan (2 \pi x)$$

```python
x = numpy.linspace(0, 2, 1000)
f = lambda x: numpy.sin(2.0 * numpy.pi * x)
f_prime = lambda x: 2.0 * numpy.pi * numpy.cos(2.0 * numpy.pi * x)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, f(x),'b')
axes.plot(x, f_prime(x), 'r')
axes.set_xlabel("x")
axes.set_ylabel("y")
axes.set_title("Comparison of $f(x)$ and $f'(x)$")
axes.set_ylim((-2,2))
axes.set_xlim((0,2))
axes.plot(x, numpy.zeros(x.shape), 'k--')

x_k = 0.3
axes.plot([x_k, x_k], [0.0, f(x_k)], 'ko')
axes.plot([x_k, x_k], [0.0, f(x_k)], 'k--')
axes.plot(x, f_prime(x_k) * (x - x_k) + f(x_k), 'k')

x_k = x_k - f(x_k) / f_prime(x_k)
axes.plot([x_k, x_k], [0.0, f(x_k)], 'ko')
axes.plot([x_k, x_k], [0.0, f(x_k)], 'k--')

plt.show()
```

```python
x = numpy.linspace(0, 2, 1000)
f = lambda x: numpy.sin(2.0 * numpy.pi * x)
x_kp = lambda x: x - 1.0 / (2.0 * numpy.pi) * numpy.tan(2.0 * numpy.pi * x)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, f(x),'b')
axes.plot(x, x_kp(x), 'r')
axes.set_xlabel("x")
axes.set_ylabel("y")
axes.set_title("Comparison of $f(x)$ and $f'(x)$")
axes.set_ylim((-2,2))
axes.set_xlim((0,2))
axes.plot(x, numpy.zeros(x.shape), 'k--')

plt.show()
```

### Basins of Attraction

Given a point $x_0$ can we determine if Newton-Raphson converges?

A *basin of attraction* $X$ for Newton's methods is defined as the set such that $\forall x \in X$ Newton iterations converges.  Unfortunately this is far from a trivial thing to determine and even for simple functions can lead to regions that are fractal.

Plotted below are two fairly simple equations which demonstrate the problem:
1. $f(x) = x^3 - 1$
2. Kepler's equation $\theta - e \sin \theta = M$

```python
f = lambda x: x**3 - 1
f_prime = lambda x: 3 * x**2

N = 1001
x = numpy.linspace(-2, 2, N)
X, Y = numpy.meshgrid(x, x)
R = X + 1j * Y

for i in range(30):
    R = R - f(R) / f_prime(R)

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)
fig.set_figheight(fig.get_figheight() * 2)
axes = fig.add_subplot(1, 1, 1, aspect='equal')
axes.contour(X, Y, R)
axes.set_xlabel("Real")
axes.set_ylabel("Imaginary")
axes.set_title("Basin of Attraction for $f(x) = x^3 - 1$")
```

```python
def f(theta, e=0.083, M=1):
    return theta - e * numpy.sin(theta) - M
def f_prime(theta, e=0.083):
    return 1 - e * numpy.cos(theta)

N = 1001
x = numpy.linspace(-30.5, -29.5, N)
y = numpy.linspace(-17.5, -16.5, N)
X, Y = numpy.meshgrid(x, y)
R = X + 1j * Y

for i in range(30):
    R = R - f(R) / f_prime(R)

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)
fig.set_figheight(fig.get_figheight() * 2)
axes = fig.add_subplot(1, 1, 1, aspect='equal')
axes.contour(X, Y, R)
axes.set_xlabel("Real")
axes.set_ylabel("Imaginary")
axes.set_title("Basin of Attraction for $f(x) = x - e \sin x - M$")
```

#### Other Issues

Need to supply both $f(x)$ and $f'(x)$, could be expensive

Example:  FTV equation $f(r) = A - \frac{m P}{r} \left[ \left(1 + \frac{r}{m} \right )^{m n} - 1\right]$

Can use symbolic differentiation (`sympy`)

### Secant Methods

Is there a method with the convergence of Newton's method but without the extra derivatives?  What way would you modify Newton's method so that you would not need $f'(x)$?

Given $x_k$ and $x_{k-1}$ represent the derivative as the approximation

$$f'(x) \approx \frac{f(x_k) - f(x_{k-1})}{x_k - x_{k-1}}$$

Combining this with the Newton approach leads to

$$x_{k+1} = x_k - \frac{f(x_k) (x_k - x_{k-1}) }{f(x_k) - f(x_{k-1})}$$

This leads to superlinear convergence and not quite quadratic as the exponent on the convergence is $\approx 1.7$.

Alternative interpretation, fit a line through two points and see where they intersect the x-axis.

$$(x_k, f(x_k)) ~~~~~ (x_{k-1}, f(x_{k-1})$$

$$y = \frac{f(x_k) - f(x_{k-1})}{x_k - x_{k-1}} (x - x_k) + b$$

$$b = f(x_{k-1}) - \frac{f(x_k) - f(x_{k-1})}{x_k - x_{k-1}} (x_{k-1} - x_k)$$

$$ y = \frac{f(x_k) - f(x_{k-1})}{x_k - x_{k-1}} (x - x_k) + f(x_k)$$

Now solve for $x_{k+1}$ which is where the line intersects the x-axies ($y=0$)

$$0 = \frac{f(x_k) - f(x_{k-1})}{x_k - x_{k-1}} (x_{k+1} - x_k) + f(x_k)$$

$$x_{k+1} = x_k - \frac{f(x_k)  (x_k - x_{k-1})}{f(x_k) - f(x_{k-1})}$$

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.11, 100)
f = lambda r, A=A, m=m, P=P, n=n: \
        A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)

# Initial guess
x_k = 0.07
x_km = 0.06

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')

axes.plot(x_k, 0.0, 'ko')
axes.plot(x_k, f(x_k), 'ko')
axes.plot([x_k, x_k], [0.0, f(x_k)], 'k--')
axes.plot(x_km, 0.0, 'ko')
axes.plot(x_km, f(x_km), 'ko')
axes.plot([x_km, x_km], [0.0, f(x_km)], 'k--')

axes.plot(r, (f(x_k) - f(x_km)) / (x_k - x_km) * (r - x_k) + f(x_k), 'k')
x_kp = x_k - (f(x_k) * (x_k - x_km) / (f(x_k) - f(x_km)))
axes.plot(x_kp, 0.0, 'ro')
axes.plot([x_kp, x_kp], [0.0, f(x_kp)], 'r--')
axes.plot(x_kp, f(x_kp), 'ro')

axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
axes.set_title("Secant Method")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

plt.show()
```

What would the algorithm look like for such a method?

#### Algorithm

Given $f(x)$, given bracket $[a,b]$, a `TOLERANCE`, and a `MAX_STEPS` (note we need two points to start).

1. Initialize $x_1 = a$, $x_2 = b$, $f_1 = f(x_1)$, and $f_2 = f(x_2)$
2. Loop until either `MAX_STEPS` is reached or `TOLERANCE` is achieved
   1. Calculate new update $x_{k+1}$ by update formula
   2. Check for convergence and break if reached
   3. Update parameters $x_1$, $x_2$, $f_1 = f(x_1)$ and $f_2(x_2)$

```python
P = 1500.0
m = 12
n = 20.0
A = 1e6
r = numpy.linspace(0.05, 0.11, 100)
f = lambda r, A=A, m=m, P=P, n=n: \
        A - m * P / r * ((1.0 + r / m)**(m * n) - 1.0)
f_prime = lambda r, A=A, m=m, P=P, n=n: \
            -P*m*n*(1.0 + r/m)**(m*n)/(r*(1.0 + r/m)) \
                + P*m*((1.0 + r/m)**(m*n) - 1.0)/r**2

# Algorithm parameters
MAX_STEPS = 50
TOLERANCE = 1e-4

# Initial bracket
x_k = 0.07
x_km = 0.06

# Setup figure to plot convergence
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(r, f(r), 'b')
axes.plot(r, numpy.zeros(r.shape),'r--')

for n in range(1, MAX_STEPS + 1):
    axes.plot(x_k, f(x_k), 'o')
    axes.text(x_k + 0.0025, f(x_k), n, fontsize="15")
    x_kp = x_k - f(x_k) * (x_k - x_km) / (f(x_k) - f(x_km))
    x_km = x_k
    x_k = x_kp
    print("Residual = ", numpy.abs(f(x_k)))
    if numpy.abs(f(x_k)) < TOLERANCE:
        break

if n == MAX_STEPS:
    print("Reached maximum number of steps!")
else:
    print("Success!")
    print("  x* = %s" % x_k)
    print("  f(x*) = %s" % f(x_k))
    print("  number of steps = %s" % n)

axes.set_xlabel("r (%)")
axes.set_ylabel("f(r)")
axes.set_title("Secant Method")
axes.ticklabel_format(axis='y', style='sci', scilimits=(-1,1))

plt.show()
```

#### Comments

 - Secant method as shown is equivalent to linear interpolation
 - Can use higher order interpolation for higher order secant methods
 - Convergence is not quite quadratic
 - Not guaranteed to converge
 - Do not preserve brackets
 - Almost as good as Newton's method if your initial guess is good.

### Hybrid Methods

Combine attributes of methods with others to make one great algorithm to rule them all (not really)

#### Goals
1. Robustness:  Given a bracket $[a,b]$, maintain bracket
1. Efficiency:  Use superlinear convergent methods when possible

#### Options
 - Methods requiring $f'(x)$
   - NewtSafe (RootSafe, Numerical Recipes)
   - Newton's Method within a bracket, Bisection otherwise
 - Methods not requiring $f'(x)$
   - Brent's Algorithm (zbrent, Numerical Recipes)
     - Combination of bisection, secant and inverse quadratic interpolation
   - `scipy.optimize` package

## Optimization (finding extrema)

I want to find the extrema of a function $f(x)$ on a given interval $[a,b]$.

A few approaches:
 - Bracketing Algorithms:  Golden-Section Search (linear)
 - Interpolation Algorithms: Repeated parabolic interpolation
 - Hybrid Algorithms

### Bracketing Algorithm (Golden Section Search)

Given $f(x) \in C[x_0,x_3]$ that is convex (concave) over an interval $x \in [x_0,x_3]$ reduce the interval size until it brackets the minimum (maximum).

Note that we no longer have the $x=0$ help we had before so bracketing and doing bisection is a bit more tricky in this case.  In particular choosing your initial bracket is important!

#### Bracket Picking

Say we start with a bracket $[x_0, x_3]$ and pick to new points $x_1 < x_2 \in [x_0, x_3]$.  We want to pick a new bracket that guarantees that the extrema exists in it.  We then can pick this new bracket with the following rules:
 - If $f(x_1) < f(x_2)$ then we know the minimum is between $x_0$ and $x_2$.
 - If $f(x_1) > f(x_2)$ then we know the minimum is between $x_1$ and $x_3$.

```python
f = lambda x: x**2

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)
fig.set_figheight(fig.get_figheight() * 2)

search_points = [-1.0, -0.5, 0.75, 1.0]
axes = fig.add_subplot(2, 2, 1)
x = numpy.linspace(search_points[0] - 0.1, search_points[-1] + 0.1, 100)
axes.plot(x, f(x), 'b')
for (i, point) in enumerate(search_points):
    axes.plot(point, f(point),'or')
    axes.text(point + 0.05, f(point), str(i))
axes.plot(0, 0, 'sk')
axes.set_xlim((search_points[0] - 0.1, search_points[-1] + 0.1))
axes.set_title("$f(x_1) < f(x_2) \Rightarrow [x_0, x_2]$")

search_points = [-1.0, -0.75, 0.5, 1.0]
axes = fig.add_subplot(2, 2, 2)
x = numpy.linspace(search_points[0] - 0.1, search_points[-1] + 0.1, 100)
axes.plot(x, f(x), 'b')
for (i, point) in enumerate(search_points):
    axes.plot(point, f(point),'or')
    axes.text(point + 0.05, f(point), str(i))
axes.plot(0, 0, 'sk')
axes.set_xlim((search_points[0] - 0.1, search_points[-1] + 0.1))
axes.set_title("$f(x_1) > f(x_2) \Rightarrow [x_1, x_3]$")

search_points = [-1.0, 0.25, 0.75, 1.0]
axes = fig.add_subplot(2, 2, 3)
x = numpy.linspace(search_points[0] - 0.1, search_points[-1] + 0.1, 100)
axes.plot(x, f(x), 'b')
for (i, point) in enumerate(search_points):
    axes.plot(point, f(point),'or')
    axes.text(point + 0.05, f(point), str(i))
axes.plot(0, 0, 'sk')
axes.set_xlim((search_points[0] - 0.1, search_points[-1] + 0.1))
axes.set_title("$f(x_1) < f(x_2) \Rightarrow [x_0, x_2]$")

search_points = [-1.0, -0.75, -0.25, 1.0]
axes = fig.add_subplot(2, 2, 4)
x = numpy.linspace(search_points[0] - 0.1, search_points[-1] + 0.1, 100)
axes.plot(x, f(x), 'b')
for (i, point) in enumerate(search_points):
    axes.plot(point, f(point),'or')
    axes.text(point + 0.05, f(point), str(i))
axes.plot(0, 0, 'sk')
axes.set_xlim((search_points[0] - 0.1, search_points[-1] + 0.1))
axes.set_title("$f(x_1) > f(x_2) \Rightarrow [x_1, x_3]$")

plt.show()
```

#### Picking Brackets and Points

Again say we have a bracket $[x_0,x_3]$ and suppose we have two new search points $x_1$ and $x_2$ that separates $[x_0,x_3]$ into two new overlapping brackets.

Define
$$\begin{aligned}
    a &= x_1 - x_0, \\
    b &= x_3 - x_1,\\
    c &= x_2 - x_1 \quad \text{and} \\
    d &= x_3 - x_2.
\end{aligned}$$

For **Golden Section Search** we require two conditions:
 - The two new possible brackets are of equal length.  If we pick the left bracket $[x_0, x_2]$ then
 $$
     a+c = b
 $$
 and the right bracket $[x_1, x_3]$
 $$
     d + c = b.
 $$
 - The distances between subsequent triplets is proportional.

```python
f = lambda x: (x - 0.25)**2 + 0.5
phi = (1.0 + numpy.sqrt(5.0)) / 2.0

x = [-1.0, None, None, 1.0]
x[1] = x[3] - 1.0 / phi * (x[3] - x[0])
x[2] = x[0] + 1.0 / phi * (x[3] - x[0])

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)
axes = []
axes.append(fig.add_subplot(1, 2, 1))
axes.append(fig.add_subplot(1, 2, 2))
t = numpy.linspace(-2.0, 2.0, 100)
for i in range(2):
    axes[i].plot(t, f(t), 'k')

    # First set of intervals
    axes[i].plot([x[0], x[2]], [0.0, 0.0], 'g')
    axes[i].plot([x[1], x[3]], [-0.2, -0.2], 'r')
    axes[i].plot([x[0], x[0]], [0.0, f(x[0])], 'g--')
    axes[i].plot([x[2], x[2]], [0.0, f(x[2])], 'g--')
    axes[i].plot([x[1], x[1]], [-0.2, f(x[2])], 'r--')
    axes[i].plot([x[3], x[3]], [-0.2, f(x[3])], 'r--')
    for (n, point) in enumerate(x):
        axes[i].plot(point, f(point), 'ok')
        axes[i].text(point, f(point)+0.1, n, fontsize='15')

    axes[i].set_xlim((search_points[0] - 0.1, search_points[-1] + 0.1))
    axes[i].set_ylim((-1.0, 3.0))

# Left new interval
x_new = [x[0], None, x[1], x[2]]
x_new[1] = 1.0 / phi * (x[1] - x[0]) + x[0]

axes[0].plot([x_new[0], x_new[2]], [1.5, 1.5], 'b')
axes[0].plot([x_new[1], x_new[3]], [1.75, 1.75], 'c')
axes[0].plot([x_new[0], x_new[0]], [1.5, f(x_new[0])], 'b--')
axes[0].plot([x_new[2], x_new[2]], [1.5, f(x_new[2])], 'b--')
axes[0].plot([x_new[1], x_new[1]], [1.75, f(x_new[1])], 'c--')
axes[0].plot([x_new[3], x_new[3]], [1.75, f(x_new[3])], 'c--')
axes[0].plot(x_new[1], f(x_new[1]), 'ko')
axes[0].text(x_new[1] + 0.05, f(x_new[1]) + 0.1, "*", fontsize='15')

# Right new interval
x_new = [x[1], x[2], None, x[3]]
x_new[2] = (x[2] - x[1]) / phi + x[2]

axes[1].plot([x_new[0], x_new[2]], [1.25, 1.25], 'b')
axes[1].plot([x_new[1], x_new[3]], [1.5, 1.5], 'c')
axes[1].plot([x_new[0], x_new[0]], [1.25, f(x_new[0])], 'b--')
axes[1].plot([x_new[2], x_new[2]], [1.25, f(x_new[2])], 'b--')
axes[1].plot([x_new[1], x_new[1]], [1.5, f(x_new[2])], 'c--')
axes[1].plot([x_new[3], x_new[3]], [1.5, f(x_new[3])], 'c--')
axes[1].plot(x_new[2], f(x_new[2]), 'ko')
axes[1].text(x_new[2] + 0.05, f(x_new[2]) + 0.1, "*", fontsize='15')

plt.show()
```

The first rule implies:
$$\begin{aligned}
    a + c &= b \\
    x_1 - x_0 + x_2 - x_1 &= x_3 - x_1 \\
    x_2 - x_0 &= x_3 - x_1.
\end{aligned}$$

Assume that this allows us to pick $x_2$ (we need to figure out how to choose $x_1$).  We then know
$$
    x_2 = x_3 - x_1 + x_0.
$$

Subsequent proportionality implies that the distances between the 4 points at one iteration is proportional to the next.  Since we have two choices for our new interval we write down many proportionality constraints however let us focus on the two defined by the distances $a$, $b$, and $c$.

If $f(x_1) < f(x_2)$ then we choose $(x_0, x_1, x_2)$ as our new triplet meaning
$$
    \frac{a}{b} = \frac{c}{a}
$$

If $f(x_1) > f(x_2)$ then we choose $(x_1, x_2, x_3)$ as our new triplet meaning
$$
    \frac{a}{b} = \frac{c}{b-c}
$$

Using these relations we can solve for the ratio $b / a$ via the following.  Take
$$
    \frac{a}{b} = \frac{c}{a} \quad \text{and} \quad \frac{a}{b} = \frac{c}{b-c}
$$
and eliminate $c$ to find
$$\begin{aligned}
    c &= \frac{a^2}{b} \Rightarrow  \\
    \frac{a}{b} &= \frac{a^2}{b^2-a^2} \\
    ab^2 - a^3 &= a^2 b \\
    \frac{b^2}{a^2} - \frac{b}{a} - 1 &= 0
\end{aligned}$$
whose solution is
$$
    \frac{b}{a} = \frac{1 \pm \sqrt{5}}{2} = \varphi
$$
where $\varphi$ is the well known "golden ratio" (note that there are two values here, the most common definition of $\varphi$ uses the $+$ branch but in fact you can use either depending on the application).

Back to the problem at hand, we now need to pick our new set of points.  Note that we only need one new point as the the other three are left-overs from the previous iteration.  Let us concentrate on the case where the extrema is between $[x_0, x_2]$.  Denote the new bracket values with $\hat{\quad}$ and identify
$$
    \hat{x_0} = x_0, \quad \hat{x_2} = x_1, \quad \text{and} \quad \hat{x_3} = x_2.
$$
In this case we need to find $\hat{x_1}$, in that case use the subsequent intervals $a$ and $\hat{a_~}$ and equate
$$
    \varphi \hat{a~} = a \Rightarrow \varphi (\hat{x_1} - \hat{x_0}) = x_1 - x_0
$$
which in terms of the previous values can be solved for $\hat{x_1}$ to lead to
$$
    \hat{x_1} = \frac{x_1 - x_0}{\varphi} + x_0
$$

In the alternative case we have the bracket $[x_1, x_3]$ and
$$
    \hat{x_0} = x_1, \quad \hat{x_1} = x_2, \quad \text{and} \quad \hat{x_3} = x_3
$$
where we now need to find $\hat{x_2}$.  Instead of using $\hat{a~}$ we can use $\hat{b~}$ and the relationship
$$
    \varphi \hat{c~} = c \Rightarrow \varphi (\hat{x_2} - \hat{x_1}) = x_2 - x_1
$$
which again can be manipulated to lead to the value of $\hat{x_2}$ as
$$
    \hat{x_2} = \frac{x_2 - x_1}{\varphi} + x_0.
$$

#### Algorithm
1. Initialize bracket $[x_0,x_3]$
1. Initialize points $x_1 = x_3 - \frac{1}{\varphi} \cdot (x_3 - x_0)$ and $x_2 = x_0 + \frac{1}{\varphi} \cdot (x_3 - x_0)$
1. Loop
   1. Evaluate $f_1$ and $f_2$
   1. If $f_1 < f_2$ then we pick the left interval for the next iteration
   1. and otherwise pick the right interval
   1. Check size of bracket for convergence $x_3 - x_0 <$ `TOLERANCE`

```python
# New Test Function!
def f(t):
    """Simple function for minimization demos"""
    return -3.0 * numpy.exp(-(t - 0.3)**2 / (0.1)**2) \
           +      numpy.exp(-(t - 0.6)**2 / (0.2)**2) \
           +      numpy.exp(-(t - 1.0)**2 / (0.2)**2) \
           +      numpy.sin(t)                        \
           -      2.0

t = numpy.linspace(0, 2, 200)
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(t, f(t))
axes.set_xlabel("t (days)")
axes.set_ylabel("People (N)")
axes.set_title("Decrease in Population due to SPAM Poisoning")
axes.set_xlim((0.0, 2.0))

plt.show()
```

```python
def f(t):
    """Simple function for minimization demos"""
    return -3.0 * numpy.exp(-(t - 0.3)**2 / (0.1)**2) \
           +      numpy.exp(-(t - 0.6)**2 / (0.2)**2) \
           +      numpy.exp(-(t - 1.0)**2 / (0.2)**2) \
           +      numpy.sin(t)                        \
           -      2.0

phi = (1.0 + numpy.sqrt(5.0)) / 2.0

TOLERANCE = 1e-4
MAX_STEPS = 100

x = [0.2, None, None, 0.5]
x[1] = x[3] - 1.0 / phi * (x[3] - x[0])
x[2] = x[0] + 1.0 / phi * (x[3] - x[0])

t = numpy.linspace(0, 2, 200)
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(t, f(t))
axes.set_xlabel("t (days)")
axes.set_ylabel("People (N)")
axes.set_title("Decrease in Population due to SPAM Poisoning")

success = False
for n in range(1, MAX_STEPS + 1):
    axes.plot(x[0], f(x[0]),'ko')
    axes.plot(x[3], f(x[3]),'ko')
    f_1 = f(x[1])
    f_2 = f(x[2])

    if f_1 < f_2:
        # Pick the left bracket
        x_new = [x[0], None, x[1], x[2]]
        x_new[1] = 1.0 / phi * (x[1] - x[0]) + x[0]
    else:
        # Pick the right bracket
        x_new = [x[1], x[2], None, x[3]]
        x_new[2] = (x[2] - x[1]) / phi + x[2]
    x = x_new

    if numpy.abs(x[3] - x[0]) < TOLERANCE:
        success = True
        break

if success:
    print("Success!")
    print("  t* = %s" % str((x[3] + x[0]) / 2.0))
    print("  f(t*) = %s" % f((x[3] + x[0]) / 2.0))
    print("  number of steps = %s" % n)
else:
    print("Reached maximum number of steps!")

plt.show()
```

### Interpolation Approach

Successive parabolic interpolation - similar to secant method

Basic idea:  Fit polynomial to function using three points, find it's minima, and guess new points based on that minima

1. What do we need to fit a polynomial $p_n(x)$ of degree $n \geq 2$?

2. How do we construct the polynomial $p_2(x)$?

3. Once we have constructed $p_2(x)$ how would we find the minimum?

#### Algorithm

Given $f(x)$ and $[x_0,x_1]$ - Note that unlike a bracket these will be a sequence of better approximations to the minimum.
1. Initialize $x = [x_0, x_1, (x_0+x_1)/2]$
1. Loop
   1. Evaluate function $f(x)$
   1. Use a polynomial fit to the function:

      $$p(x) = p_0 x^2 + p_1 x + p_2$$

   1. Calculate the minimum:

      $$p'(x) = 2 p_0 x + p_1 = 0 \quad \Rightarrow \quad x^\ast = -p_1 / (2 p_0)$$

   1. New set of points $x = [x_1, (x_0+x_1)/2, x^\ast]$
   1. Check tolerance

```python
def f(t):
    """Simple function for minimization demos"""
    return -3.0 * numpy.exp(-(t - 0.3)**2 / (0.1)**2) \
           +      numpy.exp(-(t - 0.6)**2 / (0.2)**2) \
           +      numpy.exp(-(t - 1.0)**2 / (0.2)**2) \
           +      numpy.sin(t)                        \
           -      2.0

MAX_STEPS = 100
TOLERANCE = 1e-4

x = numpy.array([0.5, 0.2, (0.7) / 2.0])

t = numpy.linspace(0, 2, 200)
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(t, f(t))
axes.set_xlabel("t (days)")
axes.set_ylabel("People (N)")
axes.set_title("Decrease in Population due to SPAM Poisoning")
axes.plot(x[0], f(x[0]), 'ko')
axes.plot(x[1], f(x[1]), 'ko')

success = False
for n in range(1, MAX_STEPS + 1):
    axes.plot(x[2], f(x[2]), 'ko')
    poly = numpy.polyfit(x, f(x), 2)
    axes.plot(t, poly[0] * t**2 + poly[1] * t + poly[2], 'r--')
    x[0] = x[1]
    x[1] = x[2]
    x[2] = -poly[1] / (2.0 * poly[0])
    if numpy.abs(x[2] - x[1]) / numpy.abs(x[2]) < TOLERANCE:
        success = True
        break

if success:
    print("Success!")
    print("  t* = %s" % x[2])
    print("  f(t*) = %s" % f(x[2]))
    print("  number of steps = %s" % n)
else:
    print("Reached maximum number of steps!")

axes.set_ylim((-5, 0.0))
plt.show()
```

## Scipy Optimization

Scipy contains a lot of ways for optimization!

```python
import scipy.optimize as optimize
print(optimize.golden(f, brack=(0.2, 0.25, 0.5)))
```


```python
%matplotlib inline
```

# Image denoising by FFT

Denoise an image (:download:`../../../../data/moonlanding.png`) by
implementing a blur with an FFT.

Implements, via FFT, the following convolution:

\begin{align}f_1(t) = \int dt'\, K(t-t') f_0(t')\end{align}

\begin{align}\tilde{f}_1(\omega) = \tilde{K}(\omega) \tilde{f}_0(\omega)\end{align}

Read and plot the image
###########################################################

```python
import numpy as np
import matplotlib.pyplot as plt

im = plt.imread('../../../../data/moonlanding.png').astype(float)

plt.figure()
plt.imshow(im, plt.cm.gray)
plt.title('Original image')
```

Compute the 2d FFT of the input image
###########################################################

```python
from scipy import fftpack
im_fft = fftpack.fft2(im)

# Show the results

def plot_spectrum(im_fft):
    from matplotlib.colors import LogNorm
    # A logarithmic colormap
    plt.imshow(np.abs(im_fft), norm=LogNorm(vmin=5))
    plt.colorbar()

plt.figure()
plot_spectrum(im_fft)
plt.title('Fourier transform')
```

Filter in FFT
###########################################################

```python
# In the lines following, we'll make a copy of the original spectrum and
# truncate coefficients.

# Define the fraction of coefficients (in each direction) we keep
keep_fraction = 0.1

# Call ff a copy of the original transform. Numpy arrays have a copy
# method for this purpose.
im_fft2 = im_fft.copy()

# Set r and c to be the number of rows and columns of the array.
r, c = im_fft2.shape

# Set to zero all rows with indices between r*keep_fraction and
# r*(1-keep_fraction):
im_fft2[int(r*keep_fraction):int(r*(1-keep_fraction))] = 0

# Similarly with the columns:
im_fft2[:, int(c*keep_fraction):int(c*(1-keep_fraction))] = 0

plt.figure()
plot_spectrum(im_fft2)
plt.title('Filtered Spectrum')
```

Reconstruct the final image
###########################################################

```python
# Reconstruct the denoised image from the filtered spectrum, keep only the
# real part for display.
im_new = fftpack.ifft2(im_fft2).real

plt.figure()
plt.imshow(im_new, plt.cm.gray)
plt.title('Reconstructed Image')
```

Easier and better: :func:`scipy.ndimage.gaussian_filter`
###########################################################

 Implementing filtering directly with FFTs is tricky and time consuming.
 We can use the Gaussian filter from :mod:`scipy.ndimage`

```python
from scipy import ndimage
im_blur = ndimage.gaussian_filter(im, 4)

plt.figure()
plt.imshow(im_blur, plt.cm.gray)
plt.title('Blurred image')

plt.show()
```


# Linear and Logistic Regression

In this notebook, we'll outline the basic ideas of Supervised Learning through defining the simplest model in both Regression and Classification. First, let's build some datasets.

```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
from sklearn import datasets
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression

plt.style.use("fivethirtyeight")
```

```python
df_regression = pd.DataFrame({"x":[1,2,2,3,4,4,5,5], "y":[1,1,3,2,3,4,4,5]})
df_classification = pd.DataFrame({"x":[1,2,3,3,4,4,5,5.5], "y":[0,0,0,1,0,1,1,1]})
```

```python
def graph_plots():
    fig, ax = plt.subplots(1, 2, figsize=(13, 4))
    fig.subplots_adjust(wspace=1)

    ax[0].scatter(df_regression.x, df_regression.y, s=150)
    ax[0].set_title("Regression")
    ax[0].set_xlim([0,6])
    ax[0].set_ylim([0,6])
    ax[1].scatter(df_classification.x, df_classification.y, s=150)
    ax[1].set_title("Classification")
    ax[1].set_xlim([0,6])
    ax[1].set_ylim([-0.5, 1.5])
    return fig, ax

graph_plots()
plt.show()
```

In order to create functions for these datasets, we perform **Linear Regression** for the Regression dataset and **Logistic Regression** for the Classification datset.  In both functions, we use $x_i$ and $y_i$ for the contents of the $i^\text{th}$ row, and $\widehat{y}_i$ to denote our prediction for the $i^\text{th}$ response.

### Linear Regression
Here, we choose to fit the function

$$\widehat{y} = m \cdot x + b$$

So, for the $i^\text{th}$ variable, to make our prediction, we use $\widehat{y}_i = m \cdot x_i + b$.  Here $m$ and $b$ are called **parameters** of the function; they determine how our line fits our data.

### Logistic Regression
Here, we choose to fit the function

$$\widehat{y} = \dfrac{1}{1+e^{-(m \cdot x + b)}}$$

So, for the $i^\text{th}$ variable, to make our prediction, we use the appropriately rounded version of $\widehat{y}_i = \dfrac{1}{1+e^{-(m \cdot x_i + b)}}$.  Note that the values of this function are between 0 and 1, inclusive.  Thus, we interpret these values as the _probability_ that a value would be a one, and typically choose a cut off point (often, but not necessarily 50%) to round the values to 0 or 1.

## Cost functions
In order to fit these functions, we need to choose an appropriate parameters $m$ and $b$.  Here are two curves drawn for each function, with different qualities of fit:

```python
# Let's try some random values for `m` and `b`:
m_1, b_1 = 1.1, 0.9
m_2, b_2 = -2.9, 5.4

x = np.arange(0, 6, .1)
y_1_reg = m_1 * x + b_1
y_2_reg = m_2 * x + b_2

# Need to tweak the b's for demonstration purposes
y_1_cla = 1 - 1 / (1 + np.exp(-(m_1 * x + (b_1 - 3))))
y_2_cla = 1 - 1 / (1 + np.exp(-(m_2 * x + (b_2 + 3))))

fig, ax = graph_plots()
ax[0].plot(x, y_1_reg, color="red")
ax[0].plot(x, y_2_reg, color="blue")
ax[1].plot(x, y_1_cla, color="red")
ax[1].plot(x, y_2_cla, color="blue")
plt.show()
```

In the Regression example, it appears that the red line is an OK fit, and the blue line is bad.  In the Classification example, it appears that the blue line is an OK fit, and the red line is bad.  In order to choose the best $m$ and $b$, we define a **cost function** for both of these models.  One key point about cost functions is that **they are functions of the parameters of the model function**.

### Regression: Residual Sum of Squares
Given a dataset $(x, y)$ with $n$ rows, we define the cost function of a regression equation to be:
$$C(m, b) = \sum_{i=1}^n (y_i - \widehat{y}_i)^2$$
Note that there is _one term in this sum for each row in the dataset_.  This is the sum of the squares of  **residual** terms:

```python
m, b = 1.1, -0.8

y = m * x + b
for xi, yi in [(2, 3), (1, 1), (4, 3)]:
    plt.plot([xi, xi], [yi, m * xi + b], 'r')
plt.plot(x, y, 'k')
plt.scatter(df_regression.x, df_regression.y, s=150)
plt.title("Some residuals")
plt.xlim([0,6])
plt.ylim([0,6])
plt.show()
```

In order to make sure we fully understand how the cost function is generated, let's write it all the way out for our dataset:

$$
\begin{align}
C(m, b) &= \sum_{i=1}^n (y_i - \widehat{y}_i)^2\\
&= \sum_{i=1}^n (y_i - (m \cdot x_i + b))^2\\
&= (1 - (m \cdot 1 + b))^2 + (1 - (m \cdot 2 + b))^2 + (3 - (m \cdot 2 + b))^2 + (2 - (m \cdot 3 + b))^2 + (3 - (m \cdot 4 + b))^2 \\ &\ldots + (4 - (m \cdot 4 + b))^2 + (4 - (m \cdot 5 + b))^2 + (5 - (m \cdot 5 + b))^2.\\
\end{align}
$$
It's a bit of a mess, but notice that the cost function's form is dependent on the dataset.

### Classification: Cross-entropy
Given a dataset $(x, y)$ with $n$ rows, we define the cost function of a classification equation to be:
$$C(m, b) = -\sum_{i=1}^n y_i \ln(\widehat{y}_i) + (1 - y_i) \ln (1 - \widehat{y}_i)$$
A very important note is that $y_i$ is either 0 or 1, so again there will only ever be one term in the sum for each row in the dataset.  The intuition here is that the $y_i$ and $1-y_i$ terms select the appropriate choice of cost function, where the logarithm function measures a value close to 0 inside of it as bad, and a value close to 1 inside of it to be good.

The reader is encouraged to expand the cost function all the way out for the given dataset.
## Fitting: Gradient Descent
One key aspect of these cost functions is that they are differentiable, as in, we may take their derivative.  If they were nice functions, one could simply perform some standard calculus optimization technique.  After all, to best fit the model's function, we need to minimize the cost function, _i.e._ choose an $m$ and $b$ which minimize the function.  In practice, however, these are not nice functions to solve, so we need a better solution. (_Side note_: this is a slight lie.)

In general, we perform the following algorithm:
1. Pick reasonable, random values for $m$ and $b$.
2. Compute the gradient of $C$ at the current $m$ and $b$, denoted by $\nabla C (m, b)=\left< m', b'\right>$.
3. Here, we need a fact from calculus: _the gradient always points straight toward the direction of steepest increase_.  Okay, so to minimize the function, step directly away from the gradient.  If $\alpha$ is a small number, we do:
$$
\begin{align}
m &\mapsto m - \alpha \cdot m' \\
b &\mapsto b - \alpha \cdot b' \\
\end{align}
$$
The number $\alpha$ is often called the _learning rate_ of the algorithm, and taking a default of something like 0.01 usually works well.
4. Return to step 2 until the new gradient is very tiny, so that you aren't going to be making much progress anymore.

In practice, this tends to find a reasonable optimum for our fit.  Understanding the details of this is part of the Gradients Project.  In conclusion, we fit using Scikit-learn's built-in Linear and Logistic Regression functions to fit and graph our curves.

```python
# I found that I got better results from this function if I scaled the data to have mean 0 and standard deviation 1.
def scale_data(x):
    return (x - df_classification.x.mean()) / df_classification.x.std()

lin_model = LinearRegression().fit(df_regression[['x']], df_regression['y'])
log_model = LogisticRegression().fit(scale_data(df_classification[['x']]), df_classification['y'])

m_reg = lin_model.coef_[0]
b_reg = lin_model.intercept_

m_cla = log_model.coef_[0]
b_cla = log_model.intercept_

x = np.arange(0, 6, .1)
y_reg = m_reg * x + b_reg
y_cla = 1 - 1 / (1 + np.exp(m_cla * scale_data(x) + b_cla))

fig, ax = graph_plots()
ax[0].plot(x, y_reg, color="blue")
ax[1].plot(x, y_cla, color="blue")
plt.show()
```

lessons/Linear and Logistic Regression.ipynb


# Branched System 3-Reservoir Example

Consider the branched system shown in below.  In this example the friction factor is assumed constant for simplicity, but in practice would vary during the solution computations.

The hydraulics question is what is the discharge in each pipe and what is the total head at the junction (notice we don't know the junction elevation in this example --- if the elevation were specified, we could also find the pressure head).

## Solution Approach

First populate the four equations with the appropriate numerical values.

\begin{equation}
70 = h_D + (0.015)\frac{5000}{0.6}\frac{V_{AD}|V_{AD}|}{2(9.8)}
\end{equation}

\begin{equation}
100 = h_D + (0.015)\frac{3000}{0.8}\frac{V_{BD}|V_{BD}|}{2(9.8)}
\end{equation}

\begin{equation}
-80 = - h_D  + (0.015)\frac{4000}{1.2}\frac{V_{DC}|V_{DC}|}{2(9.8)}
\end{equation}

\begin{equation}
\frac{\pi(0.6)^2}{4} V_{AD}+\frac{\pi(0.8)^2}{4} V_{BD} = \frac{\pi(1.2)^2}{4} V_{DC}
\end{equation}

Next, compute all the constants, and organize the 4 equations into a system of simultaneous equations

\begin{equation}
\begin{matrix}
h_D & 6.377 V_{AD}|V_{AD}| & 0 & 0  & = 70\\
h_D & 0 & 2.869 V_{BD}|V_{BD}| & 0  & = 100\\
h_D & 0 & 0 & - 2.551 V_{DC}|V_{DC}|  & = 80\\
0 & 0.2827 V_{AD} & 0.5026 V_{BD}& -1.1309 V_{DC}  & = 0\\
\end{matrix}
\end{equation}

Because the system is non-linear we need to employ some appropriate method, herein we will use a Quasi-Newton method with numerical approximations to derivatives.

First a more conventional vector-matrix structure

\begin{gather}
\begin{pmatrix}
1 & 6.377 |V_{AD}| & 0 & 0 \\
1 & 0 & 2.869 |V_{BD}| & 0 \\
1 & 0 & 0 & - 2.551 |V_{DC}| \\
0 & 0.2827  & 0.5026 & -1.1309 \\
\end{pmatrix}
\bullet
\begin{pmatrix}
 h_D \\
V_{AD} \\
V_{BD} \\
V_{DC} \\
\end{pmatrix}
-
\begin{pmatrix}
70 \\
100 \\
 80 \\
0 \\
\end{pmatrix}
=
\begin{pmatrix}
0 \\
0 \\
0 \\
0 \\
\end{pmatrix}
\end{gather}

## Apply the multiple-variable Extension of Newton’s Method

The solution can use the methods directly from ENGR-1330 and similar courses, here we just implement the methods.  A very brief explaination is in the readings, but multi-variable Newton's method is usually taught in calculus.

Here we will define some support modules `linearsolver()` and `vector_matrix_lib` which are just python source code containing prototype functions which are embedded into the notebook, but ordinarily would save them in separate file and use `import`.

```python
# SolveLinearSystem.py
# Code to read A and b
# Then solve Ax = b for x by Gaussian elimination with back substitution
##########
def linearsolver(A,b):
    n = len(A)
# M = A  #this is object to object equivalence
# copy A into M element by element
    M=[[0.0 for jcol in range(n)]for irow in range(n)]
    for irow in range(n):
        for jcol in range(n):
            M[irow][jcol]=A[irow][jcol]
    i = 0
    for x in M:
     x.append(b[i])
     i += 1
    for k in range(n):
     for i in range(k,n):
       if abs(M[i][k]) > abs(M[k][k]):
          M[k], M[i] = M[i],M[k]
       else:
          pass
     for j in range(k+1,n):
         q = float(M[j][k]) / M[k][k]
         for m in range(k, n+1):
            M[j][m] -=  q * M[k][m]
    x = [0 for i in range(n)]
    x[n-1] =float(M[n-1][n])/M[n-1][n-1]
    for i in range (n-1,-1,-1):
      z = 0
      for j in range(i+1,n):
          z = z  + float(M[i][j])*x[j]
      x[i] = float(M[i][n] - z)/M[i][i]
#    print (x)
    return(x)
```

```python
# vector_matrix_lib.py
# useful linear algebra tools
import math   # This will import math module

def writeM(M,ir,jc,label):
    print ("------",label,"------")
    for i in range(0,ir,1):
        print (M[i][0:jc])
    print ("-----------------------------")
    #return

def writeV(V,ir,label):
    print ("------",label,"------")
    for i in range(0,ir,1):
        print (V[i])
    print ("-----------------------------")
    #return

def mmmult(amatrix,bmatrix,rowNumA,colNumA,rowNumB,colNumB):
    AB =[[0.0 for j in range(colNumB)] for i in range(rowNumA)]
    for i in range(0,rowNumA):
        for j in range(0,colNumB):
            for k in range(0,colNumA):
                AB[i][j]=AB[i][j]+amatrix[i][k]*bmatrix[k][j]
    return(AB)

def mvmult(amatrix,xvector,rowNumA,colNumA):
    bvector=[0.0 for i in range(rowNumA)]
    for i in range(0,rowNumA):
        for j in range(0,1):
            for k in range(0,colNumA):
                bvector[i]=bvector[i]+amatrix[i][k]*xvector[k]
    return(bvector)

def vvadd(avector,bvector,length):
    aplusb=[0.0 for i in range(length)]
    for i in range(length):
        aplusb[i] = avector[i] + bvector[i]
    return(aplusb)

def vvsub(avector,bvector,length):
    aminusb=[0.0 for i in range(length)]
    for i in range(length):
        aminusb[i] = avector[i] - bvector[i]
    return(aminusb)

def vdotv(avector,bvector,length):
    adotb=0.0
    for i in range(length):
        adotb=adotb+avector[i]*bvector[i]
    return(adotb)
```

Next we define the four equations, as well a a vector valued function `func` and its Jacobian `jacob` as below.

```python
#################################################################
# Newton Solver Example -- Numerical Derivatives                #
#################################################################
import math   # This will import math module from python distribution
#from LinearSolverPivot import linearsolver # This will import our solver module
#from vector_matrix_lib import writeM,writeV,vdotv,vvsub # This will import our vector functions

def eq1(x1,x2,x3,x4):
    eq1 = 1*x1 + 6.377*x2 +0*x3 + 0*x4 - 70
    return(eq1)
def eq2(x1,x2,x3,x4):
    eq2 = 1*x1 + 0*x2 +2.869*x3 + 0*x4 - 100
    return(eq2)
def eq3(x1,x2,x3,x4):
    eq3 = 1*x1 + 0*x2 +0*x3 + -2.551*x4 -80
    return(eq3)
def eq4(x1,x2,x3,x4):
    eq4 = 0*x1 + 0.2827*x2 +0.5026*x3 + -1.1309*x4 - 0
    return(eq4)
##############################################################
def func(x1,x2,x3,x4):
    func  = [0.0 for i in range(4)] # null list
    # build the function
    func[0] = eq1(x1,x2,x3,x4)
    func[1] = eq2(x1,x2,x3,x4)
    func[2] = eq3(x1,x2,x3,x4)
    func[3] = eq4(x1,x2,x3,x4)
    return(func)

def jacob(x1,x2,x3,x4):
    jacob = [[0.0 for j in range(4)] for i in range(4)] # constructed list
    #build the  jacobian
    jacob[0][0]=1
    jacob[0][1]=6.377*x2
    jacob[0][2]=0
    jacob[0][3]=0
    jacob[1][0]=1
    jacob[1][1]=0
    jacob[1][2]=2.869*x3
    jacob[1][3]=0
    jacob[2][0]=1
    jacob[2][1]=0
    jacob[2][2]=0
    jacob[2][3]=-2.551*x4
    jacob[3][0]=0
    jacob[3][1]=0.2827
    jacob[3][2]=0.5026
    jacob[3][3]=-1.1309
    return(jacob)
```

Next we create vectors to store values, and supply initial guesses to the system, and echo the inputs.

```python
deltax  = [0.0 for i in range(4)] # null list
xguess  = [0.0 for i in range(4)] # null list
myfunc  = [0.0 for i in range(4)] # null list
myjacob = [[0.0 for j in range(4)] for i in range(4)] # constructed list
# supply initial guess
xguess[0] = float(input("Value for x1 : "))
xguess[1] = float(input("Value for x2 : "))
xguess[2] = float(input("Value for x3 : "))
xguess[3] = float(input("Value for x4 : "))
# build the initial function
myfunc = func(xguess[0],xguess[1],xguess[2],xguess[3])
#build the initial jacobian
myjacob=jacob(xguess[0],xguess[1],xguess[2],xguess[3])
#write initial results
writeV(xguess,4,"Initial X vector ")
writeV(myfunc,4,"Initial FUNC vector ")
writeM(myjacob,4,4,"Initial Jacobian ")
# solver parameters
tolerancef = 1.0e-9
tolerancex = 1.0e-9
# Newton-Raphson
```

    Value for x1 :  1
    Value for x2 :  1
    Value for x3 :  1
    Value for x4 :  1

    ------ Initial X vector  ------
    1.0
    1.0
    1.0
    1.0
    -----------------------------
    ------ Initial FUNC vector  ------
    -62.623
    -96.131
    -81.551
    -0.3455999999999999
    -----------------------------
    ------ Initial Jacobian  ------
    [1, 6.377, 0, 0]
    [1, 0, 2.869, 0]
    [1, 0, 0, -2.551]
    [0, 0.2827, 0.5026, -1.1309]
    -----------------------------

Now we apply the algorithm a few times, here the count is set to 10. So eneter the loop, test for stopping, then update.

```python
for iteration in range(24):
    myfunc = func(xguess[0],xguess[1],xguess[2],xguess[3])
    testf = vdotv(myfunc,myfunc,4)
    if testf <= tolerancef :
        print("f(x) close to zero\n test value : ", testf)
        break
    myjacob=jacob(xguess[0],xguess[1],xguess[2],xguess[3])
    deltax=linearsolver(myjacob,myfunc)
    testx = vdotv(deltax,deltax,4)
    if testx <= tolerancex :
        print("solution change small\n test value : ", testx)
        break
    xguess=vvsub(xguess,deltax,4)
##    print("iteration : ",iteration)
##    writeV(xguess,2,"Current X vector ")
##    writeV(myfunc,2,"Current FUNC vector ")
print("Exiting Iteration : ",iteration)
writeV(xguess,4,"Exiting X vector ")
writeV(myfunc,4,"Exiting FUNC vector using Finite-Differences")
```

    f(x) close to zero
     test value :  2.0273725264180003e-28
    Exiting Iteration :  1
    ------ Exiting X vector  ------
    84.61708956275575
    -2.292157685864162
    5.361767318663032
    1.8099135879089594
    -----------------------------
    ------ Exiting FUNC vector using Finite-Differences ------
    -1.4210854715202004e-14
    0.0
    0.0
    -8.881784197001252e-16
    -----------------------------

Now tidy up the output, convert back to discharges (from velocities) and report results.

```python
print("Head at Junction : %.3f" % xguess[0], " feet ")
print("Discharge in Pipe AD: %.3f" % (0.2827 * xguess[1]), " cubic feet per second")
print("Discharge in Pipe BD: %.3f" % (0.5026 * xguess[2]), " cubic feet per second ")
print("Discharge in Pipe DC: %.3f" % (1.1309 * xguess[3]), " cubic feet per second ")
```

    Head at Junction : 84.617  feet
    Discharge in Pipe AD: -0.648  cubic feet per second
    Discharge in Pipe BD: 2.695  cubic feet per second
    Discharge in Pipe DC: 2.047  cubic feet per second

```python

```