```python
# import stuff from sympy
from sympy import *
import random
import numpy as np
# Visualization
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
# x, y, z, t = symbols('x y z t')
# k, m, n = symbols('k m n', integer=True)
# f, g, h = symbols('f g h', cls=Function)
```

```python
# THIS IS WRONG - SOMEHOW !!! DO NOT USE - gives transpose?
def rs_element(X, r, s, h):
    """
    Function that computes the r,s element of the matrix L for triangular kernel
    using the Nadaraya-Watson estimator, where \hat{Y} = LY

    X: input data predictors
    r: row r
    s: column s
    h: bandwidth
    """
    n = len(X)
    numerator = h - abs(X[r] - X[s])
    sum = 0
    for i in range(n):
        sum += abs(X[r] - X[i])
    denominator = n*h - sum
    print(denominator)
    return(numerator/denominator)
```

```python
def triangular_kernel(x, xi, h):
    """
    Function for triangular kernel
    """
    if np.abs(x-xi)<h:
        return 1 - np.abs(x-xi)/h
    else:
        return 0

def triangular_weights(X, i, h):
    """
    Function that computes the weights for the triangular kernel for a given X list.
    This is the same as the column i of projection matrix L.

    w_i(x) = K[(x-xi)/h]/sum{K[(x-xj)/h]}
    """
    result = []
    for x in X:
        numerator = triangular_kernel(x, X[i], h)
        denominator = 0
        for xj in X:
            denominator += triangular_kernel(x, xj, h)
        result.append(numerator/denominator)
    return(np.array(result))

def denominator_list(X, h):
    """
    Function that computes the common denominators in the matrix L for triangular kernel
    using the Nadaraya-Watson estimator, where \hat{Y} = LY

    X: input data predictors
    h: bandwidth
    """
    n = len(X)
    result = []
    for xj in X:
        result.append(n*h - np.sum(np.abs(X-xj)))
    return(np.array(result))
    #denominator = n*h - sum
    #print(denominator)
    #return(numerator/denominator)

def build_L_matrix(X, h):
    """
    Function that builds the smoothing L matrix for triangular kernel using
    the Nadaraya-Watson estimator, where \hat{Y} = LY

    X: input data predictors
    h: bandwidth
    """
    n = len(X)
    L = np.zeros((n,n))
    den_list = denominator_list(X, h)
    for s in range(n):
        L[:,s] = triangular_weights(X, s, h)
    return(L)
```

```python
X = np.array([3, 4.3, 6, 7, 9.1, 10.3])
h = 2
L = build_L_matrix(X, h)
print(L)
```

    [[0.74074074 0.25925926 0.         0.         0.         0.        ]
     [0.23333333 0.66666667 0.1        0.         0.         0.        ]
     [0.         0.09090909 0.60606061 0.3030303  0.         0.        ]
     [0.         0.         0.33333333 0.66666667 0.         0.        ]
     [0.         0.         0.         0.         0.71428571 0.28571429]
     [0.         0.         0.         0.         0.28571429 0.71428571]]

```python
# "pretty" print using sympy
Matrix(np.around(L, 3))
latex(Matrix(np.around(L, 3))) # for LaTeX output
```

    '\\left[\\begin{matrix}0.741 & 0.259 & 0.0 & 0.0 & 0.0 & 0.0\\\\0.233 & 0.667 & 0.1 & 0.0 & 0.0 & 0.0\\\\0.0 & 0.091 & 0.606 & 0.303 & 0.0 & 0.0\\\\0.0 & 0.0 & 0.333 & 0.667 & 0.0 & 0.0\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.714 & 0.286\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.286 & 0.714\\end{matrix}\\right]'

## 1.2  Computing prediction

```python
Y = np.array([0, 1, 2, 2, 4, 3])
```

```python
Y_hat = L@Y
```

```python
Y_hat
```

    array([0.25925926, 0.86666667, 1.90909091, 2.        , 3.71428571,
           3.28571429])

```python
# "pretty" print using sympy
Matrix(np.around(Y_hat, 3))
latex(Matrix(np.around(Y_hat, 3))) # for LaTeX output
```

    '\\left[\\begin{matrix}0.259\\\\0.867\\\\1.909\\\\2.0\\\\3.714\\\\3.286\\end{matrix}\\right]'

```python
# Plotting prediction
plt.figure(figsize=(10,8))
plt.plot(X, Y, '*', label="Observation")
plt.plot(X, Y_hat, 'x', label="Prediction")
plt.xlabel("X")
plt.ylabel("Y")
plt.legend()
```

## 1. 3 Computing MSE by LOOCV and GCV

The idea here is to compute the MSE by LOOCV in a "manual" way and then extend it using the Generalized Cross Validation method/formula.

$$
{\displaystyle \operatorname {MSE} ={\frac {1}{n}}\sum_{i=1}^{n}(Y_{i}-{\hat {Y_{i}}})^{2}.}
$$

```python
n = len(X)  # size of data
# choosing a random index to leave out
leave_i = random.randint(0, n-1)  # index to leave out

Y_loocv = np.delete(Y, leave_i)

# Computing model with new data
X_loocv = np.delete(X, leave_i)
n_loocv = len(X)-1
h = 2
L_loocv = build_L_matrix(X_loocv, h)
# prediction
Y_hat_loocv = L_loocv@Y_loocv

# performing MSE computation
print(np.sum((Y_loocv-Y_hat_loocv)**2))
MSE_loocv = np.sum((Y_loocv-Y_hat_loocv)**2)/n_loocv
```

    0.093257604099355

```python
MSE_loocv
```

    0.018651520819871002

```python
L_loocv
```

    array([[0.74074074, 0.25925926, 0.        , 0.        , 0.        ],
           [0.23333333, 0.66666667, 0.1       , 0.        , 0.        ],
           [0.        , 0.09090909, 0.60606061, 0.3030303 , 0.        ],
           [0.        , 0.        , 0.33333333, 0.66666667, 0.        ],
           [0.        , 0.        , 0.        , 0.        , 1.        ]])

```python
leave_i
```

    4

```python
np.sqrt(MSE_loocv)
```

    0.13657057084112595

BUT, there's another _shortcut_ to compute the MSE for the LOOCV case, as

$$
{\displaystyle \operatorname {MSE} ={\frac {1}{n}}\sum_{i=1}^{n}\left(\frac{Y_{i}-{\hat {Y_{i}}}}{1-h_{ii}}\right)^{2}.}
$$

Where $h_{ii}$ is the **leverage**, which is the diagonal element of the projection matrix $L$.

Let's see if this indeed gives the same result.

```python
# performing MSE computation
n = len(X)
MSE_loocv_2 = np.sum(((Y-Y_hat)/(1-L[leave_i, leave_i]))**2)/n
```

```python
print(MSE_loocv_2)
```

    0.5237342750361837

```python
MSE_loocv_2 - MSE_loocv
```

    0.5050827542163128

### GCV
The GCV is computed as
$$
{\displaystyle \operatorname {MSE_{GCV}} ={\frac {1}{n}}\sum_{i=1}^{n}\frac{\left(Y_{i}-{\hat {Y_{i}}}\right)^{2}}{1-v/n}.}
$$
Where $v=\text{Tr}(L)$

```python
v = L.trace()
MSE_gcv = (np.sum((Y-Y_hat)**2)/(1-v/n)**2)/n
```

```python
MSE_gcv
```

    0.4302881332603392

```python
MSE_loocv_2 - MSE_gcv
```

    0.2491508123441171

## 1.4
Trying to see $h$ dependance, is $h=2.8$ better?

```python
hlist = np.arange(1.5,3,0.01)
MSE_gcv_list = []
MSE_loocv_list = []
for hi in hlist:
    L = build_L_matrix(X, hi)
    Y_hat = L@Y
    v = L.trace()
    MSE_gcv_list.append((np.sum((Y-Y_hat)**2)/(1-v/n)**2)/n)
    leave_i = random.randint(0, n-1)  # index to leave out
    MSE_loocv_list.append(np.sum(((Y-Y_hat)/(1-L[leave_i, leave_i]))**2)/n)
# Plotting
plt.figure(figsize=(10,8))
plt.plot(hlist, MSE_gcv_list, label="GCV")
plt.plot(hlist, MSE_loocv_list, '-o', label="Leverage")
plt.xlabel("h")
plt.ylabel("MSE")
plt.legend()
```

According to this, and considering that using the leverage is very noisy, GCV tells that we get less error for a value of $h=2.0$ compared to $h=2.8$, even though that when calculating the error with the leverage formulat it gives better result for $h=2.8$ but it's very variant, so we don't trust that.

```python

```


###### Content under Creative Commons Attriubtion license CC-BY 4.0, (c)2014 Ian A. Carr, for Prof. L. A. Barba's MAE 6286 course at The George Washington University.

# Shoot First, Ask Questions Later

First, congratulations on making it to the 5th module of #numericalmooc! It's been quite a journey. There are two objectives in this notebook. One is to introduce a useful way of solving boundary value problems (BVP) called the shooting method. The second is to start applying our knowledge of numerics to the complex world of fluid motion. Let's get started!

## The Blasius Boundary Layer

Computational fluid dynamics is a very complex field. We are going to get started with a relatively simple problem. As the title suggests, it's called the Blasius Boundary Layer.

In case some of you are a little rusty in your fluid mechanics knowledge, you can check out the [wikipedia page](http://en.wikipedia.org/wiki/Boundary_layer) on boundary layers to get an idea. Put simply, a boundary layer is a thin layer of fluid just above a surface. The effect of the fluid's viscosity is greatest within this thin layer. Boundary layers can get very complex depending on the flow conditions and geometry of the surface, but today we are going to look at a very simple one.

Figure 1 shows fluid moving over a flat plate which continues infinitely in the positive x direction. The fluid moves with a unidirectional velocity we're calling $u_{\infty}$. A student named Heinrich Blasius came up with an interesting solution to this problem in 1908, and that's what we are going to explore today.

####Figure 1. Sketch of a boundary layer over a flat plate.

This flow field can be described with a single equation, a 3rd order, nonlinear ODE, but a single equation no less. To understand how this boils down to a single equation we have to go through a bit of derivation first. For those of you unfamiliar with fluids this is all the more critical in understanding this problem.

For this derivation we assume the flow is two-dimensional, steady, incompressible, and has zero pressure gradient. This means we can get rid of a lot of terms in the continuity equation and the equation of fluid motion ([Navier-Stokes](http://en.wikipedia.org/wiki/Navier%E2%80%93Stokes_equations)). Specifically the continuity equation reduces to

\begin{equation}
\frac{\partial u}{\partial x} + \frac{\partial v}{\partial y} = 0,
\end{equation}

and the equation of motion reduces to

\begin{equation}
u \frac{\partial u}{\partial x} + v \frac{\partial u}{\partial y} = \nu. \frac{\partial^2 u}{\partial y^2}.
\end{equation}

With some boundary conditions due to the no-slip condition at the surface of the plate

\begin{equation}
u = v = 0 \; \textrm{at} \; y = 0,\\
u = U, \; \frac{\partial u}{\partial y} = 0\; \textrm{at} \; y = \infty.
\end{equation}

From here Blasius reasoned the velocity profile should look *similar* for all values of x when compared to a non-dimentional distance from the wall. This means when the non-dimensional velocity is plotted against non-dimentional distance, the profile is the same for all scales. The obvious choice for non-dimentionalizing the distance from the wall is the [boundary layer thickness](http://en.wikipedia.org/wiki/Boundary_layer), $\delta$ in Figure 1. Blasius decided to call this non-dimentional distance $\eta$, termed the *similarity variable*.

Just from these assumptions we can see the solution would take the form

\begin{equation}
\frac{u}{U} = g(\eta) \; \textrm{where} \; \eta \propto \frac{y}{\delta}.
\end{equation}

Inspired by the Stokes solution, Blasius defined $\eta$ as

\begin{equation}
\eta = y \sqrt{\frac{U}{\nu x}}.
\end{equation}

You still with me? Good. Now we have to introduce what's called a [stream function](http://en.wikipedia.org/wiki/Stream_function). If you're unfamiliar with this concept it's a set of equations which describe the stream lines in a given flow. In this case the stream function, $\Psi$, is

\begin{equation}
u = \frac{\partial \Psi}{\partial y} \; \textrm{and} \; v = -\frac{\partial \Psi}{\partial x}
\end{equation}

As said, we are working with non-dimentional values due to the similarity assumption, so we want to make this stream function dimensionless. To do this we put $u$ and $v$ into the equation of motion, which gives

\begin{equation}
f(\eta) = \frac{\Psi}{\sqrt{\nu x U}}.
\end{equation}

Now that we have the stream function, $\Psi$, and the similarity variable, $\eta$, defined we can pull out the velocity components. Sparing some algebra, the velocity components are given by

\begin{equation}
u = U \frac{df}{d \eta},
\end{equation}

\begin{equation}
v = \frac{1}{2} \sqrt{\frac{\nu U}{x}} \left[ \eta \frac{df}{d \eta} -f \right].
\end{equation}

Now we want to fit these velocity components into the equation of motion and to do that we need to differentiate them. These come out to

\begin{equation}
\frac{\partial u}{\partial x} = -\frac{U}{2x} \eta \frac{d^2 f}{d \eta^2},
\end{equation}

\begin{equation}
\frac{\partial u}{\partial y} = U \sqrt{U/\nu x} \frac{d^2 f}{d \eta^2},
\end{equation}

and for that second derivative,

\begin{equation}
\frac{\partial^2 u}{\partial y^2} = \frac{U^2}{\nu x} \frac{d^3 f}{d \eta^3}.
\end{equation}

Finally, we plug those into the equation of motion to find

\begin{equation}
2 \frac{d^3 f}{d \eta^3} + f \frac{d^2 f}{d \eta^2} = 0.
\end{equation}

And that's it! That's the Blasius Boundary Layer Solution, the single 3rd order non-linear ODE we need to solve. Not too bad right? We need to make sure we have the boundary conditions in terms of $f$ and $\eta$.

\begin{equation}
f = \frac{df}{d \eta} = 0 \; \textrm{at} \; \eta = 0,\\
\frac{df}{d \eta} \rightarrow 1 \; \textrm{at} \; \eta \rightarrow \infty.
\end{equation}

Now we need to talk about how we will go about solving this equation.

## The Shooting Method

The shooting method provides us with the simplest way to solve the Blasius Boundary Layer. So to get the idea, suppose we have a (BVP) with Dirichlet boundary conditions

\begin{equation}
y'' = f(x,y,y'), \; y(a) = \alpha, \; y(b) = \beta .
\end{equation}

We then rearrange the BVP to the form

\begin{equation}
y' = z, \\
z' = f(x,y,z), \\
y(a) = \alpha, \\
y(b) = \beta.
\end{equation}

We can then turn the BVP above into an Initial Value Problem (IVP) by replacing the boundary condition $y(b)$ with

\begin{equation}
z(a) = \theta,
\end{equation}

where $\theta$ is some number. We can then solve the IVP with the method of our choosing (Euler, Runge-Kutta, etc.) to obtain the solution at $y(b)$. If, when we solve the IVP, $y(b) = \beta$, we've done it! We've solved the BVP! Mostly likely, though,  $y(b) \neq \beta$ on the first try and we will have to try other values of $\theta$ before we find the right one.

So what would it take to apply the shooting method to the Blasius boundary layer?

### Converting to an IVP

As said, the first step in implimenting the shooting method is turning the BVP into an IVP. To reiterate, we want to rewrite Blasius Boundary Layer Solution in such a way that it only involves the first derivative and can therefore be solved as an IVP. In this case the three equations are

\begin{equation}
\frac{df}{d \eta} = f_1 \; \textrm{with } f(0) = 0,
\end{equation}

\begin{equation}
\frac{df_1}{d \eta} = f_2 \; \textrm{with } f_1(0) = 0,\: f_1(\infty)=1,
\end{equation}

\begin{equation}
\frac{df_2}{d \eta} = \frac{1}{2} f_2*f \; \textrm{with } f_2(0) = ?.
\end{equation}

The term we have to 'shoot' for is $f_2(0)$, and we can use the knowledge that $f_1(\infty) = 1$ to do so.

### Euler's Method

So now that we have the problem laid out, we need to pick a way of solving the ODE. Let's use the simplest one for now, Euler's method. You'll remember we worked with this method way back in the first module on phugoid motion. Applying Euler's method to the IVP equations gives

\begin{equation}
f^{i+1} = f^i + f^i *d \eta
\end{equation}

\begin{equation}
f^{i+1}_1 = f_1^i + f_2^i * d \eta
\end{equation}

\begin{equation}
f^{i+1}_2 = f_2^i - \frac{1}{2} * f^i * f_2^i*d \eta
\end{equation}

And to make sure, the subscripts indicate the level of separation when turning the BVP into an IVP and the superscripts indicate the iterative scheme step.

Finally, let's start coding this up.

```
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

```
nfinal = 8.	# final value of eta, called n here
dn = 0.01	# step size
N = int(nfinal/dn)  # number of time steps
n = np.linspace(0.0,nfinal,N)
```

```
# arrays to be filled with values
f = np.zeros(N)
f1 = np.zeros(N)
f2 = np.zeros(N)

# initial conditions
f[0] = 0.
f1[0] = 0.
```

Now, as said, we want to guess the initial value of $f_2$ such that our $f_1(\infty) = 1$ boundary condition is met. We could do this manually, changing the first value, which would look something like this.

```
# manual way for shooting method
f2[0] = 0.25
```

But obviously, this is too much work and not what we're going to do. Instead let's just choose a range of values with an arbitrarily high maximum value for our shot $f_2(0)$, say 5. If we break it up into sufficentialy small steps we should be able to run through them until we find the one which matches our boundary condition. We then solve with each initial guess for $f_2(0)$ using Euler's method and print the final values to see how we did.

Also, let's take a look at how many guesses it took to get to the right value.

```
# our shot for the value of f at infinity
f2shot = np.linspace(0.0,5.0,1000)

for i in range(len(f2shot)):
    f2[0] = f2shot[i]

    # iterating using euler's method
    for j in range(0,N-1):
        f[j+1] = f[j] + f1[j]*dn
        f1[j+1] = f1[j] + f2[j]*dn
        f2[j+1] = f2[j] - 0.5*f[j]*f2[j]*dn
    if f1[-1] >= 1.0:
        print 'Final value of f1: ',f1[-1]
        print 'Chosen value of f2[0]: ', f2[0]
        print 'Number of guesses: ',i
        break
```

    Final value of f1:  1.00004245252
    Chosen value of f2[0]:  0.33033033033
    Number of guesses:  66

The last value of $f_1$, which is representative of it approaching $\infty$, is pretty close to 1, which is what we expect. The correct initial value for $f_2$ was about 0.33, which agrees with the literature. Also, let's make a mental note of the number of guesses it took to find the right one.

Now, let's look at how the non-dimentionalized velocity in the x direction changes as we move away from the wall.

```
# plotting x-component of velocity
plt.figure()
plt.plot(f1,n,lw=2)
plt.xlabel('x-component of velocity',fontsize=18)
plt.ylabel('$\eta$',fontsize=18)
```

Looks pretty good right? If you go to the [Blasius Boundary Layer](http://en.wikipedia.org/wiki/Blasius_boundary_layer) wikipedia page you can see our solution lines up pretty well with their figure.

We can do the same with the y-component of velocity.

```
# plotting y-component of velocity
plt.figure()
plt.plot((0.5*(f1*n-f)),n,lw=2)
plt.xlabel('y-component of velocity', fontsize=18)
plt.ylabel('$\eta$', fontsize=18)
```

Again, this figure matches the known solution.

Let's get back to the 66 guesses it took to get to the right initial value of $f_2$. We got to the solution by working our way up from 0 in small increments. That's not exactly a very clever way of doing things. Let's look into another method that might reduce the number of guesses we need to take.

## The Newton-Raphson Method

We want to improve the way we "aim" before we use the shooting method. An intuative way of doing this would be to compare each shot to the target value and adjust accordingly. This is exactly what the Newton-Raphson method does.

Given a reasonable guess, the Newton-Raphson Method uses the derivative of the function to determine the next guess. This method can then be used iteratively to narrow in on the correct answer using the following equation.

\begin{equation}
x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)}
\end{equation}

As you can see this equation requires we know the derivative, $f'(x_n)$, of the function we are working with. In the case of the Blasius Boundary Layer equation, we need to take the derivative of the 3 IVPs determined earlier. This results in

\begin{equation}
\frac{df_3}{d \eta} = f_4 \; \textrm{with} \; f_3(0) = 0,
\end{equation}

\begin{equation}
\frac{df_4}{d \eta} = f_5, \; \textrm{with } f_4(0) = 0,
\end{equation}

\begin{equation}
\frac{df_5}{d \eta} = \frac{1}{2}(f_3*f_2 + f*f_5)\; \textrm{with}\; f_5(0) = 1.
\end{equation}

As before, we discretize these equations and apply Euler's method to form

\begin{equation}
f_3^{i+1} = f_3^i + f_3^i *d \eta,
\end{equation}

\begin{equation}
f_4^{i+1} = f_4^i + f_5^i * d \eta,
\end{equation}

\begin{equation}
f_5^{i+1} = f_5^i - \frac{1}{2}(f_3^i* f_2^i + f^i * f_5^i) * d \eta .
\end{equation}

We then use these equations in the Newton-Raphson method to narrow in on the result. The adaptation of the general Newton-Raphson equation for this problem is

\begin{equation}
f_2(0) \leftarrow f_2(0) - \frac{f_2(\infty)-1}{f_5(\infty)}.
\end{equation}

Okay, so now that we have all that on the table, let's code it up and see how we've improved.

The speed of the method depends on the quality of the initial guess. With most applications it's possible to make an educated guess for this value. Let's start with 0.2 for now.

```
# starting with a reasonable guess for f2
f2guess = np.empty(N)
f2guess[0] = 0.2

# arrays to be filled by derivative functions
f3 = np.zeros(N)
f4 = np.zeros(N)
f5 = np.zeros(N)

# imposing initial condition
f5[0]=1
```

Now let's loop through both the original IVPs and the derivatives. Because this method doesn't simply grow the way the simple shooting method did, we have to set up a condition for convergence. In the code we can just call this the error bound. We will set it to some arbitrarily small value.

```
# setting up a break criterion for our loop
error_bound = 0.0001

for i in range(0,N-1):
	f2[0] = f2guess[i]
	for j in range(0,N-1):
		f[j+1] = f[j] + f1[j]*dn
		f1[j+1] = f1[j] + f2[j]*dn
		f2[j+1] = f2[j] - 0.5*f[j]*f2[j]*dn

		f3[j+1] = f3[j] + f3[j]*dn
		f4[j+1] = f4[j] + f5[j]*dn
		f5[j+1] = f5[j] - 0.5*(f3[j]*f2[j] + f[j]*f5[j])*dn

	f2guess[i+1] = f2guess[i] - ((f1[-1]-1)/f4[-1])

	if abs(f1[-1]-1)<=error_bound:
		print 'Final value of f1: ',f1[-1]
		print 'Chosen value of f2[0]: ', f2[0]
		print 'Number of guesses: ', i
		break
```

    Final value of f1:  0.999950414717
    Chosen value of f2[0]:  0.330284811375
    Number of guesses:  8

Look at that!

We converge to the same values for $f_1(\infty)$ and for the correct initial condition for $f_2$ but with an order of magnitude fewer guesses!

As said, the speed of the method depends of the quality of the initial guess. Try changing the value of the initial guess to see how it effects the number of required guesses before convergence.

In the coming notebooks we will look at other interesting applications of our knowledge to fluids and other engineering problems.

##### Dig Deeper

We've shown the Newton-Raphson method greatly reduces the number of guesses required to converge on the answer, but the computation has a few more steps. Use the timeit functionality to see how the computational time compares.

Is it faster or slower?  With a more complex problem do you think taking the time to employ the Newton-Raphson Method would be advantageous?

## References

* Pozrikidis, C. Introduction to Theoretical and Computational Fluid Dynamics. New York: Oxford UP, 1997. Print.

* Ben-Israel, Adi. "A Newton-Raphson Method for the Solution of Systems of Equations." Journal of Mathematical Analysis and Applications 15.2 (1966): 243-52. Web.

* Oteh, Uche. Mechanics of Fluids. Bloomington, IN: AuthorHouse, 2008. Print.

```
from IPython.core.display import HTML
css_file = '../../styles/numericalmoocstyle.css'
HTML(open(css_file, "r").read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 750px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
margin-top:0.8em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Alegreya Sans' sans-serif;
    line-height: 140%;
    font-size: 125%;
    font-weight: 400;
    width:600px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Nixie One', serif;
    font-style:regular;
    font-weight: 400;
    font-size: 45pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}
.text_cell_render h2 {
    font-family: 'Nixie One', serif;
    font-weight: 400;
    font-size: 30pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.1em;
    margin-top: 0.3em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Nixie One', serif;
    margin-top:16px;
	font-size: 22pt;
    font-weight: 600;
    margin-bottom: 3px;
    font-style: regular;
    color: rgb(102,102,0);
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Nixie One', serif;
    font-size: 14pt;
    text-align: center;
    margin-top: 0em;
    margin-bottom: 2em;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Nixie One', sans-serif;
    font-weight: 400;
    font-size: 16pt;
    color: rgb(163,0,0);
    font-style: italic;
    margin-bottom: .1em;
    margin-top: 0.8em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'PT Mono', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

.CodeMirror{
        font-family: "PT Mono";
        font-size: 90%;
}

</style>


# Decision Trees and Random Forests

This notebook is based on chapters 6 and 7 of *Hands-On Machine Learning with Scikit-Learn and TensorFlow* by Aurélien Géron, https://github.com/ageron/handson-ml.

## Setup

First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:

```python
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."

def image_path(fig_id):
    return os.path.join(PROJECT_ROOT_DIR, "images", fig_id)

def save_fig(fig_id, tight_layout=True):
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(image_path(fig_id) + ".png", format='png', dpi=300)
```

```python
from subprocess import call

from matplotlib.colors import ListedColormap

from sklearn.datasets import load_digits
from sklearn.datasets import load_iris
from sklearn.datasets import make_moons

from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error

from sklearn.model_selection import train_test_split

from sklearn.svm import SVC

from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import export_graphviz
```

# Decision Trees
are...
- versatile ML algorithms that can perform both classification and regression tasks, even multioutput tasks
- very powerful algorithms, capable of fitting complex datasets
- also the fundamental components of Random Forests

## Training and Visualizing a Decision Tree

```python
iris = load_iris()
X = iris.data[:, 2:] # petal length and width
y = iris.target

tree_clf = DecisionTreeClassifier(max_depth=2, random_state=42)
tree_clf.fit(X, y)
```

    DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=2,
                max_features=None, max_leaf_nodes=None,
                min_impurity_decrease=0.0, min_impurity_split=None,
                min_samples_leaf=1, min_samples_split=2,
                min_weight_fraction_leaf=0.0, presort=False, random_state=42,
                splitter='best')

```python
export_graphviz(
        tree_clf,
        out_file=image_path("iris_tree.dot"),
        feature_names=iris.feature_names[2:],
        class_names=iris.target_names,
        rounded=True,
        filled=True
    )
```

```python
call('dot -Tpng images/iris_tree.dot -o images/iris_tree.png'.split(' '));
```

## Making Predictions

One of the many qualities of Decision Tree is that they require very little data preparation. In particular, they don't reuqire feature scaling or centering at all.

*Gini impurity*
\begin{equation}
G_i = 1 - \sum_{k=1}^n p_{i,k}^2
\end{equation}
where $p_{i,k}$ is the ratio of class $k$ instances among the training instances in the $i^{\text{th}}$ node. Alternatively, one could use entropy instead Gini.

Scikit-Learn uses the Classification And Regression Tree (CART) algorithm, which produces only *binary trees*: nonleaf nodes always have two children. However, other algorithms such as ID3 can produce Decision Trees with nodes that have more than two children.

```python
def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
    x1s = np.linspace(axes[0], axes[1], 100)
    x2s = np.linspace(axes[2], axes[3], 100)
    x1, x2 = np.meshgrid(x1s, x2s)
    X_new = np.c_[x1.ravel(), x2.ravel()]
    y_pred = clf.predict(X_new).reshape(x1.shape)
    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)
    if not iris:
        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
    if plot_training:
        plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
        plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
        plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
        plt.axis(axes)
    if iris:
        plt.xlabel("Petal length", fontsize=14)
        plt.ylabel("Petal width", fontsize=14)
    else:
        plt.xlabel(r"$x_1$", fontsize=18)
        plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
    if legend:
        plt.legend(loc="lower right", fontsize=14)

plt.figure(figsize=(8, 4))
plot_decision_boundary(tree_clf, X, y)
plt.plot([2.45, 2.45], [0, 3], "k-", linewidth=2)
plt.plot([2.45, 7.5], [1.75, 1.75], "k--", linewidth=2)
plt.plot([4.95, 4.95], [0, 1.75], "k:", linewidth=2)
plt.plot([4.85, 4.85], [1.75, 3], "k:", linewidth=2)
plt.text(1.40, 1.0, "Depth=0", fontsize=15)
plt.text(3.2, 1.80, "Depth=1", fontsize=13)
plt.text(4.05, 0.5, "(Depth=2)", fontsize=11)

save_fig("decision_tree_decision_boundaries_plot")
plt.show()
```

### Predicting classes and class probabilities

```python
tree_clf.predict_proba([[5, 1.5]])
```

    array([[0.        , 0.90740741, 0.09259259]])

```python
tree_clf.predict([[5, 1.5]])
```

    array([1])

### The CART Training Algorithm

*CART* cost function for classification
\begin{equation}
J(k,t_k) = \frac{m_{\text{left}}}{m} G_{\text{left}} + \frac{m_{\text{right}}}{m} G_{\text{right}}
\end{equation}
where $G_{\text{left}/\text{right}}$ measures the impurity of the left/right subset, and $m_{\text{left}/\text{right}}$ is the number of instances in the left/right subset.

The CART algorithm is a *greedy algorithm*. It greedily searches for an optinum split at the top level, then repeats the process at each level. It does not check whether or not the split will lead to the lowest possible impurity several levels down. Unfortunately, finding the optimal tree is known to be an *NP-Complete* problem; it requires $O(e^m)$ time. This is why we must settle for a "reasonably good" solution.

## Regularization Hyperparameters

- `max_depth`
- `min_samples_split` (min. # of samples a node must have before it can be split)
- `min_samples_leaf` (min. # of samples a leaf must have)
- `min_weight_fraction_leaf` (same as min_samples_leaf but expressed as a fraction of the total number of weighted instances)
- `max_leaf_nodes` (max. # of leaf nodes)
- `max_features` (max. # of features that are evaluated for splitting at each node)

```python
Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)

deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
deep_tree_clf1.fit(Xm, ym)
deep_tree_clf2.fit(Xm, ym)

plt.figure(figsize=(11, 4))
plt.subplot(121)
plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
plt.title("No restrictions", fontsize=16)
plt.subplot(122)
plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)

save_fig("min_samples_leaf_plot")
plt.show()
```

## Regression

Quadratic training set + noise

```python
np.random.seed(42)
m = 200
X = np.random.rand(m, 1)
y = 4 * (X - 0.5) ** 2
y = y + np.random.randn(m, 1) / 10
```

```python
tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg.fit(X, y)
```

    DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,
               max_leaf_nodes=None, min_impurity_decrease=0.0,
               min_impurity_split=None, min_samples_leaf=1,
               min_samples_split=2, min_weight_fraction_leaf=0.0,
               presort=False, random_state=42, splitter='best')

```python
export_graphviz(
        tree_reg,
        out_file=image_path("regression_tree.dot"),
        feature_names=["x1"],
        rounded=True,
        filled=True
    )

call('dot -Tpng images/regression_tree.dot -o images/regression_tree.png'.split(' '));
```

```python
tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
tree_reg1.fit(X, y)
tree_reg2.fit(X, y)

def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
    x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
    y_pred = tree_reg.predict(x1)
    plt.axis(axes)
    plt.xlabel("$x_1$", fontsize=18)
    if ylabel:
        plt.ylabel(ylabel, fontsize=18, rotation=0)
    plt.plot(X, y, "b.")
    plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")

plt.figure(figsize=(11, 4))
plt.subplot(121)
plot_regression_predictions(tree_reg1, X, y)
for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
plt.text(0.21, 0.65, "Depth=0", fontsize=15)
plt.text(0.01, 0.2, "Depth=1", fontsize=13)
plt.text(0.65, 0.8, "Depth=1", fontsize=13)
plt.legend(loc="upper center", fontsize=18)
plt.title("max_depth=2", fontsize=14)

plt.subplot(122)
plot_regression_predictions(tree_reg2, X, y, ylabel=None)
for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
    plt.plot([split, split], [-0.2, 1], style, linewidth=2)
for split in (0.0458, 0.1298, 0.2873, 0.9040):
    plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
plt.text(0.3, 0.5, "Depth=2", fontsize=13)
plt.title("max_depth=3", fontsize=14)

save_fig("tree_regression_plot")
plt.show()
```

*CART* cost function for regression
\begin{equation}
J(k,t_k) = \frac{m_{\text{left}}}{m} \text{MSE}_{\text{left}} + \frac{m_{\text{right}}}{m} \text{MSE}_{\text{right}}
\end{equation}
where
\begin{equation}
\text{MSE}_{\text{node}} = \sum_{i \in \text{node}} \left(\hat{y}_{\text{node}} - y^{(i)}\right)^2
\end{equation}
and
\begin{equation}
\hat{y}_{\text{node}} = \frac{1}{m_{\text{node}}} \sum_{i \in \text{node}} y^{(i)}
\end{equation}

```python
tree_reg1 = DecisionTreeRegressor(random_state=42)
tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
tree_reg1.fit(X, y)
tree_reg2.fit(X, y)

x1 = np.linspace(0, 1, 500).reshape(-1, 1)
y_pred1 = tree_reg1.predict(x1)
y_pred2 = tree_reg2.predict(x1)

plt.figure(figsize=(11, 4))

plt.subplot(121)
plt.plot(X, y, "b.")
plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
plt.axis([0, 1, -0.2, 1.1])
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", fontsize=18, rotation=0)
plt.legend(loc="upper center", fontsize=18)
plt.title("No restrictions", fontsize=14)

plt.subplot(122)
plt.plot(X, y, "b.")
plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
plt.axis([0, 1, -0.2, 1.1])
plt.xlabel("$x_1$", fontsize=18)
plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)

save_fig("tree_regression_regularization_plot")
plt.show()
```

# Ensemble Learning and Random Forests

## Voting Classifiers
Ensemble methods work best when the predictors are as independent from one anotherr as possible. One way to get diverse classifiers is to train them using very different algorithms. This increases the chance that they will make very different types of error, improving the ensemble's accuracy.

Training diverse classifiers

Hard voting classifier predictions

```python
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
```

```python
log_clf = LogisticRegression(solver='liblinear', random_state=42)
rnd_clf = RandomForestClassifier(n_estimators=100, random_state=42)
svm_clf = SVC(gamma='auto', random_state=42)

voting_clf = VotingClassifier(
    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
    voting='hard')
voting_clf.fit(X_train, y_train)
```

    VotingClassifier(estimators=[('lr', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
              intercept_scaling=1, max_iter=100, multi_class='warn',
              n_jobs=None, penalty='l2', random_state=42, solver='liblinear',
              tol=0.0001, verbose=0, warm_start=False)), ('rf', Rando...f',
      max_iter=-1, probability=False, random_state=42, shrinking=True,
      tol=0.001, verbose=False))],
             flatten_transform=None, n_jobs=None, voting='hard', weights=None)

```python
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
```

    LogisticRegression 0.864
    RandomForestClassifier 0.896
    SVC 0.888
    VotingClassifier 0.904

```python
log_clf = LogisticRegression(solver='liblinear', random_state=42)
rnd_clf = RandomForestClassifier(n_estimators=100, random_state=42)
svm_clf = SVC(probability=True, gamma='auto', random_state=42)

voting_clf = VotingClassifier(
    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
    voting='soft')
voting_clf.fit(X_train, y_train)
```

    VotingClassifier(estimators=[('lr', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
              intercept_scaling=1, max_iter=100, multi_class='warn',
              n_jobs=None, penalty='l2', random_state=42, solver='liblinear',
              tol=0.0001, verbose=0, warm_start=False)), ('rf', Rando...bf',
      max_iter=-1, probability=True, random_state=42, shrinking=True,
      tol=0.001, verbose=False))],
             flatten_transform=None, n_jobs=None, voting='soft', weights=None)

```python
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
```

    LogisticRegression 0.864
    RandomForestClassifier 0.896
    SVC 0.888
    VotingClassifier 0.912

## Bagging and Pasting

Pasting/Bagging training set sampling and training

```python
tree_clf = DecisionTreeClassifier(random_state=42)
tree_clf.fit(X_train, y_train)
y_pred_tree = tree_clf.predict(X_test)
print(accuracy_score(y_test, y_pred_tree))
```

    0.856

```python
bag_clf = BaggingClassifier(
    DecisionTreeClassifier(random_state=42), n_estimators=500,
    max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_test)
```

```python
print(accuracy_score(y_test, y_pred))
```

    0.904

```python
def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
    x1s = np.linspace(axes[0], axes[1], 100)
    x2s = np.linspace(axes[2], axes[3], 100)
    x1, x2 = np.meshgrid(x1s, x2s)
    X_new = np.c_[x1.ravel(), x2.ravel()]
    y_pred = clf.predict(X_new).reshape(x1.shape)
    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)
    if contour:
        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
    plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
    plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
    plt.axis(axes)
    plt.xlabel(r"$x_1$", fontsize=18)
    plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
```

```python
plt.figure(figsize=(11,4))
plt.subplot(121)
plot_decision_boundary(tree_clf, X, y)
plt.title("Decision Tree", fontsize=14)
plt.subplot(122)
plot_decision_boundary(bag_clf, X, y)
plt.title("Decision Trees with Bagging", fontsize=14)
save_fig("decision_tree_without_and_with_bagging_plot")
plt.show()
```

## Random Forests

...is an ensemble of Decision Trees, generally trained via the bagging method, typically with `max_samples` set to the size of the training set. Instead of building a `BaggingClassifier` and passing it a `DecisionTreeClassifier`, you can instead use the `RandomForestClassifier` class, which is more convenient and optimized for Decision Trees. (Similarly, the is a `RandomForestRegressor` class for regression tasks.)

```python
rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
rnd_clf.fit(X_train, y_train)

y_pred_rf = rnd_clf.predict(X_test)
```

The following `BaggingClassifier` is roughly equivalent to the previous `RandomForestClassifier`

```python
bag_clf = BaggingClassifier(
    DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)

bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_test)

np.sum(y_pred == y_pred_rf) / len(y_pred)  # almost identical predictions
```

    0.976

### Feature Importance

```python
digits = load_digits()
```

```python
rnd_clf = RandomForestClassifier(n_estimators=100, random_state=42)
rnd_clf.fit(digits["data"], digits["target"])
```

    RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
                max_depth=None, max_features='auto', max_leaf_nodes=None,
                min_impurity_decrease=0.0, min_impurity_split=None,
                min_samples_leaf=1, min_samples_split=2,
                min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,
                oob_score=False, random_state=42, verbose=0, warm_start=False)

```python
def plot_digit(data):
    image = data.reshape(8, 8)
    plt.imshow(image, cmap = matplotlib.cm.hot,
               interpolation="nearest")
    plt.axis("off")
```

```python
plot_digit(rnd_clf.feature_importances_)

cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])
cbar.ax.set_yticklabels(['Not important', 'Very important'])

save_fig("mnist_feature_importance_plot")
plt.show()
```

## Boosting

...refers to any Ensemble method that can combine several weak learners into a strong learner.

There is one important drawback to this sequential learning technique: it cannot be parallelized (or only partially), since each predictor can only be trained after the previous predictor has been trained and evaluated. As a result, it does not scale as well as bagging or pasting.

In the [Insight](https://www.insightdata.ai) Interview Prep Checklist AI.SV 2018C it asks why [XGBoost](https://en.wikipedia.org/wiki/Xgboost) is good. It appears to aimed directly at the above warning, aiming to provide a "Scalable, Portable, and Distributed Gradient Boosting."

### Gradient Boosting

...works by sequentially adding predictors to an ensemble. This method tried to fit the new predictor to the *residual errors* made by the preivous predictor.

A noisy quadratic training set

```python
np.random.seed(42)
X = np.random.rand(100, 1) - 0.5
y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
```

```python
tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg1.fit(X, y)
```

    DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,
               max_leaf_nodes=None, min_impurity_decrease=0.0,
               min_impurity_split=None, min_samples_leaf=1,
               min_samples_split=2, min_weight_fraction_leaf=0.0,
               presort=False, random_state=42, splitter='best')

```python
y2 = y - tree_reg1.predict(X)
tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg2.fit(X, y2)
```

    DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,
               max_leaf_nodes=None, min_impurity_decrease=0.0,
               min_impurity_split=None, min_samples_leaf=1,
               min_samples_split=2, min_weight_fraction_leaf=0.0,
               presort=False, random_state=42, splitter='best')

```python
y3 = y2 - tree_reg2.predict(X)
tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg3.fit(X, y3)
```

    DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,
               max_leaf_nodes=None, min_impurity_decrease=0.0,
               min_impurity_split=None, min_samples_leaf=1,
               min_samples_split=2, min_weight_fraction_leaf=0.0,
               presort=False, random_state=42, splitter='best')

```python
def plot_predictions(regressors, X, y, axes, label=None, style="r-", data_style="b.", data_label=None):
    x1 = np.linspace(axes[0], axes[1], 500)
    y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)
    plt.plot(X[:, 0], y, data_style, label=data_label)
    plt.plot(x1, y_pred, style, linewidth=2, label=label)
    if label or data_label:
        plt.legend(loc="upper center", fontsize=16)
    plt.axis(axes)

plt.figure(figsize=(11,11))

plt.subplot(321)
plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h_1(x_1)$", style="g-", data_label="Training set")
plt.ylabel("$y$", fontsize=16, rotation=0)
plt.title("Residuals and tree predictions", fontsize=16)

plt.subplot(322)
plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1)$", data_label="Training set")
plt.ylabel("$y$", fontsize=16, rotation=0)
plt.title("Ensemble predictions", fontsize=16)

plt.subplot(323)
plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_2(x_1)$", style="g-", data_style="k+", data_label="Residuals")
plt.ylabel("$y - h_1(x_1)$", fontsize=16)

plt.subplot(324)
plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
plt.ylabel("$y$", fontsize=16, rotation=0)

plt.subplot(325)
plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label="$h_3(x_1)$", style="g-", data_style="k+")
plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16)
plt.xlabel("$x_1$", fontsize=16)

plt.subplot(326)
plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
plt.xlabel("$x_1$", fontsize=16)
plt.ylabel("$y$", fontsize=16, rotation=0)

save_fig("gradient_boosting_plot")
plt.show()
```

A simpler way to train GBRT ensembles is to use Scikit-Learn's `GradientBoostingRegressor` class. The following code creates the same ensemble as the previous one:

```python
gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1, random_state=42)
gbrt.fit(X, y)
```

    GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
                 learning_rate=0.1, loss='ls', max_depth=2, max_features=None,
                 max_leaf_nodes=None, min_impurity_decrease=0.0,
                 min_impurity_split=None, min_samples_leaf=1,
                 min_samples_split=2, min_weight_fraction_leaf=0.0,
                 n_estimators=3, n_iter_no_change=None, presort='auto',
                 random_state=42, subsample=1.0, tol=0.0001,
                 validation_fraction=0.1, verbose=0, warm_start=False)

```python
gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)
gbrt_slow.fit(X, y)
```

    GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
                 learning_rate=0.1, loss='ls', max_depth=2, max_features=None,
                 max_leaf_nodes=None, min_impurity_decrease=0.0,
                 min_impurity_split=None, min_samples_leaf=1,
                 min_samples_split=2, min_weight_fraction_leaf=0.0,
                 n_estimators=200, n_iter_no_change=None, presort='auto',
                 random_state=42, subsample=1.0, tol=0.0001,
                 validation_fraction=0.1, verbose=0, warm_start=False)

```python
plt.figure(figsize=(11,4))

plt.subplot(121)
plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label="Ensemble predictions")
plt.title("learning_rate={}, n_estimators={}".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)

plt.subplot(122)
plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
plt.title("learning_rate={}, n_estimators={}".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)

save_fig("gbrt_learning_rate_plot")
plt.show()
```

### Early Stopping

```python
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)

gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)
gbrt.fit(X_train, y_train)

errors = [mean_squared_error(y_val, y_pred)
          for y_pred in gbrt.staged_predict(X_val)]
bst_n_estimators = np.argmin(errors)

gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)
gbrt_best.fit(X_train, y_train)
```

    GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
                 learning_rate=0.1, loss='ls', max_depth=2, max_features=None,
                 max_leaf_nodes=None, min_impurity_decrease=0.0,
                 min_impurity_split=None, min_samples_leaf=1,
                 min_samples_split=2, min_weight_fraction_leaf=0.0,
                 n_estimators=55, n_iter_no_change=None, presort='auto',
                 random_state=42, subsample=1.0, tol=0.0001,
                 validation_fraction=0.1, verbose=0, warm_start=False)

```python
min_error = np.min(errors)
```

```python
plt.figure(figsize=(11, 4))

plt.subplot(121)
plt.plot(errors, "b.-")
plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], "k--")
plt.plot([0, 120], [min_error, min_error], "k--")
plt.plot(bst_n_estimators, min_error, "ko")
plt.text(bst_n_estimators, min_error*1.2, "Minimum", ha="center", fontsize=14)
plt.axis([0, 120, 0, 0.01])
plt.xlabel("Number of trees")
plt.title("Validation error", fontsize=14)

plt.subplot(122)
plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])
plt.title("Best model (%d trees)" % bst_n_estimators, fontsize=14)

save_fig("early_stopping_gbrt_plot")
plt.show()
```

```python
gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)

min_val_error = float("inf")
error_going_up = 0
for n_estimators in range(1, 120):
    gbrt.n_estimators = n_estimators
    gbrt.fit(X_train, y_train)
    y_pred = gbrt.predict(X_val)
    val_error = mean_squared_error(y_val, y_pred)
    if val_error < min_val_error:
        min_val_error = val_error
        error_going_up = 0
    else:
        error_going_up += 1
        if error_going_up == 5:
            break  # early stopping
```

```python
print(gbrt.n_estimators)
```

    61

## Exercise

If your Gradient Boosting ensemble overfits the training set, should you increase or decrease the learning rate?

```python

```


# What is it?
spyIVP is in principle YET ANOTHER(!!) a general(ish)-purpose nnumerical IVP solver with some symbollic elements. It has a lot of overlap with Mathematica's NDSolve- NDSolve obviously is way more powerful for solving general equations (including PDEs).  But symODE is good for a range of useful problems, and has (or will have) a few features that aren't easily reproduced in Mathematics.  Overall it is less of a 'black-box' than NDSolve, for better or worse.

1. Can easily add explicit numerical code ( can in Mathematica too, but more hoops)
2. Can (shortly) integrate with tensorflow, bot for increased speed and to enable automatic differentiation.
3. The driving use case is ensembles of similar atoms interacting by long-range electric fields: large ensembles of states with long-range coupling

Other python ODE solving packages that are likely more approtriate for general use:

* https://github.com/neurophysik/jitcode,
* http://bjodah.github.io/pyodesys/latest/index.html

They both seem more powerful, could likely replace this package for even the desired purpose. However I only learned about them after writing this, and I haven't tried them. To my mind it does seem unlikely that they'll be easily extended to large scales or to automatic differentiation of the whole system as is possible with Tensorflow, but please let me know if I"m mistaken

# Some examples

Mostly what it's written for though is to enable assembling/solving master-equation based systems in python. That will be covered in [Solving master equations using pySODE]

## Imports

```python
%matplotlib inline
%load_ext autoreload
%autoreload 2
from pylab import *
from spylind import spyIVP as so
import sympy as sm;
sm.init_printing()
```

## Projectile motion

$$\frac{dx}{dt} = vx$$
$$\frac{dy}{dt} = vy$$

$$\frac{dv_x}{dt} = g$$
$$\frac{dv_y}{dt} = 0$$

```python
xS, yS, vxS, vyS = sm.symbols("x, y, v_x, v_y", real=True)
g= -10.0
dy_dtD = {xS:vxS,
       yS:vyS,
       vxS:0,
       vyS:g }
```

```python
ode_s = so.ODESolver(dy_dtD,  dims={})
ode_s.set_initial_conditions({xS:0,yS:0,vxS:10,vyS:10})
ode_s.setup()
tSteps = linspace(0,2,30)
arr =ode_s.integrate(tSteps, 1.0)

sm.ode
plot(tSteps, arr.real)
xlabel('t')
legend(dy_dtD.keys())
```

```python
tEff = tSteps #+ (tSteps[1]-tSteps[0])/2
sym_res = dict(
    x= 10 *tEff,
    y = 10*tEff -5*tEff**2
    )
allclose(arr[:,0], sym_res['x'])
allclose(arr[:,1], sym_res['y'])
```

    True

```python
tS = sm.symbols("t")

xS = sm.Function('x')(tS)
vxS = sm.Function('v_x')(tS)
yS = sm.Function('y')(tS)
vyS = sm.Function('v_y')(tS)

```

```python
from sympy import ode
sm.dsolve([sm.Eq(xS.diff(), vxS), sm.Eq(vxS.diff(), 0)])
eqL = [sm.Eq(yS.diff(),vyS), sm.Eq(vyS.diff(), g)]
sm.dsolve(sm.Eq(yS.diff(), g*tS) )
```

```python

```

## An ensemble of similar projectiles

```python
xS, yS, vxS, vyS = sm.symbols("x, y, v_x, v_y", real=True)
zS = sm.symbols("z", real=True)
g= -10.0
dy_dtD = {xS:vxS,
       yS:vyS ,
       vxS:1 +zS,
       vyS: g  }

ode_s = so.ODESolver(dy_dtD,  dims={zS:linspace(-2,2,10)})
ode_s.set_initial_conditions({xS:0,yS:0,vxS:10,vyS:10})
ode_s.setup(bForceStateDimensions=True)
tSteps = linspace(0,1,100)
arr =ode_s.integrate(tSteps, 1.0)

plot(tSteps, arr[:,:,0].real)
xlabel('t')
legend(dy_dtD.keys())
figure()
plot(tSteps, arr[:,:,-1].real)
xlabel('t')
legend(dy_dtD.keys())
figure()
imshow(arr[:,1,:].real, aspect='auto')
```

## An ensemble of interacting projectiles (via a mutual attraction)

What happens if the projectiles experience a force pushing toward their center-of-mass.

Currently there's no easy way to decsribe the interaction symbollically (though may yet be added). It's easy to do numerically however.

The interaction is $F_j = \sum_{i\neq j}^n k \frac{y_i-y_j}{|(y_i-y_j)^3|}$. This is infinite as particles get close, which is obviously a problem for a simulation. So we'll need to damp it at close range.

```python
xS, yS, vxS, vyS, kS = sm.symbols("x, y, v_x, v_y, k", real=True)
indS = sm.symbols("ind", real=True)
g= -10.0
dy_dtD = {xS:vxS,
       yS:vyS,
       vxS:1,
       vyS: g + 30*kS }

ode_s = so.ODESolver(dy_dtD,  dims={indS:arange(10)})
ode_s.set_initial_conditions({xS:0*ones(10),yS:linspace(-10,10,10),vxS:10*ones(10),vyS:15*ones(10) })
def couplingF(t, dimAxes, state, driving_vals):
    x,y,vx,vy = state
    diff = y[:,newaxis] - y[newaxis,:]
    diff[np.diag_indices_from(diff)]=1e20 # Stop divide by zeros due to distance of particles from themselves
    sq_diff = diff**2 / (1- exp(-(diff/0.3)**2 ))#  Need to damp the interaction for close particles to avoid singularities
    #sq_diff[np.diag_indices_from(sq_diff)]=1.
    F = -np.sum(sign(diff)/sq_diff, axis=1)
    return F
ode_s.set_state_dep_funcs({kS:couplingF})

ode_s.setup(bForceStateDimensions=True)
tSteps = linspace(0,3.,2000)
arr =ode_s.integrate(tSteps, 1.0)

plot(tSteps, arr[:,1,:].real)
xlabel('t')
ylabel('y')
title("mutually attractive projectiles")

```

## Using a custom output function

```python
xS, yS, vxS, vyS, kS = sm.symbols("x, y, v_x, v_y, k", real=True)
indS = sm.symbols("ind", real=True)
g= -10.0
dy_dtD = {xS:vxS,
       yS:vyS,
       vxS:1,
       vyS: g + 30*kS }

ode_s = so.ODESolver(dy_dtD,  dims={indS:arange(10)})
ode_s.set_initial_conditions({xS:0*ones(10),yS:linspace(-10,10,10),vxS:10*ones(10),vyS:15*ones(10) })
def couplingF(t, dimAxes, state, driving_vals):
    x,y,vx,vy = state
    diff = y[:,newaxis] - y[newaxis,:]
    diff[np.diag_indices_from(diff)]=1e20 # Stop divide by zeros due to distance of particles from themselves
    sq_diff = diff**2 / (1- exp(-(diff/0.3)**2 ))#  Need to damp the interaction for close particles to avoid singularities
    #sq_diff[np.diag_indices_from(sq_diff)]=1.
    F = -np.sum(sign(diff)/sq_diff, axis=1)
    return F
ode_s.set_state_dep_funcs({kS:couplingF})

def output_mean_state(state):
    mean_state = state.mean(axis=1)
    return mean_state
ode_s.set_online_processing(output_mean_state)

ode_s.setup(bForceStateDimensions=True)
tSteps = linspace(0,3.,2000)
arr =ode_s.integrate(tSteps, 1.0)

plot(tSteps, arr[:,:].real)
xlabel('t')
ylabel('y')
title("mutually attractive projectiles")
legend(ode_s.symsD.prop_state_syms)

```

```python

```


$\frac{{\partial {1}}}{{\partial {2}}}$

```python
# 如果对带有一个变量或是未赋值语句的cell执行操作，Jupyter将会自动打印该变量而无需一个输出语句。
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all" #默认为'last'

# display(Math(latex_s))和display(Latex(latex_s))输出的是latex类型，
# display(Markdown(latex_s))输出的是markdown
# 推荐markdown和Latex；而Math只支持纯latex
from IPython.display import display,Latex, SVG, Math, Markdown
```

```python
1
2
x = 3
x

```

    1

    2

    3

```python
s_1 = r"$\frac{{\partial {}}}{{\partial {}}}$".format(1, 2)
a = 13.49
b = 2.2544223
P = 302.99
V = 90.02
s_2 =  rf"""\
# xx

Dims: ${a}m \times{b:5.2}m$

Area: ${P}m^2$

Volume: ${V}m^3$
"""
```

```python
# 只能作为最后一个输出，才能显示
Latex(s_1)
```

$\frac{\partial 1}{\partial 2}$

```python
tobj1_1= Markdown(s_1)
tobj1_2= Math(s_1)
tobj1_3= Latex(s_1)
display(tobj1_1)
display(tobj1_2)
display(tobj1_3)
```

$\frac{\partial 1}{\partial 2}$

$\displaystyle \frac{\partial 1}{\partial 2}$

$\frac{\partial 1}{\partial 2}$

```python
tobj2_1=Markdown(s_2)
tobj2_2=Math(s_2)
tobj2_3= Latex(s_2)
display(tobj2_1)
display(tobj2_2)
display(tobj2_3)
```

\
# xx

Dims: $13.49m \times  2.3m$

Area: $302.99m^2$

Volume: $90.02m^3$

$\displaystyle \
# xx

Dims: $13.49m \times  2.3m$

Area: $302.99m^2$

Volume: $90.02m^3$
$

\
# xx

Dims: $13.49m \times  2.3m$

Area: $302.99m^2$

Volume: $90.02m^3$

```python
# https://docs.sympy.org/dev/tutorial/printing.html
from sympy import MatrixSymbol, Inverse, symbols, Determinant, Trace, Derivative
from sympy import sqrt,latex,Integral,pprint,init_printing

x = symbols("x")
y = symbols("y")
s = "1+2**(x+y)"
latex(eval(s))  # prints '$1 + {2}^{x + y}$'

# Integral是积分的意思
print(latex(Integral(sqrt(1/x), x)))

pprint(Integral(sqrt(1/x), x), use_unicode=True)
```

```python
import math
import latexify

# 写pi而不是math.pi，就可以了

@latexify.with_latex
def f_1(x):
    if x==1:
        return 1
    elif x==2:
        return 2
    else:
        return f_1(x-1) + f_1(x-2)

print(f_1)

```

    \mathrm{f_1}(x)\triangleq \left\{ \begin{array}{ll} 1, & \mathrm{if} \ x=1 \\ 2, & \mathrm{if} \ x=2 \\ \mathrm{f_1}\left(x - 1\right) + \mathrm{f_1}\left(x - 2\right), & \mathrm{otherwise} \end{array} \right.

```python
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams['text.usetex'] = True
plt.rcParams['text.latex.preamble'] = r'\usepackage{amsmath} \usepackage{amsfonts}'
# plt.rcParams.update({
#     #'font.size': 8,
#     #'text.usetex': True,
#     'text.latex.preamble':  [r'\usepackage{amsmath}', #for \text command
#                             r'\usepackage{amsfonts}', # \mathbb is provided by the LaTeX package amsfonts
#                             ]
# })
# 参数'text.latex.preamble' or 'pgf.preamble'（这两个不是同一个参数） 以前是用数组，现在用字符串
# 罗马体 operatorname -> mbox or mathrm

# https://matplotlib.org/stable/tutorials/text/mathtext.html

# matplotlib.rcParams['text.latex.unicode'] = True
# matplotlib.rcParams['text.latex.preamble'] = [
#        '\\usepackage{CJK}',
#        r'\AtBeginDocument{\begin{CJK}{UTF8}{gbsn}}',
#        r'\AtEndDocument{\end{CJK}}',
# ]

# Matplotlib中文显示和Latex
#import matplotlib.font_manager as mf # 导入字体管理器
#my_font= mf.FontProperties(fname='C://Windows//Fonts/simsun.ttc') # 加载字体
def show_latex(eqn:int, latex_s:str , validated:bool):
    fig, ax = plt.subplots(figsize=(20, 0.7))
    #latex_s = r"$\alpha _ { 1 } ^ { r } \gamma _ { 1 } + \dots + \alpha _ { N } ^ { r } \gamma _ { N } = 0 \quad ( r = 1 , . . . , R ) ,$"
    plt.text(0.01, 0.5, "({})".format(eqn), ha='left', va='center', fontsize=20)
    #水平和垂直方向居中对齐
    plt.text(0.5, 0.5, latex_s, ha='center', va='center', fontsize=20)
    if validated:
        plt.text(0.97, 0.5, r"ok", ha='right', va='center', fontsize=20, color = "g")
    else:
        plt.text(0.97, 0.5, r"error", ha='right', va='center', fontsize=20, color = "r")
    # 隐藏框线
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.spines['left'].set_visible(False)
    # 隐藏坐标轴的刻度信息
    plt.xticks([])
    plt.yticks([])
show_latex(1,s_1,True)
show_latex(1,s_1,False)

```


# Task 2: General Hartree-Fock solver

Having set up an initial single-particle basis set $\{\chi_{\alpha}\}_{\alpha = 1}^{l}$ we now set out to find the ground state of the two-body fermion problem.
The full time-independent Hamiltonian used by {cite}`zanghellini_2004` is given by
\begin{align}
    \hat{H}
    = \hat{h} + \hat{u}
    = \sum_{i = 1}^{n}\left(
        -\frac{1}{2}\frac{\text{d}^2}{\text{d}x^2}
        + \frac{1}{2}\omega^2 x^2
    \right)
    + \sum_{i = 1}^{n} \sum_{j = i + 1}^{n} \frac{\alpha}{\sqrt{(x_i - x_j)^2 + a^2}},
\end{align}
where the number of particles $n = 2$, $a = 0.25$, $\alpha = 1$ (not to be confused with the index $\alpha$ in the basis of harmonic oscillator eigenstates), and $\omega = 0.25$.

We start from the time-independent Schrödinger equation
\begin{align}
    \hat{H} | \Psi \rangle = E | \Psi \rangle,
\end{align}
where $| \Psi \rangle$ is a many-body wave function.
We know that in the non-interacting case the exact ground state of the Hamiltonian will be a single Slater determinant (which we denote $| \Phi \rangle$) of the lowest single-particle eigenstates.
This motivates the ansatz of using a single Slater determinant for the full many-body problem
\begin{align}
    | \Psi \rangle \approx | \Phi \rangle
    = | \phi_1 \phi_2 \dots \phi_n \rangle,
\end{align}
where the $n$ lowest _molecular orbitals_ (MO) (also known as the Hartree-Fock orbitals) from $\{\phi_p\}_{p = 1}^{k}$ ($k \leq n$) are the primary unknowns.
```{note}
We use a convention where split up the set of molecular orbitals into a set of _occupied_ and _virtual_ orbitals.
That is
\begin{align}
    \{ \phi_p \}_{p = 1}^{k}
    = \{ \phi_i \}_{i = 1}^{n} \cup \{ \phi_a \}_{a = n + 1}^{k},
\end{align}
where we let the indices $i, j, k, l, \dots \in \{1, \dots, n\}$ denote the occupied orbitals, i.e., the orbitals that are contained in the Hartree-Fock Slater determinant, $a, b, c, d, \dots \in \{n + 1, \dots, k \}$ are the indices of the virtual orbitals that are not contained in the reference determinant, and $p, q, r, s, \dots \in \{1, \dots, k\}$ the full set of orbitals.
```
The Hartree-Fock method method finds the Slater determinant that minimizes the energy, i.e., gets closest to the "true" many-body ground state energy $E_{gs}$ of the problem using the variational principle
\begin{align}
    E_{gs} \leq E[\Phi, \Phi^{*}] = E[\phi_1, \dots, \phi_n] = \langle \Phi | \hat{H} | \Phi \rangle,
\end{align}
subject to the constraint that the molecular orbitals are orthonormal, $\langle \phi_i | \phi_j \rangle = \delta_{ij}$ ($\implies \langle \Phi | \Phi \rangle = 1$).
To include this requirement in the variational principle we use Lagrange's method of undetermined multipliers

$$
    L[\phi_1, \dots, \phi_n, \boldsymbol{\lambda}]
    = E[\phi_1, \dots, \phi_n]
    - \lambda_{qp} \left(
        \langle \phi_p | \phi_q \rangle
        - \delta_{pq}
    \right),
$$ (eq:hf-lagrangian)
where $\boldsymbol{\lambda}$ are the Lagrange multipliers.

```{note}
Add derivation of the (canonical) GHF-equations.
```

Minimization of the Lagrangian in {eq}`eq:hf-lagrangian` yields the Hartree-Fock eigenvalue equation

$$
    \hat{f} | \phi_p \rangle = \varepsilon_p | \phi_p \rangle,
$$ (eq:canonical-hf-equation)
where $\hat{f}$ is the single-particle Fock-operator.
Note that this equation is not limited to the occupied states, but to all the molecular orbitals.
The matrix elements of the single-particle Fock operator are given by
\begin{align}
    \langle \psi | \hat{f} | \xi \rangle
    = \langle \psi | \hat{h} | \xi \rangle
    + \sum_{i = 1}^{n} \langle \psi \phi_i | \hat{u} | \xi \phi_i \rangle
    - \sum_{i = 1}^{n} \langle \psi \phi_i | \hat{u} | \phi_i \xi \rangle
    = \langle \psi | \hat{h} | \xi \rangle
    + \sum_{i = 1}^{n} \langle \psi \phi_i | \hat{u} | \xi \phi_i \rangle_{AS},
\end{align}
where $|\psi\rangle$ and $|\xi \rangle$ are two arbitrary single-particle states.

## The Roothan-Hall equations

The way we solve the Hartree-Fock equation in {eq}`eq:canonical-hf-equation` is by expanding the Hartree-Fock orbitals $\{ \phi_p \}_{p = 1}^k$ in an atomic orbital basis $\{ \psi_{\mu} \}_{\mu = 1}^{l}$ (which we here take to be the orthonormal harmonic oscillator basis states with spin-doubling), and then projecting the equations onto the known basis set.
```{note}
In general $k \leq l$ for the molecular orbitals.
The consequence of this is that Hartree-Fock can be used as a method to truncate the basis set.
This is common when using computationally more complex post Hartree-Fock methods such as e.g., coupled-cluster, to lower simulation times.
However, in this project we limit our attention to $k = l$.
```
We express a basis transformation from the AO basis to the MO basis by

$$
    |\phi_p\rangle = \sum_{\mu = 1}^{l} C_{\mu p} | \psi_{\mu} \rangle,
$$ (eq:basis-transformation)
where the coefficients $C_{\mu p}$ now become our primary unknowns in the Hartree-Fock method.
Projecting the AO basis onto equation {eq}`eq:canonical-hf-equation` we have
\begin{gather}
    \langle \psi_{\mu} | \hat{f} | \phi_p \rangle
    = \varepsilon_{p} \langle \psi_{\mu} | \phi_p \rangle
    \implies
    \sum_{\nu = 1}^{l} f_{\mu \nu} C_{\nu p}
    = \varepsilon_{p} \sum_{\nu = 1}^{l} \delta_{\mu \nu} C_{\nu p}
    =  C_{\mu p} \varepsilon_{p}
    \\
    \implies
    \mathbf{F} \mathbf{C}
    = \mathbf{C} \boldsymbol{\varepsilon},
\end{gather}
where $[\mathbf{F}]_{\mu \nu} = f_{\mu \nu} = \langle \psi_{\mu} | \hat{f} | \psi_{\nu} \rangle$ are the Fock matrix elements in the AO basis, $[\mathbf{C}]_{\mu p} = C_{\mu p}$ the basis transformation coefficient matrix, and $\boldsymbol{\varepsilon} = [\varepsilon_1, \dots, \varepsilon_k]^T$ the vector of Hartree-Fock single-particle eigenenergies (not to be confused with the full many-body energy).
The last eigenvalue equation is known as the Roothan-Hall equations.
Here both $\mathbf{C}$ and $\boldsymbol{\varepsilon}$ are the primary unknowns.
We now need to construct the Fock matrix in the atomic orbital basis, that is
\begin{align}
    f_{\mu \nu}
    &= \langle \psi_{\mu} | \hat{f} | \psi_{\nu} \rangle
    = \langle \psi_{\mu} | \hat{h} | \psi_{\nu} \rangle
    + \sum_{i = 1}^{n}
    \langle \psi_{\mu} \phi_i | \hat{u} | \psi_{\nu} \phi_i \rangle_{AS}.
\end{align}
Replacing the sum over the occupied molecular orbitals in the two-body elements by equation {eq}`eq:basis-transformation` we get
\begin{align}
    f_{\mu \nu}
    &= \langle \psi_{\mu} | \hat{h} | \psi_{\nu} \rangle
    + \sum_{i = 1}^{n}
    C^{*}_{\kappa i} C_{\lambda i}
    \langle \psi_{\mu} \psi_{\kappa} | \hat{u} | \psi_{\nu} \psi_{\lambda} \rangle_{AS}
    = h_{\mu \nu}
    + D_{\lambda \kappa}
    u^{\mu \kappa}_{\nu \lambda},
\end{align}
where we have defined the density matrix $D_{\lambda \kappa} = \sum_{i = 1}^{n} C^{*}_{\kappa i} C_{\lambda i}$.

A complicating factor with the Roothan-Hall equations is that the Fock matrix $\mathbf{F}$ depends on the coefficient matrix $\mathbf{C}$.
To get around this we use a technique called self-consistent field iterations.


# Session 6:
## Machine learning for causal inference

## Agenda

1. [A general problem](#A-general-problem)
1. [Inference with Lasso](#Inference-with-Lasso)
1. [Double Machine Learning](#Double-Machine-Learning)

# A general problem

## Estimate treatment effect with selection

Often we are interested in estimating average treatment effect (ATE) of $T$ on $y$ in observational data.

- Matching has for a long time been the defacto standard
  - (assuming we measure enough variables to assignment conditionally random)
- Can we somehow solve the problem using machine learning?

## Estimate treatment effect with selection
[Chernozhukov et al. (2018)](http://economics.mit.edu/files/12538) assume that we have the following data generating process
\begin{align}
y=& T\theta_0+g_0(X)+U\\
T=&m_0(X) + V\\
E[U|X,T]=&0\\
E[V|X]=&0\\
\end{align}

Basic model properties:
- The outcome $y$ is confounded by unknown nuisance function $g_0(\cdot)$
- The treatment $T$ suffer from selection on observable, where $m_0$ is  unknown propensity function
  - Note assumed no selection on unobservables (only "mild" econometric problem)

## Estimate treatment effect with selection

There have been multiple ways proposed to solve the problem:
- *Directly* modify ML to allow for estimation of treatment effect
- *Indirectly* modify estimation procedure to incorporate ML

# Inference with Lasso

## Treatment effects in linear models

Suppose we want to estimate a linear model parameter for the causal effect of treatment $T_i$ on $y_i$.

\begin{equation}y_i=\alpha T_i+x_i\beta+r_{yi}+\zeta_i\end{equation}

- We follow notation in [Belloni et al., 2015](https://doi.org/10.1257/jep.28.2.29)
- We let $r_{yi}$ be an approximation error (we don't know the functional form)

## Treatment effects in linear models (2)

How to select model, i.e. subset of $x$?

- Classic econometrics:
  - Use **OLS** and include covariates based on theory or inference
  - Problem how to delete covariates systematically? Adjust for multiple hypothesis testing?

- Machine learning:
  - Use **LASSO** to perform covariate selection
  - Note - estimates are biased towards zero!
    - Problem we omit potentially relevant variables!!
    - LASSO excludes  possible confounders if little predictive power $y_i$.
    - Excluded variables may still have an effect through $T_i$, e.g. covariates correlated with treatment.

## Fixing the LASSO

A simple solution suggested by [Belloni et al. (2015)](https://doi.org/10.1257/jep.28.2.29) us to use Post-LASSO to correct for bias:
- Step 1: estimate two LASSO models
    - a) Regress $y_i$ on $x_i$
    - b) Regress $T_i$ on $x_i$
- Step 2: run OLS using only variables that were kept in either LASSOs

What about inference?

- We need further assumptions on sparsity.
- See [Chernozhukov et al. (2015)](https://doi.org/10.1257/aer.p20151022) online appendix for details.

## Fixing the LASSO (2)
The LASSO picks the correct variables even if there are more variables than observations!
- In high dimensions this does not work - can return more variables to use estimate than possible in OLS

# Double Machine Learning

## Linear DML

[Belloni et al. (2015)](https://doi.org/10.1257/jep.28.2.29), [Chernozhukov et al. (2015)](https://doi.org/10.1257/aer.p20151022) write down the two "prediction equations":

\begin{align}
y_i=&\alpha T_i+x_i^{\prime}\theta_y+r_{yi}+\zeta_i\\
T_{i}=&x_{i}^{\prime} \theta_{t}+r_{t i}+v_{i}
\end{align}

The two equations can be combined into a single structural equation (substite $T_i$ into $y_i$):

\begin{align}y_{i}=&x_{i}^{\prime}\left(\alpha \theta_{t}+\theta_{y}\right)+\left(\alpha r_{t i}+r_{y i}\right)+\left(\alpha v_{i}+\zeta_{i}\right)\\=&x_{i}^{\prime} \pi+r_{c i}+\varepsilon_{i}\end{align}

## Linear DML (2)

[Chernozhukov et al. (2015)](https://doi.org/10.1257/aer.p20151022) states the following algorithm for :

1. run the two LASSO equations (as in POST-LASSO) and obtain residuals
  - $\hat{\rho}_i^y$ from $y_i$ on $x_i$
  - $\hat{\rho}_i^d$ from $T_i$ on $x_i$
1. run a regression of $\hat{\rho}_i^y$ on $\hat{\rho}_i^d$

What is the intuition?

- Similar to Frisch-Waugh-Lowell where we partial out effects.
  - We partial out effect of $x_i$ on both $y_i$ and on $T_i$ seperately
- Innovation: We make double selection of variables using LASSO

## Linear DML (3)

[Belloni et al. (2015)](https://doi.org/10.1257/jep.28.2.29) simulates the performance of post selection estimators

## Naive solution

What happens when use machine learning estimator to estimate directly estimate in $\theta_0$ and $g_0(\cdot)$ in $y=T\theta_0+g_0(X)$ to control for confounders?
- Where we use a an auxiliary subsample $I^c$ to compute $\hat{g}_0(\cdot)$ using a possibly non-linear model.
- Assume subsample is half of the sample size.

$$\hat{\theta}_0=\frac{\frac{1}{n}\sum_{i\in I}T_i(y_i-\hat{g}_0(X_i))}{\frac{1}{n}\sum_{i\in I}T_i^2}$$

## Naive solution

We decompose estimator into scaled estimation error

$$\sqrt{n}(\theta-\hat{\theta}_0)=
\frac{\frac{1}{\sqrt{n}}\sum_{i\in I}T_iU_i}{\frac{1}{n}\sum_{i\in I}T_i^2}+\frac{\frac{1}{\sqrt{n}}\sum_{i\in I}T_i(g_0-\hat{g}_0(X_i))}{\frac{1}{n}\sum_{i\in I}T_i^2}
$$

What could be problematic here?

## Naive problem

Issue is that $\hat{g}$ will be systematically biased as we are curbing overfitting, e.g. through regularization.
- Same problem arises for tree-based and neural network models.
- Estimator will have bias term that diverges and is not centered:

$$\frac{\frac{1}{\sqrt{n}}\sum_{i\in I}T_i(g_0-\hat{g}_0(X_i))}{E[T_i^2]}
$$

## Orthogonalization

Suppose we also estimate $\hat{m}_0(\cdot)$ on the auxiliary sample $I^c$. We can then make the following estimate:

$$\check{\theta}_0=\frac{\frac{1}{n}\sum_{i\in I}\hat{V}_i(y_i-\hat{g}_0(X_i))}{\frac{1}{n}\sum_{i\in I}T_i^2}$$

where we use the

## Orthogonalization

We decompose estimator into scaled estimation error

$$\sqrt{n}(\theta-\hat{\theta}_0)=
\frac{\frac{1}{\sqrt{n}}\sum_{i\in I}T_iU_i}{\frac{1}{n}\sum_{i\in I}T_i^2}+\frac{\frac{1}{\sqrt{n}}\sum_{i\in I}(m_0(X_i)-\hat{m}_0(X_i))(g_0-\hat{g}_0(X_i))}{\frac{1}{n}\sum_{i\in I}T_i^2}
$$

This solves the problem as the product of estimation errors vanishes.

## Orthogonalization

The first major contribution of [Chernozhukov et al. (2018)](http://economics.mit.edu/files/12538) is to show that in general the second double debiasing procedure leads to consistent estimates and can be used estimate average treatment effects.

- The proof depends on sample splitting - using an independent auxiliary sample for estimating $\hat{m}_0,\hat{g}_0$.

## Implementation details

Problem - what to do with auxiliary sample?

To gain efficient estimates [Chernozhukov et al. (2018)](http://economics.mit.edu/files/12538)
- Problem - what to do with auxiliary sample?
- We rotate sample using **cross-fitting**: first use one part as auxiliary sample, then the other. Like cross-validation in supervised ML.
- This is second major contribution.

## Implementation details

How do we estimate $\hat{m}_0,\hat{g}_0$? This can be done using cross-validation on auxiliary sample $I^c$. Available estimators:

- linear/logistic models, including regularized
- tree based inclding random forests, boosted trees
- neural networks
- kernel models including suppert vector machine

## Extensions

The DML approach is extended in the paper to:
- compute Local Average Treatment Effects (LATE)
- compute Instrumental Variables

## The end


# Cycling at the Tour de France

```
red,lime,blue,green,purple = "#CF5970","#94B848","#878CC0","#5EBAA1","#BD5DC7"
```

## Define all the constant parameters of the model

```
g,mu,m,rho,CA = 9.8,0.003,77,1.1,0.35
```

### Define the ODE integrator

\begin{align}
v[n] &=  v[n-1] + \Delta t \Big\{\frac{P}{m v[n-1]} - \frac{1}{2 m}C_D \rho A v[n-1]^2 - g (\mu_r \cos\theta + \sin\theta) \Big\} \\
x[n] &= x[n-1] + v[n]\Delta t \\
\end{align}

```
def integrate(t0,x0,v0,dt,d,P,th):
    ''' Return the time, position and velocity arrays for a stage segment
        at the Tour de France.'''

    # the initial conditions
    cx = [x0]
    cv = [v0]
    ct = [t0]
    n = 0
    while cx[n] <= d:
        n += 1
        # construct the time array
        ct.append(ct[n-1]+dt)

        # all the forces acting on the rider
        F_rider = P/(m*cv[n-1])
        F_air = (CA*rho/(2.0*m))*cv[n-1]**2
        F_g = g*sin(th)
        F_friction = mu*g*cos(th)

        # numerically integrate the equations of motion
        cv.append(cv[n-1] + dt*(F_rider - F_air - F_g - F_friction))
        cx.append(cx[n-1] + dt*cv[n])

    return ct,cx,cv
```

## Solve for the first part of the stage

```
t,x,v = integrate(0.0,0.0,6.0,0.01,1500,325,2.0*pi*0.8/360)
```

### Graph the motion $v(t)$ and $x(t)$

```
figure(figsize=(10,5))
plot(t,x,'-', color=blue, linewidth=4)
xlabel('time (s)')
ylabel('$x$ (m)')
tight_layout()
savefig('bike_x_vs_t.pdf')
```

```
figure(figsize=(10,5))
plot(t,v,'-', color=green, linewidth=4)
xlabel('time (s)')
ylabel('$v$ (m/s)')
tight_layout()
savefig('bike_v_vs_t.pdf')
```

## Putting all the segments together

### The distances and heights

```
d = [1500,2500,2000,3500,3000,3000]
y = [20,237,163,275,265,170]
```

### Compute the angles

```
# store the angles in radians, but display in degrees
theta = [arcsin(1.0*y[n]/d[n]) for n in range(len(d))]

for n,th in enumerate(theta):
    print "theta_%d = %4.2f degrees" % (n+1,th*360/(2.0*pi))
```

    theta_1 = 0.76 degrees
    theta_2 = 5.44 degrees
    theta_3 = 4.67 degrees
    theta_4 = 4.51 degrees
    theta_5 = 5.07 degrees
    theta_6 = 3.25 degrees

### Define the angle dependent power function

```
def power(theta):
    ''' The power output of an elite cyclist for different inclines.'''
    th = 360*theta/(2.0*pi)
    if th < 3.55:
        return 325.0
    elif theta < 5.16:
        return 425.0
    else:
        return 500.0
```

## Integrate over all stages to get the position and velocity and total stage time

```
dt = 0.1 # s
t = []
x = []
v = []
t0 = 0.0 #s
x0 = 0.0 # m
v0 = 6.0 # m/s
print "%14s\t%14s\t%14s" % ("time (min)","distance (km)","speed (m/s)")
for n,cd in enumerate(d):
    ct,cx,cv = integrate(t0,x0,v0,dt,sum(d[:n+1]),power(theta[n]),theta[n])
    t.extend(ct)
    x.extend(cx)
    v.extend(cv)
    t0 = t[-1]
    x0 = x[-1]
    v0 = v[-1]
    print "%14.2f\t%14.2f\t%14.2f" % (t0/60,x0/1000,v0)
```

        time (min)	 distance (km)	   speed (m/s)
              2.54	          1.50	         10.13
             10.23	          4.00	          5.36
             15.78	          6.00	          6.01
             25.24	          9.50	          6.17
             34.06	         12.50	          5.66
             42.14	         15.50	          6.20

## The full stage 16 plots

```
figure(figsize=(10,5))
plot(array(t)/60,array(x)/100,'-', color=blue, linewidth=4)
xlabel('time (min)')
ylabel('$x$ (km)')
tight_layout()
savefig('stage_16_x_vs_t.pdf')
```

```
figure(figsize=(10,5))
plot(array(t)/60,array(v),'-', color=green, linewidth=4)
xlabel('time (min)')
ylabel('$v$ (m/s)')
tight_layout()
savefig('stage_16_v_vs_t.pdf')
```