```python
import matplotlib.pyplot as pl
import matplotlib as mpl
import numpy as np
from matplotlib.patches import Polygon
from sympy.solvers import solve
```

```python
# Kinematics
Q = 91
qT = 10
y = 1
M = np.sqrt(Q**2+qT**2)

etamin = -2.4
etamax = 2.4
pTmin = 20
```

```python
# Define generalised theta function
def GenTheta(eta, x):
    pTo = Q**2 / 2 / ( M * np.cosh(eta - y) - qT * x)
    etao = np.log( ( M * np.exp(y) - pTo * np.exp(eta) ) / ( M * np.exp(-y) - pTo * np.exp(-eta) ) ) / 2
    hs1 = np.heaviside(pTo - pTmin, 1.)
    hs2 = np.heaviside(eta - etamin, 1.) * np.heaviside(etamax - eta, 1.)
    hs3 = np.heaviside( np.sqrt(qT**2 + pTo**2 - 2 * qT * pTo * x) - pTmin, 1. )
    hs4 = np.heaviside(etao - etamin, 1.) * np.heaviside(etamax - etao, 1.)
    return hs1 * hs2 * hs3 * hs4
```

```python
# Solution obtained with SymPy (Solutions.ipynb)
def f2(eta, pTlim):
    return ( 2 * M * pTlim * np.cosh(eta - y) - Q**2 ) / ( 2 * pTlim * qT )
```

```python
# Solution obtained with SymPy (Solutions.ipynb)
def f3(eta, etalim):
    return M * np.cosh(eta - y) / qT - Q**2 * np.sinh(eta - etalim) / ( 2 * qT * M * np.sinh(y - etalim) )
```

```python
# Solution obtained with SymPy (Solutions.ipynb)
def f4(eta, pTlim):
    return ( M * np.cosh(eta - y) * ( Q**2 - 2 * pTlim**2 + 2 * qT**2 )
            - Q**2 * np.sqrt( M**2 * np.sinh(eta - y)**2 + pTlim**2) ) / ( 2 * qT * ( M**2 - pTlim**2 ) )
```

```python
## Define plot parameters
mpl.style.use('seaborn-talk')
mpl.rc('axes', linewidth=2)
mpl.rc("font", **{"family": "serif", "serif": ["Computer Modern"]})
mpl.rc("text", usetex = True)
mpl.rcParams["xtick.direction"] = "in"
mpl.rcParams["ytick.direction"] = "in"
mpl.rcParams["xtick.top"] = True
mpl.rcParams["ytick.right"] = True
mpl.rcParams["axes.linewidth"] = 1.3
mpl.rcParams['xtick.major.size'] = 6
mpl.rcParams['xtick.major.width'] = 1.3
mpl.rcParams['xtick.minor.size'] = 3
mpl.rcParams['xtick.minor.width'] = 1.3
mpl.rcParams['ytick.major.size'] = 6
mpl.rcParams['ytick.major.width'] = 1.3
mpl.rcParams['ytick.minor.size'] = 3
mpl.rcParams['ytick.minor.width'] = 1.3

ax = pl.gca()

# Axis labels
pl.xlabel(r'$\eta$', fontsize=22)
pl.ylabel(r'$\cos\phi$', fontsize=22, labelpad=-2)

# Title
pl.title(r"$Q$ = " + str(Q) + " GeV, $|\mathbf{q}_T|$ = " + str(qT) + " GeV, $y$ = " + str(y)
        + r", $p_{T,\rm min}=$ " + str(pTmin) + r" GeV, $\eta_{\rm min}=$ $" + str(etamin)
        + r"$, $\eta_{\rm max}=$ " + str(etamax),
         fontsize = 16)

pl.xticks(size=20)
pl.yticks(size=20)

# Adjust canvas
pl.xlim(-1.2 + etamin, 1.2 + etamax)
pl.ylim(-1.2, 1.6)

# Labels on plot
ax.text(etamin-0.2, -1.5,  r'$\eta_{\rm min}$', fontsize=22)
ax.text(etamax-0.1, -1.5,  r'$\eta_{\rm max}$', fontsize=22)

# Points in eta and x = cos\phi
n = 500
alleta = np.linspace(- 1.2 + etamin, 1.2 + etamax, num = n)
allx   = np.linspace(-1, 1, num = n)
E, X = np.meshgrid(alleta, allx)
T = GenTheta(E, X)

# Countour plot
pl.contourf(E, X, T, levels = [0.0001, 1.], colors = ["red", "blue"], alpha = 0.2)

# Shaded region
verts = [(-10*etamax, -1)] + [(10*etamax, -1)] + [(10*etamax, 1)] + [(-10*etamax, 1)]
poly = Polygon(verts, color = "gray", alpha = 0.1)
ax.add_patch(poly)

# Dashed lines
pl.axhline(y=-1, xmin=0, xmax=1, linewidth=1, color = 'k', linestyle='--')
pl.axhline(y=1, xmin=0, xmax=1, linewidth=1, color = 'k', linestyle='--')

# Theta 1
pl.axvline(x=etamin, ymin=0, ymax=1, linewidth=2, color = 'k', linestyle='-')
pl.axvline(x=etamax, ymin=0, ymax=1, linewidth=2, color = 'k', linestyle='-')

# Theta 2
pl.plot(alleta, f2(alleta, pTmin), 'r', linewidth=2, label=r"$f^{(2)}(\eta,p_{T,\rm min})$")

# Theta 3
pl.plot(alleta, f3(alleta, etamin), 'b', linewidth=2, label=r"$f^{(3)}(\eta,\eta_{\rm min})$")
pl.plot(alleta, f3(alleta, etamax), 'b', linewidth=2, label=r"$f^{(3)}(\eta,\eta_{\rm max})$", linestyle='--')

# Theta 4
pl.plot(alleta, f4(alleta, pTmin), 'g', linewidth=2, label=r"$f^{(4)}(\eta, p_{T,\rm min})$")

pl.legend(loc = 'upper left', ncol = 2, fontsize = 18)
pl.savefig("../plots/IntDomain.pdf")
```

```python
## Redefine Kinematics
Q = 91
qT = 10
y = 3
M = np.sqrt(Q**2+qT**2)

etamin = 2
etamax = 4.5
pTmin = 20
```

```python
## Define plot parameters
pl.style.use('seaborn-talk')
pl.rc('axes', linewidth=2)
ax = pl.gca()

# Axis labels
pl.xlabel(r'$\eta$', fontsize=22)
pl.ylabel(r'$\cos\phi$', fontsize=22, labelpad=-2)

# Title
pl.title(r"$Q$ = " + str(Q) + " GeV, $|\mathbf{q}_T|$ = " + str(qT) + " GeV, $y$ = " + str(y)
        + r", $p_{T,\rm min}=$ " + str(pTmin) + r" GeV, $\eta_{\rm min}=$ $" + str(etamin)
        + r"$, $\eta_{\rm max}=$ " + str(etamax),
         fontsize = 16)

pl.xticks(size=20)
pl.yticks(size=20)

# Adjust canvas
pl.xlim(-1.2 + etamin, 1.2 + etamax)
pl.ylim(-1.2, 1.6)

# Labels on plot
ax.text(etamin-0.2, -1.5,  r'$\eta_{\rm min}$', fontsize=22)
ax.text(etamax-0.1, -1.5,  r'$\eta_{\rm max}$', fontsize=22)

# Points in eta and x = cos\phi
n = 500
alleta = np.linspace(- 1.2 + etamin, 1.2 + etamax, num = n)
allx   = np.linspace(-1, 1, num = n)
E, X = np.meshgrid(alleta, allx)
T = GenTheta(E, X)

# Countour plot
pl.contourf(E, X, T, levels = [0.0001, 1.], colors = ["red", "blue"], alpha = 0.2)

# Shaded region
verts = [(-10*etamax, -1)] + [(10*etamax, -1)] + [(10*etamax, 1)] + [(-10*etamax, 1)]
poly = Polygon(verts, color = "gray", alpha = 0.1)
ax.add_patch(poly)

# Dashed lines
pl.axhline(y=-1, xmin=0, xmax=1, linewidth=1, color = 'k', linestyle='--')
pl.axhline(y=1, xmin=0, xmax=1, linewidth=1, color = 'k', linestyle='--')

# Theta 1
pl.axvline(x=etamin, ymin=0, ymax=1, linewidth=2, color = 'k', linestyle='-')
pl.axvline(x=etamax, ymin=0, ymax=1, linewidth=2, color = 'k', linestyle='-')

# Theta 2
pl.plot(alleta, f2(alleta, pTmin), 'r', linewidth=2, label=r"$f^{(2)}(\eta,p_{T,\rm min})$")

# Theta 3
pl.plot(alleta, f3(alleta, etamin), 'b', linewidth=2, label=r"$f^{(3)}(\eta,\eta_{\rm min})$")
pl.plot(alleta, f3(alleta, etamax), 'b', linewidth=2, label=r"$f^{(3)}(\eta,\eta_{\rm max})$", linestyle='--')

# Theta 4
pl.plot(alleta, f4(alleta, pTmin), 'g', linewidth=2, label=r"$f^{(4)}(\eta, p_{T,\rm min})$")

pl.legend(loc = 'upper left', ncol = 2, fontsize = 18)
pl.savefig("../plots/IntDomainAsy.pdf")
```

```python

```


# Thesis Notebook - Sebastian Gsell

This notebook provides the interface to the estimation of Marginal Treatment Effects (MTE) via *grmpy_semipar*.
First, I replicate the semiparametric results from

> Carneiro, P., Heckman, J. J., & Vytlacil, E. J. (2011). [Estimating marginal returns to education.](https://pubs.aeaweb.org/doi/pdfplus/10.1257/aer.101.6.2754) *American Economic Review, 101*(6), 2754-81.

The authors analyze the returns to college for white males born between 1957 and 1963 using data from the National Longitudinal Survey of Youth 1979. The authors provide some [replication material]((https://pubs.aeaweb.org/doi/pdfplus/10.1257/aer.101.6.2754)) on their website but do not include geographic identifiers. Therefore, I make use of a mock data merging background characteristics and local data randomly.

Below, I'll also present the results of my empirical analysis of the GSOEP. Using the method of Local Instrumental Variables (LIV), I'll show that returns to higher education in Germany vary considerable, as indiviudals select into university based on their unobserved benefits.

In a future update, the semiparametric estimation method will be included in the open-source package *grmpy* for the simulation and estimation of the generalized Roy model in Python. Currently, *grmpy* is limited to the estimation of a parametric normal version of the generalized Roy model.
 For more, see the [online documentation](https://grmpy.readthedocs.io/en/develop/).

## 0) Imports

```python
import numpy as np
import warnings

from semipar.estimation.estimate import fit

from thesis_auxiliary import plot_common_support
from thesis_auxiliary import plot_mte_carneiro
from thesis_auxiliary import plot_mte_soep
from thesis_auxiliary import plot_mte_par

warnings.filterwarnings('ignore')
```

## 1) Local Instrumental Variables

The method of Local Instrumental Variables (LIV) is based on the generalized Roy model, which is characterized by the following equations:

\begin{align*}
 &\textbf{Potential Outcomes} & & \textbf{Choice}  &\\
 & Y_1 = \beta_1 X + U_{1} &  & I = Z \gamma - V &\\
 & Y_0 = \beta_0 X + U_{0} &  & D_i = \left\{
\begin{array}{ll}
1 & if \ I > 0 \\
0 & if \ I \leq 0\\
\end{array}
\right.
&&&&\\
& \textbf{Observed Outcome} &&&\\
& Y = D Y_1 + (1-D) Y_0 &&&
\end{align*}

We work with the linear-in-the-parameters version of the generalized Roy model:

\begin{align}
E(Y|X = \overline{x}, P(Z) = p) =  \overline{x} \beta_0 + p \overline{x} (\beta_1 - \beta_0) + K(p),
\end{align}

where $K(p) = E(U_1 - U_0 | D = 1, P(Z) = p)$ is a nonlinear function of $p$ that captures heterogeneity along the unobservable resistance to treatment $u_D$.

In addition, assume that $(X, Z)$ is independent of $\{U_1, U_0, V\}$. Then, the MTE is

1) additively separable in $X$ and $U_D$, which means that the shape of the MTE is independent of $X$, and

2) identified over the common support of $P(Z)$, unconditional on $X$.

The common support, $P(Z)$, plays a crucial role for the identification of the MTE.
It denotes the probability of going to university ($D=1$). Common support is defined as the intersection of the support of $P(Z)$ given $D = 1$ and the support of $P(Z)$ given $D = 0$. i.e., those evaluations of $P(Z)$ for which we obtain positive frequencies in both subsamples. We will plot it below. The larger the common support, the larger the region over which the MTE is identified.

The LIV estimator, $\Delta^{LIV}$, is derived as follows (Heckman and Vytlacil [2001](https://www.aeaweb.org/articles?id=10.1257/aer.91.2.107), [2005](https://www.jstor.org/stable/3598865?seq=1#page_scan_tab_contents)):

\begin{equation}
\begin{split}
\Delta^{LIV} (\overline{x}, u_D) &= \frac{\partial E(Y|X = \overline{x}, P(Z) = p)}{\partial p}  \bigg\rvert_{p = u_D} \\
& \\
&= \overline{x}(\beta_1 - \beta_0) + E(U_1 - U_0 | U_D = u_D) \\
&\\
& = \underbrace{\overline{x}(\beta_1 - \beta_0)}_{\substack{observable \\ component}} + \underbrace{\frac{\partial K}{\partial p} \bigg\rvert_{p = u_D}}_{\substack{k(p): \ unobservable \\ component}} = MTE(\overline{x}, u_D)
\end{split}
%\frac{[E(U_1 - U_0 | U_D \leq p] p}{\partial p}  \bigg\rvert_{p = u_D}
%E(U_1 - U_0 | U_D = u_D)
\end{equation}

Since we do not make any assumption about the functional form of the unobservables, we estimate $k(p)$ non-parametrically. In particualr, $k(p)$ is the first derivative of a locally quadratic kernel regression.

## 2) The Initialization File

For the semiparametric estimation, we need information on the following sections:

* __ESTIMATION__: Specify the dependent (wage) and indicator variable (treatment dummy) of the input data frame.
For the estimation of the propensity score $P(Z)$, we choose a probability model, here logit. Furthermore, we select 30 bins to determine the common support in the treated and untreated subsamples. For the locally quadratic regression, we follow the specification of [Carneiro et al. (2011)](https://pubs.aeaweb.org/doi/pdfplus/10.1257/aer.101.6.2754) and choose a bandwidth of 0.322. The respective gridsize for the locally quadratic regression is set to 500. [Fan and Marron (1994)](https://www.tandfonline.com/doi/abs/10.1080/10618600.1994.10474629) find that a gridsize of 400 is a good default for graphical analysis. Since the data set is large (1785 observations) and the kernel regression function has a very low runtime, I increase the gridsize to 500. Setting it to the default or increasing it even more does not affect the final MTE. The other parameters in this section are set by default and, normally, do not need to be adjusted.

* __TREATED, UNTREATED, CHOICE__: In this section, the variables of the outcome equations (treated, untreated) and the college decision (choice) are specified.

* __DIST__: The distribution of the unobservables is not of relevance in the semiparametric apporach and can be ignored.

```python
%%file notebook/files/replication_carneiro.yml
---
ESTIMATION:
    file: notebook/data/aer-replication-mock.pkl
    dependent: wage
    indicator: state
    semipar: True
    show_output: True
    logit: True
    nbins: 30
    bandwidth: 0.322
    gridsize: 500
    trim_support: True
    reestimate_p: False
    rbandwidth: 0.05
    derivative: 1
    degree: 2
    ps_range: [0.005, 0.995]
TREATED:
    order:
    - exp
    - expsq
    - lwage5
    - lurate
    - cafqt
    - cafqtsq
    - mhgc
    - mhgcsq
    - numsibs
    - numsibssq
    - urban14
    - lavlocwage17
    - lavlocwage17sq
    - avurate
    - avuratesq
    - d57
    - d58
    - d59
    - d60
    - d61
    - d62
    - d63
UNTREATED:
    order:
    - exp
    - expsq
    - lwage5
    - lurate
    - cafqt
    - cafqtsq
    - mhgc
    - mhgcsq
    - numsibs
    - numsibssq
    - urban14
    - lavlocwage17
    - lavlocwage17sq
    - avurate
    - avuratesq
    - d57
    - d58
    - d59
    - d60
    - d61
    - d62
    - d63
CHOICE:
    params:
    - 1.0
    order:
    - const
    - cafqt
    - cafqtsq
    - mhgc
    - mhgcsq
    - numsibs
    - numsibssq
    - urban14
    - lavlocwage17
    - lavlocwage17sq
    - avurate
    - avuratesq
    - d57
    - d58
    - d59
    - d60
    - d61
    - d62
    - d63
    - lwage5_17numsibs
    - lwage5_17mhgc
    - lwage5_17cafqt
    - lwage5_17
    - lurate_17
    - lurate_17numsibs
    - lurate_17mhgc
    - lurate_17cafqt
    - tuit4c
    - tuit4cnumsibs
    - tuit4cmhgc
    - tuit4ccafqt
    - pub4
    - pub4numsibs
    - pub4mhgc
    - pub4cafqt
DIST:
    params:
    - 0.1
    - 0.0
    - 0.0
    - 0.1
    - 0.0
    - 1.0
```

    Overwriting notebook/files/replication_carneiro.yml

Note that I do not include a constant in the __TREATED, UNTREATED__ section. The reason for this is that in the semiparametric setup, $\beta_1$ and $\beta_0$ are determined by running a Double Residual Regression without an intercept:
	$$ e_Y =e_X \beta_0 \ + \ e_{X \ \times \ p} (\beta_1 - \beta_0) \ + \ \epsilon $$

where $e_X$, $e_{X \ \times \ p}$, and $e_Y$ are the residuals of a local linear regression of $X$, $X$ x $p$, and $Y$ on $\widehat{P}(Z)$.

We now proceed to the replication.

## 2) Replication of Carneiro et al. (2011)

Conduct the estimation based on the initialization file.

```python
rslt = fit("notebook/files/replication_carneiro.yml", semipar=True)
```

The rslt dictionary contains information on the estimated parameters and the final MTE.

```python
list(rslt)
```

    ['quantiles', 'mte', 'mte_x', 'mte_u', 'mte_min', 'mte_max', 'X', 'b1-b0']

Before plotting the MTE, let's see what else we can learn.
For instance, we can account for the variation in $X$.

Note that we divide the MTE by 4 to investigate the effect of one additional year of college education.

```python
np.min(rslt['mte_min']) / 4, np.max(rslt['mte_max']) / 4
```

    (-0.5703098725428841, 0.6420576895047017)

Next we plot the MTE based on the estimation results. As shown in the figure below, the replicated MTE gets very close to the original, but its 90 percent confidence bands are wider. This is due to the use of a mock data set which merges basic and local variables randomly. The bootsrap method, which is used to estimate the confidence bands, is sensitive to the discrepancies in the data.

```python
mte, quantiles = plot_mte_carneiro(rslt, "notebook/files/replication_carneiro.yml", nbootstraps=250)
```

People with the highest returns to education (those who have low unobserved resistance $u_D$
) are more likely to go to college. Note that the returns vary considerably with $u_D$
. Low $u_D$ students have returns of up to 40% per year of college, whereas high $u_D$
 people, who would loose from attending college, have returns of approximately -18%.

The magnitude of total heterogeneity is probably even higher, as the MTE depicts the average gain of
college attendance at the mean values of X,
 i.e. $\bar{x} (\beta_1 - \beta_0)$.
Conditioning instead on $X (\beta_1 - \beta_0)$,
we observe returns as high as 64% and as low as -57%.

## 3) Estimating the Returns to Higher Education in Germany

We now turn to the analysis of the returns to schooling in Germany. I study a sample of 451 indiviudals without migration background, including men and women born between 1950 and 1971 in West Germany. As mentioned above, the common support assumption is critical for the identification of the MTE via Local Instrumental Variables (LIV). I plot it below.

First, specify the initialization file. It also contains information for the parametric model, which we'll estimate below. Note that I have to lower the default gridsize, since the sample has below 400 observations after trimming the data outside the common support.

```python
%%file notebook/files/soep_analysis.yml
---
SIMULATION:
    seed: 5062
    agents: 991
    source: 8EF73AA0
ESTIMATION:
    file: notebook/data/soep_data.pkl
    start: auto
    agents: 1000
    optimizer: SCIPY-BFGS
    maxiter: 80000
    dependent: wage
    indicator: state
    output_file: replication.grmp.info
    comparison: 0
    print_output: 0
    semipar: True
    show_output: True
    logit: False
    nbins: 29
    bandwidth: 0.35
    gridsize: 360
    trim_support: True
    reestimate_p: False
    rbandwidth: 0.1
    derivative: 1
    degree: 2
    ps_range: [0.005, 0.995]
TREATED:
    order:
    - sex
    - exp
    - exp2
    - avloc2006
    - local_unemp2006
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - perm_local_unemp
    - perm_local_unempsq
    - avpermlocwage
    - avpermlocwagesq
UNTREATED:
    order:
    - sex
    - exp
    - exp2
    - avloc2006
    - local_unemp2006
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - perm_local_unemp
    - perm_local_unempsq
    - avpermlocwage
    - avpermlocwagesq
CHOICE:
    params:
    - 1.0
    order:
    - const
    - sex
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - pub_uni
    - pub_uniiq
    - pub_unimotheduc
    - pub_uninumsibs
    - perm_local_unemp
    - perm_local_unempsq
    - local_unemp18sq
    - locwage18
    - locwage18iq
    - locwage18motheduc
    - locwage18numsibs
    - local_unemp18
    - local_unemp18iq
    - local_unemp18motheduc
    - local_unemp18numsibs
    - avpermlocwage
    - avpermlocwagesq
DIST:
    params:
    - 0.1
    - 0.0
    - 0.0
    - 0.1
    - 0.0
    - 1.0
VARTYPES:
    const: nonbinary
SCIPY-BFGS:
    gtol: 1.0e-05
    eps: 1.4901161193847656e-08
SCIPY-POWELL:
    xtol: 0.0001
    ftol: 0.0001
```

    Overwriting notebook/files/soep_analysis.yml

```python
plot_common_support("notebook/files/soep_analysis.yml", nbins=29, fs=16)
```

Common support is defined as the region where the support of $P(Z)$ given $D=1$ and the support of $P(Z)$ given $D=0$ overlap. It is not the simply the range from the smallest propensity observed in the treated sample to the maximum propensity observed in the untreated sample, as their marginal supports might not overlap in between. This would become evident by "holes" and "dents" in the histograms above. Although we observe positive frequencies for $P(Z)$ up to 0.83 in the untreated subpopulation, the continuous overlap of positive frequencies stops at 0.69. Subsequently, observations outside the common support are trimmed.

### 3.1 Local Instrumental Variables

We now turn to the estimation of the semiparametric MTE using LIV.

```python
rslt = fit("notebook/files/soep_analysis.yml", semipar=True)
```

Note that *grmpy-semipar* plots the MTE over almost the full unit interval although common support lies only between $[0.040, 0.690]$. The reason for this is that we set *p_range* in the initialization file to $[0.005, 0.995]$. *grmpy-semipar* utilizes the information within the common support to extrapolate the MTE for points outside the common support, as specified in *p_range*. Technically speaking, however, interpretations of the MTE are only valid within the actual common support of $[0.040, 0.690]$, for which the MTE is identified.

```python
mte, quantiles = plot_mte_soep(rslt, "notebook/files/soep_analysis.yml", nbootstraps=250)
```

Similar to the US-American case presented above, returns to education in Germany also vary considerably with $u_D$. People with the highest returns to education (those who have low unobserved resistance $u_D$ ) are more likely to attend university. Low $u_D$ students have returns of up to $40\%$ per year of university, whereas high $u_D$ people, who would loose from pursuing/completing a post-secondary degree, have returns of approximately $-0.38\%$.

The total degree of heterogeneity is probably even higher, as the MTE depicts the average gain of one year of post-secondary education at the mean values of $X$, i.e. $\bar{x} (\beta_1 - \beta_0)$. Accounting for the variation in $X$, returns can be as low as $-94\%$ and as high as $72\%$.

The dotted lines depict 90 percent confidence bands based on 250 bootstrap replications. As the bootstrap method is sensitive to small data sets, it produces fairly large standard errors.

### 3.2 Parametric Normal Model

In addition, I use the current functionality of [*grmpy*](https://grmpy.readthedocs.io/en/develop/) to estimate the returns to education in Germany based on a parametric model. Non-parametric techniques to estimate $K(P)$ are not required, as the unobservables are assumed to have a known functional form, i.e., joint normality.
Note that this condition is stronger than the assumption of independence and additive separability in the LIV framework.

Update the initialization file to include constants in the __TREATED, UNTREATED__ section:

```python
%%file notebook/files/soep_analysis.yml
---
SIMULATION:
    seed: 5062
    agents: 991
    source: 8EF73AA0
ESTIMATION:
    file: notebook/data/soep_data.pkl
    start: auto
    agents: 1000
    optimizer: SCIPY-BFGS
    maxiter: 80000
    dependent: wage
    indicator: state
    output_file: replication.grmp.info
    comparison: 0
    print_output: 0
    semipar: True
    show_output: True
    logit: False
    nbins: 29
    bandwidth: 0.35
    gridsize: 360
    trim_support: True
    reestimate_p: False
    rbandwidth: 0.1
    derivative: 1
    degree: 2
    ps_range: [0.005, 0.995]
TREATED:
    params:
    - 1.0
    order:
    - const
    - sex
    - exp
    - exp2
    - avloc2006
    - local_unemp2006
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - perm_local_unemp
    - perm_local_unempsq
    - avpermlocwage
    - avpermlocwagesq
UNTREATED:
    params:
    - 1.0
    order:
    - const
    - sex
    - exp
    - exp2
    - avloc2006
    - local_unemp2006
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - perm_local_unemp
    - perm_local_unempsq
    - avpermlocwage
    - avpermlocwagesq
CHOICE:
    params:
    - 1.0
    order:
    - const
    - sex
    - iq
    - iqsq
    - motheduc
    - motheducsq
    - white_collar
    - numsibs
    - numsibssq
    - urban15
    - pub_uni
    - pub_uniiq
    - pub_unimotheduc
    - pub_uninumsibs
    - perm_local_unemp
    - perm_local_unempsq
    - local_unemp18sq
    - locwage18
    - locwage18iq
    - locwage18motheduc
    - locwage18numsibs
    - local_unemp18
    - local_unemp18iq
    - local_unemp18motheduc
    - local_unemp18numsibs
    - avpermlocwage
    - avpermlocwagesq
DIST:
    params:
    - 0.1
    - 0.0
    - 0.0
    - 0.1
    - 0.0
    - 1.0
VARTYPES:
    const: nonbinary
SCIPY-BFGS:
    gtol: 1.0e-05
    eps: 1.4901161193847656e-08
SCIPY-POWELL:
    xtol: 0.0001
    ftol: 0.0001
```

    Overwriting notebook/files/soep_analysis.yml

```python
rslt = fit("notebook/files/soep_analysis.yml", semipar=False)
```

```python
mte = plot_mte_par(rslt, "notebook/files/soep_analysis.yml")
```

Observe that the parametric MTE is flatter than the semiparametric MTE above. This is in line with Carneiro et al. (2011) who find that the parametric MTE underestimates the returns to schooling in the US.


## 背景

我们从一个二分类问题说起，即给定N个二维向量，它们一半被标记为正例，一半被标记为负例，我们希望有一个模型，或者分类器，或者超平面，什么都好，它的输入是一个二维向量，输出是一个一维，或者二维向量，代表它属于正例或者负例的概率。

在科技发达的现代，能解这个二分类问题的方法数不胜数，感知机、Logistic Regression、SVM、神经网络等等。在本节中我们主要介绍SVM，即支持向量机，Support Vector Machine。

## 线性回归（Linear Regression）

我们希望从Linear Regression引出支持向量机，先来看看Linear Regression能不能解这个问题，我们首先写几行代码生成实验数据：

```python
%matplotlib inline

import matplotlib.pyplot as plt
import numpy as np

data_count = 100

x1_positive = np.linspace(-10, 10, data_count)
x2_positive = 0.3 * x1_positive + 10 + np.random.randint(-5, 5, data_count)
y_positive = np.array([1] * data_count)

x1_negative = np.linspace(-10, 10, data_count)
x2_negative = 0.3 * x1_negative - 10 + np.random.randint(-5, 5, data_count)
y_negative = np.array([-1] * data_count)

x1 = np.concatenate([x1_positive, x1_negative])
x2 = np.concatenate([x2_positive, x2_negative])

y_label = np.concatenate([y_positive, y_negative])

plt.scatter(x1_positive, x2_positive, c='r')
plt.scatter(x1_negative, x2_negative, c='b')
plt.show()
```

那么Linear Regression希望做什么呢，对于一个Linear Regression模型（函数）：

$$
y = \boldsymbol{w^Tx} + b
$$

直观地，Linear Regression希望最最小化下式：

$$
loss(\boldsymbol{w}, b) = \frac{1}{N}\sum^{N}_{i=0}(\boldsymbol{w^Tx_i} + b - y_i)^2
$$

这里使用平方损失函数其实问题很大，不过没关系，我们先忽略这点，稍后再来分析这里有什么问题。所以我们可以很自然的对$\boldsymbol{w}, b$求梯度：

$$
\begin{aligned}
        \frac{\partial loss(\boldsymbol{w}, b)}{\partial \boldsymbol{w}} &= \frac{2}{N}\sum^{N}_{i=0}(\boldsymbol{w^Tx_i} + b - y_i)\cdot \boldsymbol{x_i}, \\
        \frac{\partial loss(\boldsymbol{w}, b)}{\partial b} &= \frac{2}{N}\sum^{N}_{i=0}(\boldsymbol{w^Tx_i}+b-y_i)
\end{aligned}
$$

然后通过梯度下降更新$\boldsymbol{w}, b$：

$$
\begin{aligned}
\boldsymbol{w}_{t+1} &= \boldsymbol{w}_t - \eta \frac{\partial l(\boldsymbol{w}_t, b_t)}{\partial \boldsymbol{w}_t} \\
b_{t+1} &= b_t - \eta \frac{\partial l(\boldsymbol{w}_t, b_t)}{\partial b_t}
\end{aligned}
$$

然后通过几行代码简单实现一下：

```python
w1 = np.random.normal(0, 0.002)
w2 = np.random.normal(0, 0.002)
b = np.random.normal(0, 0.002)

training_steps = 1000

eta = 0.001

for step in range(training_steps):
    grad_w1 = np.mean((w1 * x1 + w2 * x2 + b - y_label) * x1)
    grad_w2 = np.mean((w1 * x1 + w2 * x2 + b - y_label) * x2)
    grad_b = np.mean(w1 * x1 + w2 * x2 + b)

    w1 -= eta * grad_w1
    w2 -= eta * grad_w2
    b -= eta * grad_b

plt.scatter(x1_positive, x2_positive, c='r')
plt.scatter(x1_negative, x2_negative, c='b')
plt.plot(x1, -(w1 * x1 + b) / w2, c='g')
plt.show()
```

## 梯度下降（Gradient Decent）

为什么能用梯度下降呢？这里快速预习一下泰勒公式：

$$
f(x) = \sum^{N}_{n=0}\frac{f^{(n)}(x_0)}{n!}(x - x_0)^n + R_n(x)
$$

泰勒公式的核心精神在于，某一个函数可以用该函数在某一点的函数值及其n阶导数值来近似表示，在我们的问题里，对于$\boldsymbol{w}$就有：

$$
loss(\boldsymbol{w}_{t+1}) = loss(\boldsymbol{w}_t + \Delta{\theta}) \approx loss(\boldsymbol{w}_t) + loss'(\boldsymbol{w}_{t})\Delta{\theta}
$$

如果希望梯度下降在本次迭代中使得损失函数值变小：

$$
loss(\boldsymbol{w}_{t+1}) -  loss(\boldsymbol{w}_{t}) = \nabla loss(\boldsymbol{w}_{t})\Delta{\theta} < 0
$$

通过观察可以发现，$\nabla loss(\boldsymbol{w}_{t})$与$\Delta{\theta}\ $都是二维向量，当$\Delta{\theta} = -\eta \nabla loss(\boldsymbol{w}_{t})$时，即两向量反向时，其内积有最小值。

## 平方损失函数的问题

理想情况下，初始化$\boldsymbol{w}, b$后，在误差反向传播计算梯度时，那些被错误分类的点，会带来很大误差，同时贡献可观的梯度用来更新权重。但是平方损失却不能很好的达到我们的目的。

最小化平方损失函数的目的在于对样本所有的数据点，通过调整$\ \boldsymbol{w}, b$，使他们的输出更靠近他们的标签，即-1或者1，而不是完美地解决我们的二分类问题，或者说，解决了二分类问题是最小化平方损失函数的一个顺带的结果。

同时，直觉地，一些被分类正确的点，反而会通过平方损失函数计算出极大的误差，贡献出可观的权重来修正自己，希望自己靠近标签，这是不合理的。

既然这个问题是平方损失函数带来的，那么理论上我们就应该通过重新设计损失函数解决这个问题。事实上，我们最原始的目标函数，或者损失函数，应该是：

$$
loss(\boldsymbol{w}, b) = \sum^{N}_{i=0}\delta \left( \boldsymbol{w^Tx_i} + b \ne y_i \right)
$$

但是这个损失函数是不可微的，也就没有办法用梯度下降的方式求解了。

## 合页损失函数（Hinge Loss）

事实上，带Sigmoid函数的交叉熵损失函数，或者带Sigmoid激活函数的平方损失函数也能解决这个问题，而前者一般叫Logistic Regression，而后者，好像还没有名字。在这里我们选用合页损失函数（Hingle Loss）：

$$
loss(\boldsymbol{w}, b) = \sum^{N}_{i=0} \max(0, 1 - y_i \cdot (\boldsymbol{w^Tx_i} + b))
$$

对于一维铰链损失函数，它的图像将会是：

直观地，我们可以看出，对于一个样本点，如果分类正确，那么$y_i$与$\boldsymbol{w^Tx_i} + b$应该同号，且如果他们的乘积大于1，那么损失函数的值为0，如果他们的乘积大于0（分类正确），小于1，那么损失函数的值就等于$1 - y_i \cdot (\boldsymbol{w^Tx_i} + b)$，而对于分类错误的情况，可以发现$y_i$与$\boldsymbol{w^Tx_i} + b$应该异号，则损失函数的值为$1 - y_i \cdot (\boldsymbol{w^Tx_i} + b)$。

不同于平方损失，我们可以发现，Hinge Loss在我们的二分类问题中更为合理，相对于平方损失函数“认为”模型对所有样本的预测结果都应该靠近标签，即-1或1而言（即与标签越相似越好），Hingle Loss则“认为”一些已经“显然”被分类正确的样本（$1 - y_i \cdot (\boldsymbol{w^Tx_i} + b) < 0$）显然地不应该传播误差，贡献梯度更新权重，即显然被分类正确的样本点不应该参与更新$\ \boldsymbol{w}, b$，因为他们的梯度为0。

而$0 < 1 - y_i \cdot (\boldsymbol{w^Tx_i} + b) < 1$，这些“险些”被分类错误的点，但是仍然被分类正确的点，被Hingle Loss“认为”分类还不够完美，仍然需要反向传播误差，贡献梯度更新权重。

而对于分类错误的点，它们的处理方法则与平方损失函数相同，直接反向传播误差贡献梯度更新权重即可。

从上我们还可以看出Hingle Loss的另外一个好处，即一些“显然”被分类正确点的不参与梯度计算，从而降低了一些计算的复杂度。

## 合页函数的梯度

而给定Hinge Loss的函数，则$\boldsymbol{w}$的梯度则有如下形式：

\begin{equation}
\frac{\partial loss(\boldsymbol{w}, b)}{\partial \boldsymbol{w}} =
\left\{
             \begin{array}
              \\-y_i \cdot \boldsymbol{x_i}, &  y_i \cdot (\boldsymbol{w^Tx_i} + b ) < 1\\
              0, & y_i \cdot (\boldsymbol{w^Tx_i} + b ) > 1
             \end{array}
\right.
\end{equation}

而对于$b$的梯度，则有如下形式：

\begin{equation}
\frac{\partial loss(\boldsymbol{w}, b)}{\partial b} =
\left\{
             \begin{array}
             \\-y_i, &  y_i \cdot (\boldsymbol{w^Tx_i} + b ) < 1\\
             0, & y_i \cdot (\boldsymbol{w^Tx_i} + b ) > 1
             \end{array}
\right.
\end{equation}

然后通过梯度下降更新$\boldsymbol{w}, b$：

$$
\begin{aligned}
\boldsymbol{w}_{t+1} &= \boldsymbol{w}_t - \eta \frac{\partial l(\boldsymbol{w}_t, b_t)}{\partial \boldsymbol{w}_t} \\
b_{t+1} &= b_t - \eta \frac{\partial l(\boldsymbol{w}_t, b_t)}{\partial b_t}
\end{aligned}
$$

通过几行代码简单实现一下：

```python
for step in range(training_steps):
    hinge_judge_term = y_label * (w1 * x1 + w2 * x2 + b)

    mask_no_grad = hinge_judge_term > 1

    grad_before_mean_w1 = -y_label * x1
    grad_before_mean_w1[mask_no_grad] = 0
    grad_w1 = np.mean(grad_before_mean_w1)

    grad_before_mean_w2 = -y_label * x2
    grad_before_mean_w2[mask_no_grad] = 0
    grad_w2 = np.mean(grad_before_mean_w2)

    grad_before_mean_b = -y_label * 1
    grad_before_mean_b[mask_no_grad] = 0
    grad_b = np.mean(grad_before_mean_b)

    w1 -= eta * grad_w1
    w2 -= eta * grad_w2
    b -= eta * grad_b

plt.scatter(x1_positive, x2_positive, c='r')
plt.scatter(x1_negative, x2_negative, c='b')
plt.plot(x1, -(w1 * x1 + b) / w2, c='g')
plt.show()
```

## 支持向量机（Support Vector Machine）

较为直观地，我们可以发现，随着梯度下降迭代的进行，渐渐地诸多样本点将不再反向传播误差，贡献梯度用来更新权重，因为他们已经“显然”被分类正确了，即$y_i \cdot (\boldsymbol{w^Tx_i} + b ) > 1 $，而那些直至迭代结束还在贡献梯度的样本点，将会被称之为支持向量，这也是支持向量机的来历。

其实到这里支持向量机就介绍完了。

有些观众朋友们可能就会问了，好像学过的支持向量机不是这么推导的呀，似乎应该是先写出最大化间隔，然后放缩，然后引出拉格朗日乘子法解等式约束的最值问题，然后引出KKT条件和不等式约束，然后构建广义拉格朗日方程使之与原始问题同解，然后领各种偏微分为0，解其对偶问题呢？

是这样的没错，上面一系列复杂的推导似乎是凸优化（虽然我没系统的学过）的一部分，这些内容将会在下一节预习，所以在这里先简要的说一下结论：

即如果我们为上面的Hinge Loss加上一个L2正则化项：

$$
loss(\boldsymbol{w}, b) = \sum^{N}_{i=0} \max(0, 1 - y_i \cdot (\boldsymbol{w^Tx_i} + b)) + \lambda \Vert \boldsymbol{w} \Vert ^2
$$

那么，通过这两种方法得到的结果本质上是相同的，为什么是相同的这里打算在下一节一起预习，所以这里先暂且结束了。

## 总结

这次主要从梯度下降的线性回归入手，从梯度下降和最小化Hinge Loss的角度预习了支持向量机，下一节将从传统凸优化的角度预习支持向量机。


# Vertical Line Test

## 1.1 Create two graphs, one that passes the vertical line test and one that does not.

```
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

# Plot Linearly Independent Vectors

# Axis Bounds
plt.xlim(-2,5)
plt.ylim(-1,3)

# Original Vector
a = [-1.5,.5]
b = [3, 1]

# Plot Vectors
plt.arrow(1,1, a[0], a[1], linewidth=3, head_width=.05, head_length=0.05, color ='blue')
plt.arrow(0,0, b[0], b[1], linewidth=3, head_width=.05, head_length=0.05, color ='red')

plt.title("This graph does not pass the vertical line test")
plt.show()
```

```
# Plot Linearly Independent Vectors

# Axis Bounds
plt.xlim(-3,4)
plt.ylim(-1,3)

# Original Vector
a = [-1.5,.5]
b = [3, 1]

# Plot Vectors
plt.arrow(-0.5,0, a[0], a[1], linewidth=3, head_width=.05, head_length=0.05, color ='blue')
plt.arrow(.5,0, b[0], b[1], linewidth=3, head_width=.05, head_length=0.05, color ='red')

plt.title("This graph passes the vertical line test")
plt.show()
```

## 1.2 Why are graphs that don't pass the vertical line test not considered "functions?"

Because a function is defined where no two outputs are mapped to the same point. In graph #1, there are places where the two vectors share x, while in graph #2 that does not occur anywhere.

# Functions as Relations

## 2.1 Which of the following relations are functions? Why?

\begin{align}
\text{Relation 1: } \{(1, 2), (3, 2), (1, 3)\}
\\
\text{Relation 2: } \{(1, 3), (2, 3), (6, 7)\}
\\
\text{Relation 3: } \{(9, 4), (2, 1), (9, 6)\}
\\
\text{Relation 4: } \{(6, 2), (8, 3), (6, 4)\}
\\
\text{Relation 5: } \{(2, 6), (2, 7), (2, 4)\}
\end{align}

####(Zhenya)

---

Only Relation #2 is a function because none of the coordinates share an X value so it passes the vertical line test.

# Functions as a mapping between dimensions

## 3.1 for the following functions what is the dimensionality of the domain (input) and codomain (range/output)?

\begin{align}
m(𝑥_1,𝑥_2,𝑥_3)=(x_1+x_2, x_1+x_3, x_2+x_3)
\\
n(𝑥_1,𝑥_2,𝑥_3,𝑥_4)=(x_2^2 + x_3, x_2x_4)
\end{align}

#### (Zhenya)

---

1st function: R^3 to R^2

2nd function: R^4 to R^2

## 3.2 Do you think it's possible to create a function that maps from a lower dimensional space to a higher dimensional space? If so, provide an example.

####Zhenya

---

This goes from R^1 to R^3

$m(x_1) = (x_1 / 2 + x_1 / 3,x_1 / 3 + x_1 / 4),x_1 / 4 + x_1 / 5) $

# Vector Transformations

## 4.1 Plug the corresponding unit vectors into each function. Use the output vectors to create a transformation matrix.

\begin{align}
p(\begin{bmatrix}x_1 \\ x_2 \end{bmatrix}) = \begin{bmatrix} x_1 + 3x_2 \\2 x_2 - x_1 \\  \end{bmatrix}
\\
\\
q(\begin{bmatrix}x_1 \\ x_2 \\ x_3\end{bmatrix}) = \begin{bmatrix} 4x_1 + x_2 + 2x_3 \\2 x_2 - x_1 + 3x_3 \\ 5x_1 - 2x_3 + x_2  \end{bmatrix}
\end{align}

####(Zhenya)

THIS IS CORRECT YAYYY!!!!

---

First way to transform the function into a transformation matrix:

\begin{align}
p(\begin{bmatrix}0 \\ 1 \end{bmatrix}) = \begin{bmatrix} x_1 + 3x_2 \\2 x_2 - x_1 \\  \end{bmatrix} =\begin{bmatrix}3 \\ 2 \end{bmatrix} \end{align}

\begin{align}
p(\begin{bmatrix}1 \\ 0 \end{bmatrix}) = \begin{bmatrix} x_1 + 3x_2 \\2 x_2 - x_1 \\  \end{bmatrix}=\begin{bmatrix}1 \\ -1 \end{bmatrix}
\end{align}

\begin{align}
q(\begin{bmatrix}1 \\ 0 \\ 0 \end{bmatrix}) = \begin{bmatrix} 4x_1 + x_2 + 2x_3 \\2 x_2 - x_1 + 3x_3 \\ 5x_1 - 2x_3 + x_2  \end{bmatrix}=\begin{bmatrix}4 \\ -1 \\ 5 \end{bmatrix}
\end{align}

\begin{align}
q(\begin{bmatrix}0 \\ 1 \\ 0 \end{bmatrix}) = \begin{bmatrix} 4x_1 + x_2 + 2x_3 \\2 x_2 - x_1 + 3x_3 \\ 5x_1 - 2x_3 + x_2  \end{bmatrix}=\begin{bmatrix}1 \\ 2 \\ 1 \end{bmatrix}
\end{align}

\begin{align}
q(\begin{bmatrix}0 \\ 0 \\ 1 \end{bmatrix}) = \begin{bmatrix} 4x_1 + x_2 + 2x_3 \\2 x_2 - x_1 + 3x_3 \\ 5x_1 - 2x_3 + x_2  \end{bmatrix}=\begin{bmatrix}2 \\ 3 \\ -2 \end{bmatrix}
\end{align}

---

Here are the transformation matrices

\begin{align}
p = \begin{bmatrix} 1 & 3 \\ -1 & 2 \end{bmatrix}
\end{align}

\begin{align}
q = \begin{bmatrix} 4 & 1 & 2 \\ -1 & 2 & 3 \\ 5 & 1 & -2 \end{bmatrix}
\end{align}

## 4.2 Verify that your transformation matrices are correct by choosing an input matrix and calculating the result both via the traditional functions above and also via vector-matrix multiplication.

```
#not sure how to check my work

x1 = 1
x2 = 2

p = array([[x1,3*x2],
           [2,2]])
```

# Eigenvalues and Eigenvectors

#Zhenya - need to learn don't understand!!!!!! FUCK

## 5.1 In your own words, give an explanation for the intuition behind eigenvalues and eigenvectors.

What about the vector represented by the earth's axis? Does it move?

We could call this vector the **eigenvector** of the earth's rotation brecause the transformation does not reorient it. Any transformation that you could conceive of possesses eigenvectors - dimensions of space that are not pointed in a new direction by the transformation described by a certain transformation matrix.

## So what's an eigenvalue?

An eigenvector is any vector that doesn't change its orientation during a given transformation, but an eigenvector might still get scaled by some scalar. It can get stretched or squished, it just can't change the direction in which it is pointing. An **eigenvalue** is the scalar that represents how its corresponding eigenvector was scaled during the transformation. Eigenvectors and eigenvalues always come in pairs.

# The Curse of Dimensionality

## 6.1 What are some of the challenges of working with high dimensional spaces?

* Cannot visualize beyond 3 dimensions reliably. This limitation is a challenge.

* Increased computational workload.

* Measures of distance begin to lose meaning.

## 6.2 What is the rule of thumb for how many observations you should have compared to parameters in your model?

```
#You should have at least 5 observations for every parameter (feature) in your model
```

# Principal Component Analysis

## 7.1 Code for loading and cleaning the 2013 national dataset from the [Housing Affordability Data System (HADS)](https://www.huduser.gov/portal/datasets/hads/hads.html) --housing data, can be found below.

## Perform PCA on the processed dataset `national_processed` (Make sure you standardize your data!) and then make a scatterplot of PC1 against PC2. Some of our discussion and work around PCA with this dataset will continue during tomorrow's lecture and assignment.

Not only does this dataset have decent amount columns to begin with (99), but in preparing the data for PCA we have also [one-hot-encoded](https://hackernoon.com/what-is-one-hot-encoding-why-and-when-do-you-have-to-use-it-e3c6186d008f#targetText=One%20hot%20encoding%20is%20a,the%20entry%20in%20the%20dataset.) all of the categorical variables. This has the effect of creating a new column for each individual category of each categorical variable. After processing this dataset has 64738 columns. --Das a lot of columns.

Don't worry too much about the mechanics of one-hot encoding right now, you will learn and experiment with a whole bunch of categorical encoding approaches in unit 2.

The code below will read in the dataset and perform the one-hot encoding of the categorical variables. Start adding your PCA code at the bottom of the provided code.

```
import os.path
from urllib.request import urlopen
from zipfile import ZipFile
from io import BytesIO
import pandas as pd

# Read National Data
national_url = 'https://www.huduser.gov/portal/datasets/hads/hads2013n_ASCII.zip'
national_file = 'thads2013n.txt'

if os.path.exists(national_file):
    national = pd.read_csv(national_file)
else:
    z_national = urlopen(national_url)
    zip_national = ZipFile(BytesIO(z_national.read())).extract(national_file)
    national = pd.read_csv(zip_national)
```

```
# Look at first 5 rows
print(national.shape)
national.head()
```

    (64535, 99)

      CONTROL
      AGE1
      METRO3
      REGION
      LMED
      FMR
      L30
      L50
      L80
      IPOV
      BEDRMS
      BUILT
      STATUS
      TYPE
      VALUE
      VACANCY
      TENURE
      NUNITS
      ROOMS
      WEIGHT
      PER
      ZINC2
      ZADEQ
      ZSMHC
      STRUCTURETYPE
      OWNRENT
      UTILITY
      OTHERCOST
      COST06
      COST12
      COST08
      COSTMED
      TOTSAL
      ASSISTED
      GLMED
      GL30
      GL50
      GL80
      APLMED
      ABL30
      ...
      COST08RELPOVCAT
      COST08RELFMRPCT
      COST08RELFMRCAT
      COST12RELAMIPCT
      COST12RELAMICAT
      COST12RELPOVPCT
      COST12RELPOVCAT
      COST12RELFMRPCT
      COST12RELFMRCAT
      COSTMedRELAMIPCT
      COSTMedRELAMICAT
      COSTMedRELPOVPCT
      COSTMedRELPOVCAT
      COSTMedRELFMRPCT
      COSTMedRELFMRCAT
      FMTZADEQ
      FMTMETRO3
      FMTBUILT
      FMTSTRUCTURETYPE
      FMTBEDRMS
      FMTOWNRENT
      FMTCOST06RELPOVCAT
      FMTCOST08RELPOVCAT
      FMTCOST12RELPOVCAT
      FMTCOSTMEDRELPOVCAT
      FMTINCRELPOVCAT
      FMTCOST06RELFMRCAT
      FMTCOST08RELFMRCAT
      FMTCOST12RELFMRCAT
      FMTCOSTMEDRELFMRCAT
      FMTINCRELFMRCAT
      FMTCOST06RELAMICAT
      FMTCOST08RELAMICAT
      FMTCOST12RELAMICAT
      FMTCOSTMEDRELAMICAT
      FMTINCRELAMICAT
      FMTASSISTED
      FMTBURDEN
      FMTREGION
      FMTSTATUS

      0
      '100003130103'
      82
      '3'
      '1'
      73738
      956
      15738
      26213
      40322
      11067
      2
      2006
      '1'
      1
      40000
      -6
      '1'
      1
      6
      3117.394239
      1
      18021
      '1'
      533
      1
      '1'
      169.000000
      213.750000
      648.588189
      803.050535
      696.905247
      615.156712
      0
      -9
      73738
      15738
      26213
      40322
      51616.6
      20234.571429
      ...
      4
      72.898038
      2
      48.402635
      2
      290.250487
      4
      84.001102
      2
      37.077624
      2
      222.339102
      4
      64.346936
      2
      '1 Adequate'
      '-5'
      '2000-2009'
      '1 Single Family'
      '2 2BR'
      '1 Owner'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '3 150-200% Poverty'
      '2 50.1 - 100% FMR'
      '2 50.1 - 100% FMR'
      '2 50.1 - 100% FMR'
      '2 50.1 - 100% FMR'
      '1 LTE 50% FMR'
      '2 30 - 50% AMI'
      '2 30 - 50% AMI'
      '2 30 - 50% AMI'
      '2 30 - 50% AMI'
      '2 30 - 50% AMI'
      '.'
      '2 30% to 50%'
      '-5'
      '-5'

      1
      '100006110249'
      50
      '5'
      '3'
      55846
      1100
      17165
      28604
      45744
      24218
      4
      1980
      '1'
      1
      130000
      -6
      '1'
      1
      6
      2150.725544
      4
      122961
      '1'
      487
      1
      '1'
      245.333333
      58.333333
      1167.640781
      1669.643405
      1324.671218
      1058.988479
      123000
      -9
      55846
      17165
      28604
      45744
      55846.0
      19911.400000
      ...
      4
      120.424656
      3
      103.094063
      6
      275.768999
      4
      151.785764
      3
      65.388468
      4
      174.909320
      3
      96.271680
      2
      '1 Adequate'
      '-5'
      '1980-1989'
      '1 Single Family'
      '4 4BR+'
      '1 Owner'
      '3 150-200% Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '3 150-200% Poverty'
      '4 200%+ Poverty'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '2 50.1 - 100% FMR'
      '3 GT FMR'
      '4 60 - 80% AMI'
      '4 60 - 80% AMI'
      '6 100 - 120% AMI'
      '4 60 - 80% AMI'
      '7 120% AMI +'
      '.'
      '1 Less than 30%'
      '-5'
      '-5'

      2
      '100006370140'
      53
      '5'
      '3'
      55846
      1100
      13750
      22897
      36614
      15470
      4
      1985
      '1'
      1
      150000
      -6
      '1'
      1
      7
      2213.789404
      2
      27974
      '1'
      1405
      1
      '1'
      159.000000
      37.500000
      1193.393209
      1772.627006
      1374.582175
      1068.025168
      28000
      -9
      55846
      13750
      22897
      36614
      44676.8
      19937.500000
      ...
      4
      124.962016
      3
      109.452905
      6
      458.339239
      4
      161.147910
      3
      65.946449
      4
      276.153890
      4
      97.093197
      2
      '1 Adequate'
      '-5'
      '1980-1989'
      '1 Single Family'
      '4 4BR+'
      '1 Owner'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '3 150-200% Poverty'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '2 50.1 - 100% FMR'
      '2 50.1 - 100% FMR'
      '4 60 - 80% AMI'
      '5 80 - 100% AMI'
      '6 100 - 120% AMI'
      '4 60 - 80% AMI'
      '4 60 - 80% AMI'
      '.'
      '3 50% or More'
      '-5'
      '-5'

      3
      '100006520140'
      67
      '5'
      '3'
      55846
      949
      13750
      22897
      36614
      13964
      3
      1985
      '1'
      1
      200000
      -6
      '1'
      1
      6
      2364.585097
      2
      32220
      '1'
      279
      1
      '1'
      179.000000
      70.666667
      1578.857612
      2351.169341
      1820.442900
      1411.700224
      0
      -9
      55846
      13750
      22897
      36614
      44676.8
      17875.000000
      ...
      4
      191.827492
      3
      161.926709
      7
      673.494512
      4
      247.752301
      3
      97.224801
      5
      404.382763
      4
      148.756610
      3
      '1 Adequate'
      '-5'
      '1980-1989'
      '1 Single Family'
      '3 3BR'
      '1 Owner'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '4 200%+ Poverty'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '2 50.1 - 100% FMR'
      '6 100 - 120% AMI'
      '7 120% AMI +'
      '7 120% AMI +'
      '5 80 - 100% AMI'
      '4 60 - 80% AMI'
      '.'
      '1 Less than 30%'
      '-5'
      '-5'

      4
      '100007130148'
      26
      '1'
      '3'
      60991
      737
      14801
      24628
      39421
      15492
      2
      1980
      '1'
      1
      -6
      -6
      '2'
      100
      4
      2314.524902
      2
      96874
      '1'
      759
      5
      '2'
      146.000000
      12.500000
      759.000000
      759.000000
      759.000000
      759.000000
      96900
      0
      60991
      14801
      24628
      39421
      48792.8
      16651.125000
      ...
      3
      102.985075
      3
      55.308707
      3
      195.972115
      3
      102.985075
      3
      55.308707
      3
      195.972115
      3
      102.985075
      3
      '1 Adequate'
      'Central City'
      '1980-1989'
      '5 50+ units'
      '2 2BR'
      '2 Renter'
      '3 150-200% Poverty'
      '3 150-200% Poverty'
      '3 150-200% Poverty'
      '3 150-200% Poverty'
      '4 200%+ Poverty'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '3 GT FMR'
      '3 50 - 60% AMI'
      '3 50 - 60% AMI'
      '3 50 - 60% AMI'
      '3 50 - 60% AMI'
      '7 120% AMI +'
      '0 Not Assisted'
      '1 Less than 30%'
      '-5'
      '-5'

5 rows × 99 columns

```
# Look at datatypes
# a lot of object datatypes even though they look to be strings of numbers.
national.dtypes
```

    CONTROL                 object
    AGE1                     int64
    METRO3                  object
    REGION                  object
    LMED                     int64
    FMR                      int64
    L30                      int64
    L50                      int64
    L80                      int64
    IPOV                     int64
    BEDRMS                   int64
    BUILT                    int64
    STATUS                  object
    TYPE                     int64
    VALUE                    int64
    VACANCY                  int64
    TENURE                  object
    NUNITS                   int64
    ROOMS                    int64
    WEIGHT                 float64
    PER                      int64
    ZINC2                    int64
    ZADEQ                   object
    ZSMHC                    int64
    STRUCTURETYPE            int64
    OWNRENT                 object
    UTILITY                float64
    OTHERCOST              float64
    COST06                 float64
    COST12                 float64
                            ...
    COSTMedRELAMICAT         int64
    COSTMedRELPOVPCT       float64
    COSTMedRELPOVCAT         int64
    COSTMedRELFMRPCT       float64
    COSTMedRELFMRCAT         int64
    FMTZADEQ                object
    FMTMETRO3               object
    FMTBUILT                object
    FMTSTRUCTURETYPE        object
    FMTBEDRMS               object
    FMTOWNRENT              object
    FMTCOST06RELPOVCAT      object
    FMTCOST08RELPOVCAT      object
    FMTCOST12RELPOVCAT      object
    FMTCOSTMEDRELPOVCAT     object
    FMTINCRELPOVCAT         object
    FMTCOST06RELFMRCAT      object
    FMTCOST08RELFMRCAT      object
    FMTCOST12RELFMRCAT      object
    FMTCOSTMEDRELFMRCAT     object
    FMTINCRELFMRCAT         object
    FMTCOST06RELAMICAT      object
    FMTCOST08RELAMICAT      object
    FMTCOST12RELAMICAT      object
    FMTCOSTMEDRELAMICAT     object
    FMTINCRELAMICAT         object
    FMTASSISTED             object
    FMTBURDEN               object
    FMTREGION               object
    FMTSTATUS               object
    Length: 99, dtype: object

```
# check for null values
national.isnull().sum().any()
```

    False

```
# check for number of categorical vs numeric columns
cat_cols = national.columns[national.dtypes=='object']
num_cols = national.columns[national.dtypes!='object']

print(f'{len(cat_cols)} categorical columns')
print(f'{len(num_cols)} numerical columns')
```

    32 categorical columns
    67 numerical columns

```
national_processed = national.copy()

# Get One-Hot-Encoding of categorical variables
national_processed = pd.get_dummies(national_processed)
print(national_processed.shape)
national_processed.head()
```

    (64535, 64738)

      AGE1
      LMED
      FMR
      L30
      L50
      L80
      IPOV
      BEDRMS
      BUILT
      TYPE
      VALUE
      VACANCY
      NUNITS
      ROOMS
      WEIGHT
      PER
      ZINC2
      ZSMHC
      STRUCTURETYPE
      UTILITY
      OTHERCOST
      COST06
      COST12
      COST08
      COSTMED
      TOTSAL
      ASSISTED
      GLMED
      GL30
      GL50
      GL80
      APLMED
      ABL30
      ABL50
      ABL80
      ABLMED
      BURDEN
      INCRELAMIPCT
      INCRELAMICAT
      INCRELPOVPCT
      ...
      FMTCOST08RELAMICAT_'1 LTE 30% AMI'
      FMTCOST08RELAMICAT_'2 30 - 50% AMI'
      FMTCOST08RELAMICAT_'3 50 - 60% AMI'
      FMTCOST08RELAMICAT_'4 60 - 80% AMI'
      FMTCOST08RELAMICAT_'5 80 - 100% AMI'
      FMTCOST08RELAMICAT_'6 100 - 120% AMI'
      FMTCOST08RELAMICAT_'7 120% AMI +'
      FMTCOST12RELAMICAT_'1 LTE 30% AMI'
      FMTCOST12RELAMICAT_'2 30 - 50% AMI'
      FMTCOST12RELAMICAT_'3 50 - 60% AMI'
      FMTCOST12RELAMICAT_'4 60 - 80% AMI'
      FMTCOST12RELAMICAT_'5 80 - 100% AMI'
      FMTCOST12RELAMICAT_'6 100 - 120% AMI'
      FMTCOST12RELAMICAT_'7 120% AMI +'
      FMTCOSTMEDRELAMICAT_'1 LTE 30% AMI'
      FMTCOSTMEDRELAMICAT_'2 30 - 50% AMI'
      FMTCOSTMEDRELAMICAT_'3 50 - 60% AMI'
      FMTCOSTMEDRELAMICAT_'4 60 - 80% AMI'
      FMTCOSTMEDRELAMICAT_'5 80 - 100% AMI'
      FMTCOSTMEDRELAMICAT_'6 100 - 120% AMI'
      FMTCOSTMEDRELAMICAT_'7 120% AMI +'
      FMTINCRELAMICAT_'.'
      FMTINCRELAMICAT_'1 LTE 30% AMI'
      FMTINCRELAMICAT_'2 30 - 50% AMI'
      FMTINCRELAMICAT_'3 50 - 60% AMI'
      FMTINCRELAMICAT_'4 60 - 80% AMI'
      FMTINCRELAMICAT_'5 80 - 100% AMI'
      FMTINCRELAMICAT_'6 100 - 120% AMI'
      FMTINCRELAMICAT_'7 120% AMI +'
      FMTASSISTED_'.'
      FMTASSISTED_'0 Not Assisted'
      FMTASSISTED_'1 Assisted'
      FMTBURDEN_'.'
      FMTBURDEN_'1 Less than 30%'
      FMTBURDEN_'2 30% to 50%'
      FMTBURDEN_'3 50% or More'
      FMTBURDEN_'4 No Income'
      FMTREGION_'-5'
      FMTREGION_'West'
      FMTSTATUS_'-5'

      0
      82
      73738
      956
      15738
      26213
      40322
      11067
      2
      2006
      1
      40000
      -6
      1
      6
      3117.394239
      1
      18021
      533
      1
      169.000000
      213.750000
      648.588189
      803.050535
      696.905247
      615.156712
      0
      -9
      73738
      15738
      26213
      40322
      51616.6
      20234.571429
      33702.428571
      51842.571429
      66364.20
      0.354919
      34.913187
      2
      162.835457
      ...
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      1
      0
      1

      1
      50
      55846
      1100
      17165
      28604
      45744
      24218
      4
      1980
      1
      130000
      -6
      1
      6
      2150.725544
      4
      122961
      487
      1
      245.333333
      58.333333
      1167.640781
      1669.643405
      1324.671218
      1058.988479
      123000
      -9
      55846
      17165
      28604
      45744
      55846.0
      19911.400000
      33180.640000
      53063.040000
      64781.36
      0.047527
      220.178706
      7
      507.725659
      ...
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      1
      1
      0
      0
      0
      1
      0
      0
      0
      1
      0
      1

      2
      53
      55846
      1100
      13750
      22897
      36614
      15470
      4
      1985
      1
      150000
      -6
      1
      7
      2213.789404
      2
      27974
      1405
      1
      159.000000
      37.500000
      1193.393209
      1772.627006
      1374.582175
      1068.025168
      28000
      -9
      55846
      13750
      22897
      36614
      44676.8
      19937.500000
      33200.650000
      53090.300000
      64781.36
      0.602703
      62.614153
      4
      180.827408
      ...
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      1
      0
      0
      0
      0
      0
      1
      0
      1
      0
      1

      3
      67
      55846
      949
      13750
      22897
      36614
      13964
      3
      1985
      1
      200000
      -6
      1
      6
      2364.585097
      2
      32220
      279
      1
      179.000000
      70.666667
      1578.857612
      2351.169341
      1820.442900
      1411.700224
      0
      -9
      55846
      13750
      22897
      36614
      44676.8
      17875.000000
      29766.100000
      47598.200000
      58079.84
      0.103911
      72.117967
      4
      230.736179
      ...
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      1
      0
      0
      0
      1
      0
      0
      0
      1
      0
      1

      4
      26
      60991
      737
      14801
      24628
      39421
      15492
      2
      1980
      1
      -6
      -6
      100
      4
      2314.524902
      2
      96874
      759
      5
      146.000000
      12.500000
      759.000000
      759.000000
      759.000000
      759.000000
      96900
      0
      60991
      14801
      24628
      39421
      48792.8
      16651.125000
      27706.500000
      44348.625000
      54891.90
      0.094019
      198.541588
      7
      625.316292
      ...
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      1
      0
      1
      0
      0
      1
      0
      0
      0
      1
      0
      1

5 rows × 64738 columns

```
national1 = national.select_dtypes(include=["float64","int64"])
```

```
mean(national1.T, axis=1)
```

    AGE1                    47.968932
    LMED                 68109.744309
    FMR                   1164.395181
    L30                  17712.125436
    L50                  29511.032076
    L80                  46207.567630
    IPOV                 15986.664229
    BEDRMS                   2.660308
    BUILT                 1966.432835
    TYPE                     1.065360
    VALUE               142032.790532
    VACANCY                 -5.458433
    NUNITS                  14.889052
    ROOMS                    5.631208
    WEIGHT                1885.195459
    PER                      1.886015
    ZINC2                61355.837623
    ZSMHC                 1061.613744
    STRUCTURETYPE            1.920973
    UTILITY                183.099576
    OTHERCOST               67.873030
    COST06                1525.576425
    COST12                2074.054208
    COST08                1697.144663
    COSTMED               1406.865145
    TOTSAL               44910.928117
    ASSISTED                -5.516262
    GLMED                68109.744309
    GL30                 17712.125436
    GL50                 29511.032076
                            ...
    INCRELAMIPCT           104.784829
    INCRELAMICAT             3.377795
    INCRELPOVPCT           358.745915
    INCRELPOVCAT             2.280158
    INCRELFMRPCT           127.677315
    INCRELFMRCAT             1.463113
    COST06RELAMIPCT         87.021707
    COST06RELAMICAT          3.949144
    COST06RELPOVPCT        352.582333
    COST06RELPOVCAT          2.487921
    COST06RELFMRPCT        124.754094
    COST06RELFMRCAT          2.376307
    COST08RELAMIPCT         96.214197
    COST08RELAMICAT          4.162997
    COST08RELPOVPCT        392.790898
    COST08RELPOVCAT          2.526288
    COST08RELFMRPCT        137.651042
    COST08RELFMRCAT          2.424901
    COST12RELAMIPCT        116.408711
    COST12RELAMICAT          4.501278
    COST12RELPOVPCT        481.123050
    COST12RELPOVCAT          2.574758
    COST12RELFMRPCT        165.983691
    COST12RELFMRCAT          2.488990
    COSTMedRELAMIPCT        80.661251
    COSTMedRELAMICAT         3.770900
    COSTMedRELPOVPCT       324.761274
    COSTMedRELPOVCAT         2.450376
    COSTMedRELFMRPCT       115.830454
    COSTMedRELFMRCAT         2.328922
    Length: 67, dtype: float64

```
#having trouble finding Z

from numpy import array
from numpy import mean
from numpy import std
from numpy import cov
from numpy.linalg import eig

print("Data: \n",national.head(1) )
# calculate the mean of each column
means = mean(national1.T, axis=1)
print("\n Means: \n", means)
# center columns by subtracting column means
centered_data = national1 - means
print("\n Centered Data: \n", centered_data)
# calculate the standard deviation of each column
std_devs = std(national1.T, axis=1)
print('\n Standard Deviations: \n', std_devs)
# complete the standardization by dividing by std_dev
standardized_data = centered_data / std_devs
print("\n Standardized Data: \n", standardized_data)
# calculate covariance matrix of centered matrix
# We're not going to find the covaraicne matrix by the (Z^T)(Z) method, I trust this function more
covariance_matrix = cov(standardized_data.T)
print("\n Covariance Matrix: \n", covariance_matrix)
# eigendecomposition of covariance matrix
values, vectors = eig(covariance_matrix)
print("\n Eigenvectors: \n", vectors)
print("\n Eigenvalues: \n", values)
# project data
P = vectors.T.dot(covariance_matrix.T)
print("\n Projected Data: \n", P.T)
```

    Data:
               CONTROL  AGE1 METRO3  ...       FMTBURDEN  FMTREGION  FMTSTATUS
    0  '100003130103'    82    '3'  ...  '2 30% to 50%'       '-5'       '-5'

    [1 rows x 99 columns]

     Means:
     AGE1                    47.968932
    LMED                 68109.744309
    FMR                   1164.395181
    L30                  17712.125436
    L50                  29511.032076
    L80                  46207.567630
    IPOV                 15986.664229
    BEDRMS                   2.660308
    BUILT                 1966.432835
    TYPE                     1.065360
    VALUE               142032.790532
    VACANCY                 -5.458433
    NUNITS                  14.889052
    ROOMS                    5.631208
    WEIGHT                1885.195459
    PER                      1.886015
    ZINC2                61355.837623
    ZSMHC                 1061.613744
    STRUCTURETYPE            1.920973
    UTILITY                183.099576
    OTHERCOST               67.873030
    COST06                1525.576425
    COST12                2074.054208
    COST08                1697.144663
    COSTMED               1406.865145
    TOTSAL               44910.928117
    ASSISTED                -5.516262
    GLMED                68109.744309
    GL30                 17712.125436
    GL50                 29511.032076
                            ...
    INCRELAMIPCT           104.784829
    INCRELAMICAT             3.377795
    INCRELPOVPCT           358.745915
    INCRELPOVCAT             2.280158
    INCRELFMRPCT           127.677315
    INCRELFMRCAT             1.463113
    COST06RELAMIPCT         87.021707
    COST06RELAMICAT          3.949144
    COST06RELPOVPCT        352.582333
    COST06RELPOVCAT          2.487921
    COST06RELFMRPCT        124.754094
    COST06RELFMRCAT          2.376307
    COST08RELAMIPCT         96.214197
    COST08RELAMICAT          4.162997
    COST08RELPOVPCT        392.790898
    COST08RELPOVCAT          2.526288
    COST08RELFMRPCT        137.651042
    COST08RELFMRCAT          2.424901
    COST12RELAMIPCT        116.408711
    COST12RELAMICAT          4.501278
    COST12RELPOVPCT        481.123050
    COST12RELPOVCAT          2.574758
    COST12RELFMRPCT        165.983691
    COST12RELFMRCAT          2.488990
    COSTMedRELAMIPCT        80.661251
    COSTMedRELAMICAT         3.770900
    COSTMedRELPOVPCT       324.761274
    COSTMedRELPOVCAT         2.450376
    COSTMedRELFMRPCT       115.830454
    COSTMedRELFMRCAT         2.328922
    Length: 67, dtype: float64

     Centered Data:
                 AGE1          LMED  ...  COSTMedRELFMRPCT  COSTMedRELFMRCAT
    0      34.031068   5628.255691  ...        -51.483518         -0.328922
    1       2.031068 -12263.744309  ...        -19.558774         -0.328922
    2       5.031068 -12263.744309  ...        -18.737257         -0.328922
    3      19.031068 -12263.744309  ...         32.926155          0.671078
    4     -21.968932  -7118.744309  ...        -12.845380          0.671078
    5       8.031068  -6043.744309  ...        -10.046588          0.671078
    6       2.031068  -7118.744309  ...         68.546431          0.671078
    7     -21.968932 -15787.744309  ...         10.430865          0.671078
    8      12.031068 -17813.744309  ...         -7.447178          0.671078
    9     -21.968932  -4888.744309  ...         83.444908          0.671078
    10     11.031068 -12263.744309  ...         47.499921          0.671078
    11      6.031068  -3747.744309  ...         21.749601          0.671078
    12    -22.968932  -3747.744309  ...         16.393247          0.671078
    13      5.031068  -3927.744309  ...         80.797989          0.671078
    14      4.031068  -3927.744309  ...        117.767430          0.671078
    15     19.031068  -3747.744309  ...         71.730778          0.671078
    16    -25.968932 -12263.744309  ...        -30.844619         -0.328922
    17      7.031068   9456.255691  ...         18.469096          0.671078
    18      9.031068  -4709.744309  ...          0.452711          0.671078
    19     36.031068   1390.255691  ...         85.525136          0.671078
    20      7.031068  -8953.744309  ...        -58.700723         -0.328922
    21     26.031068 -11564.744309  ...        -69.267062         -1.328922
    22     19.031068  -8953.744309  ...        126.312534          0.671078
    23     22.031068  -8953.744309  ...         37.724167          0.671078
    24     11.031068  -5851.744309  ...         96.286054          0.671078
    25     14.031068  -6817.744309  ...        164.684445          0.671078
    26    -15.968932 -11496.744309  ...        -41.612382         -0.328922
    27    -56.968932 -11496.744309  ...        -31.613864         -0.328922
    28      6.031068  -8049.744309  ...         56.465927          0.671078
    29    -22.968932  -8049.744309  ...        -60.623868         -0.328922
    ...          ...           ...  ...               ...               ...
    64505  -1.968932  37790.255691  ...        -33.225984         -0.328922
    64506 -17.968932  -5109.744309  ...        -41.976874         -0.328922
    64507 -21.968932   5898.255691  ...        -26.369670         -0.328922
    64508  -3.968932   5490.255691  ...         -4.650330          0.671078
    64509  14.031068  21416.255691  ...         87.598456          0.671078
    64510 -10.968932  18214.255691  ...        -31.693325         -0.328922
    64511   2.031068  11090.255691  ...        192.263958          0.671078
    64512  10.031068  27262.255691  ...         29.032916          0.671078
    64513  15.031068  27371.255691  ...          0.084909          0.671078
    64514  36.031068  18214.255691  ...         20.808635          0.671078
    64515 -56.968932  11090.255691  ...          0.217267          0.671078
    64516 -17.968932  -5109.744309  ...        -78.891140         -1.328922
    64517   9.031068  21090.255691  ...         29.063006          0.671078
    64518  13.031068   -209.744309  ...         20.998190          0.671078
    64519  10.031068  33090.255691  ...        -21.663202         -0.328922
    64520 -56.968932  -3509.744309  ...        -36.961407         -0.328922
    64521 -15.968932 -19035.744309  ...         44.616307          0.671078
    64522  -0.968932  -5109.744309  ...        -61.556370         -0.328922
    64523 -22.968932  -1909.744309  ...        -84.383913         -1.328922
    64524 -15.968932  39190.255691  ...         90.897591          0.671078
    64525 -56.968932   1390.255691  ...        -34.624161         -0.328922
    64526  10.031068  -6409.744309  ...        -54.165501         -0.328922
    64527   6.031068  -4883.744309  ...        -16.115355         -0.328922
    64528  23.031068 -29609.744309  ...        -66.137203         -1.328922
    64529 -15.968932  37790.255691  ...          0.901063          0.671078
    64530   7.031068 -12180.744309  ...          8.450121          0.671078
    64531 -10.968932   5490.255691  ...       -115.830454         -1.328922
    64532 -24.968932  18190.255691  ...        -33.823790         -0.328922
    64533   9.031068  11549.255691  ...        -61.674610         -0.328922
    64534  18.031068 -17386.744309  ...       -115.830454         -1.328922

    [64535 rows x 67 columns]

     Standard Deviations:
     AGE1                    22.869197
    LMED                 12371.081326
    FMR                    394.116135
    L30                   4441.530079
    L50                   7406.955023
    L80                  10640.534524
    IPOV                  7219.632457
    BEDRMS                   1.093770
    BUILT                   26.304522
    TYPE                     0.459338
    VALUE               249025.885846
    VACANCY                  2.019715
    NUNITS                  54.745798
    ROOMS                    1.904518
    WEIGHT                1245.581424
    PER                      2.566314
    ZINC2                74404.724446
    ZSMHC                  982.266961
    STRUCTURETYPE            1.470123
    UTILITY                130.168606
    OTHERCOST              145.527203
    COST06                1671.817021
    COST12                2596.899568
    COST08                1957.408460
    COSTMED               1477.748430
    TOTSAL               64806.436888
    ASSISTED                 4.499189
    GLMED                12371.081326
    GL30                  4441.530079
    GL50                  7406.955023
                            ...
    INCRELAMIPCT           123.348924
    INCRELAMICAT             4.086854
    INCRELPOVPCT           430.184089
    INCRELPOVCAT             3.282747
    INCRELFMRPCT           144.157302
    INCRELFMRCAT             2.957528
    COST06RELAMIPCT         84.322646
    COST06RELAMICAT          2.036991
    COST06RELPOVPCT        422.723235
    COST06RELPOVCAT          3.282609
    COST06RELFMRPCT        109.647861
    COST06RELFMRCAT          0.725376
    COST08RELAMIPCT         98.154626
    COST08RELAMICAT          2.103786
    COST08RELPOVPCT        492.453025
    COST08RELPOVCAT          3.287969
    COST08RELFMRPCT        127.421285
    COST08RELFMRCAT          0.722480
    COST12RELAMIPCT        129.605616
    COST12RELAMICAT          2.195651
    COST12RELPOVPCT        648.784998
    COST12RELPOVCAT          3.294748
    COST12RELFMRPCT        168.182286
    COST12RELFMRCAT          0.713634
    COSTMedRELAMIPCT        75.057722
    COSTMedRELAMICAT         1.975247
    COSTMedRELPOVPCT       375.381179
    COSTMedRELPOVCAT         3.276667
    COSTMedRELFMRPCT        97.840356
    COSTMedRELFMRCAT         0.726535
    Length: 67, dtype: float64

     Standardized Data:
                AGE1      LMED  ...  COSTMedRELFMRPCT  COSTMedRELFMRCAT
    0      1.488075  0.454953  ...         -0.526199         -0.452728
    1      0.088812 -0.991324  ...         -0.199905         -0.452728
    2      0.219993 -0.991324  ...         -0.191508         -0.452728
    3      0.832170 -0.991324  ...          0.336529          0.923669
    4     -0.960634 -0.575434  ...         -0.131289          0.923669
    5      0.351174 -0.488538  ...         -0.102683          0.923669
    6      0.088812 -0.575434  ...          0.700595          0.923669
    7     -0.960634 -1.276181  ...          0.106611          0.923669
    8      0.526082 -1.439950  ...         -0.076116          0.923669
    9     -0.960634 -0.395175  ...          0.852868          0.923669
    10     0.482355 -0.991324  ...          0.485484          0.923669
    11     0.263720 -0.302944  ...          0.222297          0.923669
    12    -1.004361 -0.302944  ...          0.167551          0.923669
    13     0.219993 -0.317494  ...          0.825815          0.923669
    14     0.176266 -0.317494  ...          1.203669          0.923669
    15     0.832170 -0.302944  ...          0.733141          0.923669
    16    -1.135542 -0.991324  ...         -0.315255         -0.452728
    17     0.307447  0.764384  ...          0.188768          0.923669
    18     0.394901 -0.380706  ...          0.004627          0.923669
    19     1.575528  0.112379  ...          0.874129          0.923669
    20     0.307447 -0.723764  ...         -0.599964         -0.452728
    21     1.138259 -0.934821  ...         -0.707960         -1.829124
    22     0.832170 -0.723764  ...          1.291006          0.923669
    23     0.963351 -0.723764  ...          0.385569          0.923669
    24     0.482355 -0.473018  ...          0.984114          0.923669
    25     0.613536 -0.551103  ...          1.683195          0.923669
    26    -0.698273 -0.929324  ...         -0.425309         -0.452728
    27    -2.491077 -0.929324  ...         -0.323117         -0.452728
    28     0.263720 -0.650690  ...          0.577123          0.923669
    29    -1.004361 -0.650690  ...         -0.619620         -0.452728
    ...         ...       ...  ...               ...               ...
    64505 -0.086095  3.054725  ...         -0.339594         -0.452728
    64506 -0.785726 -0.413039  ...         -0.429034         -0.452728
    64507 -0.960634  0.476778  ...         -0.269517         -0.452728
    64508 -0.173549  0.443798  ...         -0.047530          0.923669
    64509  0.613536  1.731155  ...          0.895320          0.923669
    64510 -0.479638  1.472325  ...         -0.323929         -0.452728
    64511  0.088812  0.896466  ...          1.965078          0.923669
    64512  0.438628  2.203708  ...          0.296738          0.923669
    64513  0.657263  2.212519  ...          0.000868          0.923669
    64514  1.575528  1.472325  ...          0.212679          0.923669
    64515 -2.491077  0.896466  ...          0.002221          0.923669
    64516 -0.785726 -0.413039  ...         -0.806325         -1.829124
    64517  0.394901  1.704803  ...          0.297045          0.923669
    64518  0.569809 -0.016954  ...          0.214617          0.923669
    64519  0.438628  2.674807  ...         -0.221414         -0.452728
    64520 -2.491077 -0.283706  ...         -0.377773         -0.452728
    64521 -0.698273 -1.538729  ...          0.456011          0.923669
    64522 -0.042368 -0.413039  ...         -0.629151         -0.452728
    64523 -1.004361 -0.154372  ...         -0.862465         -1.829124
    64524 -0.698273  3.167892  ...          0.929040          0.923669
    64525 -2.491077  0.112379  ...         -0.353884         -0.452728
    64526  0.438628 -0.518123  ...         -0.553611         -0.452728
    64527  0.263720 -0.394771  ...         -0.164711         -0.452728
    64528  1.007078 -2.393465  ...         -0.675971         -1.829124
    64529 -0.698273  3.054725  ...          0.009210          0.923669
    64530  0.307447 -0.984614  ...          0.086366          0.923669
    64531 -0.479638  0.443798  ...         -1.183872         -1.829124
    64532 -1.091815  1.470385  ...         -0.345704         -0.452728
    64533  0.394901  0.933569  ...         -0.630360         -0.452728
    64534  0.788443 -1.405434  ...         -1.183872         -1.829124

    [64535 rows x 67 columns]

     Covariance Matrix:
     [[ 1.0000155   0.0231727   0.0687571  ...  0.66315292  0.04648537
      -0.00189556]
     [ 0.0231727   1.0000155   0.53782979 ...  0.0817588   0.0603434
       0.06963308]
     [ 0.0687571   0.53782979  1.0000155  ...  0.17514851  0.15078091
       0.1436183 ]
     ...
     [ 0.66315292  0.0817588   0.17514851 ...  1.0000155   0.13227387
       0.23607225]
     [ 0.04648537  0.0603434   0.15078091 ...  0.13227387  1.0000155
       0.5670847 ]
     [-0.00189556  0.06963308  0.1436183  ...  0.23607225  0.5670847
       1.0000155 ]]

     Eigenvectors:
     [[-4.62478561e-02+0.00000000e+00j -8.21314989e-02+0.00000000e+00j
      -1.85732547e-01+0.00000000e+00j ... -1.51563132e-11-9.92199330e-13j
      -2.09262019e-11+0.00000000e+00j -1.52943470e-12+0.00000000e+00j]
     [-6.40107358e-02+0.00000000e+00j -1.30915732e-01+0.00000000e+00j
       1.91813318e-01+0.00000000e+00j ... -4.19682725e-01-0.00000000e+00j
       2.34546495e-02+0.00000000e+00j -7.22010628e-02+0.00000000e+00j]
     [-1.17300434e-01+0.00000000e+00j -1.18726867e-01+0.00000000e+00j
       1.67540933e-01+0.00000000e+00j ...  3.73789904e-11+2.71507165e-12j
       5.22638183e-11+0.00000000e+00j  4.01947387e-12+0.00000000e+00j]
     ...
     [-9.22579617e-02+0.00000000e+00j -1.50161717e-01+0.00000000e+00j
      -2.38642401e-01+0.00000000e+00j ...  4.36739690e-12+1.09309782e-13j
       5.57697997e-12+0.00000000e+00j  1.99518597e-13+0.00000000e+00j]
     [-1.57731289e-01+0.00000000e+00j  1.47227566e-01+0.00000000e+00j
      -2.78502258e-03+0.00000000e+00j ...  8.63530849e-02+8.06292213e-02j
       2.42076412e-01+0.00000000e+00j  1.35626706e-01+0.00000000e+00j]
     [-1.26200468e-01+0.00000000e+00j  7.98132697e-02+0.00000000e+00j
      -3.92928252e-02+0.00000000e+00j ...  8.32916508e-11+5.48859943e-12j
       1.15322684e-10+0.00000000e+00j  8.36708549e-12+0.00000000e+00j]]

     Eigenvalues:
     [ 2.47053173e+01+0.00000000e+00j  1.10217612e+01+0.00000000e+00j
      7.96502310e+00+0.00000000e+00j  4.35570130e+00+0.00000000e+00j
      3.04968737e+00+0.00000000e+00j  2.84186674e+00+0.00000000e+00j
      2.12131212e+00+0.00000000e+00j  1.30990735e+00+0.00000000e+00j
      1.01979814e+00+0.00000000e+00j  9.99759644e-01+0.00000000e+00j
      9.04961412e-01+0.00000000e+00j  8.33231109e-01+0.00000000e+00j
      8.05831737e-01+0.00000000e+00j  6.80028166e-01+0.00000000e+00j
      6.01906909e-01+0.00000000e+00j  5.44122222e-01+0.00000000e+00j
      4.31134122e-01+0.00000000e+00j  4.05073585e-01+0.00000000e+00j
      3.45892440e-01+0.00000000e+00j  3.16121334e-01+0.00000000e+00j
      2.90139059e-01+0.00000000e+00j  2.49330558e-01+0.00000000e+00j
      1.91950339e-01+0.00000000e+00j  1.61653151e-01+0.00000000e+00j
      1.43182652e-01+0.00000000e+00j  1.11790472e-01+0.00000000e+00j
      8.78724054e-02+0.00000000e+00j  5.68228835e-02+0.00000000e+00j
      5.40889351e-02+0.00000000e+00j  5.06565124e-02+0.00000000e+00j
      4.78988994e-02+0.00000000e+00j  4.41337874e-02+0.00000000e+00j
      4.14488936e-02+0.00000000e+00j  3.50630868e-02+0.00000000e+00j
      2.76531253e-02+0.00000000e+00j  2.62193760e-02+0.00000000e+00j
      2.16445090e-02+0.00000000e+00j  1.77614989e-02+0.00000000e+00j
      1.66007844e-02+0.00000000e+00j  1.40949861e-02+0.00000000e+00j
      1.16897059e-02+0.00000000e+00j  1.06990459e-02+0.00000000e+00j
      8.58345179e-03+0.00000000e+00j  5.07073611e-03+0.00000000e+00j
      4.22025230e-03+0.00000000e+00j  3.49285304e-03+0.00000000e+00j
      3.03335569e-03+0.00000000e+00j  2.13778754e-03+0.00000000e+00j
      1.39056606e-03+0.00000000e+00j  1.12390230e-03+0.00000000e+00j
      6.64734415e-04+0.00000000e+00j  4.67661272e-04+0.00000000e+00j
      2.07459320e-05+0.00000000e+00j  1.40112108e-07+0.00000000e+00j
      5.24509486e-13+0.00000000e+00j  1.73414237e-15+0.00000000e+00j
      1.16252103e-15+3.89268665e-17j  1.16252103e-15-3.89268665e-17j
     -1.29844142e-15+0.00000000e+00j -8.16828107e-16+0.00000000e+00j
      3.00347474e-16+2.89754682e-17j  3.00347474e-16-2.89754682e-17j
      1.24345616e-16+0.00000000e+00j  2.17937170e-17+1.03271846e-16j
      2.17937170e-17-1.03271846e-16j -1.58099483e-16+0.00000000e+00j
     -6.46308724e-17+0.00000000e+00j]

     Projected Data:
     [[-1.14256796e+00+0.00000000e+00j -9.05233769e-01+0.00000000e+00j
      -1.47936403e+00+0.00000000e+00j ... -7.59207661e-17-3.19667151e-17j
      -1.09827199e-16+0.00000000e+00j -1.09330555e-16+0.00000000e+00j]
     [-1.58140554e+00+0.00000000e+00j -1.44292194e+00+0.00000000e+00j
       1.52779751e+00+0.00000000e+00j ...  1.23974996e-16+2.19583827e-16j
      -1.02524693e-16+0.00000000e+00j  2.86139681e-16+0.00000000e+00j]
     [-2.89794444e+00+0.00000000e+00j -1.30857918e+00+0.00000000e+00j
       1.33446740e+00+0.00000000e+00j ...  1.18337750e-16+1.62079450e-16j
      -2.13834522e-16+0.00000000e+00j  2.05391369e-16+0.00000000e+00j]
     ...
     [-2.27926222e+00+0.00000000e+00j -1.65504658e+00+0.00000000e+00j
      -1.90079224e+00+0.00000000e+00j ...  2.08102529e-16+4.39208711e-18j
      -3.05519966e-16+0.00000000e+00j -7.24335151e-17+0.00000000e+00j]
     [-3.89680154e+00+0.00000000e+00j  1.62270707e+00+0.00000000e+00j
      -2.21827692e-02+0.00000000e+00j ...  1.96334965e-16+2.88764832e-17j
      -4.51328259e-16+0.00000000e+00j -6.20752697e-17+0.00000000e+00j]
     [-3.11782260e+00+0.00000000e+00j  8.79682800e-01+0.00000000e+00j
      -3.12968261e-01+0.00000000e+00j ...  2.35427049e-16+6.34783697e-17j
      -2.60346761e-16+0.00000000e+00j  1.71421275e-16+0.00000000e+00j]]

```
# Principal Component Analysis
from numpy import array
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
# define a matrix
print("Data: \n", national1)
# Standardize the Data
# Instantiate a Standard Scaler object
scaler = StandardScaler()
# Use the object to fit_transform our data
Z = scaler.fit_transform(national1)
print("\n Standardized Data: \n", Z)
# create the PCA instance
# The integer value that we're passing in says
# how many new columns we want to retain from PCA
pca = PCA(2)
# fit on data
pca.fit(Z)
# access values and vectors
print("\n Eigenvectors: \n", pca.components_)
print("\n Eigenvalues: \n",pca.explained_variance_)
# transform data
B = pca.transform(Z)
print("\n Projected Data: \n", B)
```

    Data:
            AGE1    LMED   FMR  ...  COSTMedRELPOVCAT  COSTMedRELFMRPCT  COSTMedRELFMRCAT
    0        82   73738   956  ...                 4         64.346936                 2
    1        50   55846  1100  ...                 3         96.271680                 2
    2        53   55846  1100  ...                 4         97.093197                 2
    3        67   55846   949  ...                 4        148.756610                 3
    4        26   60991   737  ...                 3        102.985075                 3
    5        56   62066   657  ...                 4        105.783866                 3
    6        50   60991   988  ...                 4        184.376885                 3
    7        26   52322   773  ...                 4        126.261320                 3
    8        60   50296  1125  ...                 4        108.383276                 3
    9        26   63221   552  ...                 4        199.275362                 3
    10       59   55846   949  ...                 4        163.330375                 3
    11       54   64362  1239  ...                 4        137.580056                 3
    12       25   64362   751  ...                 4        132.223702                 3
    13       53   64182  1002  ...                 4        196.628443                 3
    14       52   64182  1114  ...                 4        233.597884                 3
    15       67   64362  1239  ...                 4        187.561232                 3
    16       22   55846   706  ...                 2         84.985836                 2
    17       55   77566  1277  ...                 4        134.299550                 3
    18       57   63400   993  ...                 3        116.283166                 3
    19       84   69500   975  ...                 4        201.355590                 3
    20       55   59156   794  ...                 2         57.129731                 2
    21       74   56545   752  ...                 2         46.563392                 1
    22       67   59156   794  ...                 4        242.142988                 3
    23       70   59156  1102  ...                 4        153.554621                 3
    24       59   62258   973  ...                 4        212.116508                 3
    25       62   61292   718  ...                 4        280.514900                 3
    26       32   56613  1349  ...                 2         74.218072                 2
    27       -9   56613   868  ...                -9         84.216590                 2
    28       54   60060  1528  ...                 4        172.296381                 3
    29       25   60060  1282  ...                 4         55.206586                 2
    ...     ...     ...   ...  ...               ...               ...               ...
    64505    46  105900  2058  ...                 4         82.604470                 2
    64506    30   63000  1243  ...                 4         73.853580                 2
    64507    26   74008   816  ...                 4         89.460784                 2
    64508    44   73600   966  ...                 3        111.180124                 3
    64509    62   89526  1256  ...                 4        203.428910                 3
    64510    37   86324  1721  ...                 4         84.137130                 2
    64511    50   79200   929  ...                 4        308.094413                 3
    64512    58   95372  1877  ...                 4        144.863370                 3
    64513    63   95481  1087  ...                 4        115.915363                 3
    64514    84   86324  1721  ...                 4        136.639089                 3
    64515    -9   79200   929  ...                -9        116.047722                 3
    64516    30   63000  1895  ...                 2         36.939314                 1
    64517    57   89200  1361  ...                 4        144.893461                 3
    64518    61   67900   782  ...                 4        136.828644                 3
    64519    58  101200  1423  ...                 4         94.167252                 2
    64520    -9   64600   672  ...                -9         78.869048                 2
    64521    32   49074   873  ...                 4        160.446761                 3
    64522    47   63000  1474  ...                 4         54.274084                 2
    64523    25   66200   636  ...                 1         31.446541                 1
    64524    32  107300  1412  ...                 4        206.728045                 3
    64525    -9   69500  1430  ...                -9         81.206294                 2
    64526    58   61700   973  ...                 3         61.664954                 2
    64527    54   63226   702  ...                 4         99.715100                 2
    64528    71   38500   652  ...                 2         49.693252                 1
    64529    32  105900  1285  ...                 4        116.731518                 3
    64530    55   55929   556  ...                 4        124.280576                 3
    64531    37   73600   966  ...                 1          0.000000                 1
    64532    23   86300  2701  ...                 4         82.006664                 2
    64533    57   79659   770  ...                 2         54.155844                 2
    64534    66   50723   542  ...                 1          0.000000                 1

    [64535 rows x 67 columns]

     Standardized Data:
     [[ 1.48807451  0.45495261 -0.52876592 ...  0.47292705 -0.52619921
      -0.45272761]
     [ 0.08881241 -0.99132355 -0.16339138 ...  0.16773884 -0.19990498
      -0.45272761]
     [ 0.21999323 -0.99132355 -0.16339138 ...  0.47292705 -0.19150847
      -0.45272761]
     ...
     [-1.09181499  1.47038526  3.89886301 ...  0.47292705 -0.34570387
      -0.45272761]
     [ 0.39490099  0.93356881 -1.00070803 ... -0.13744938 -0.63035962
      -0.45272761]
     [ 0.78844346 -1.40543448 -1.57921771 ... -0.44263759 -1.18387196
      -1.82912447]]

     Eigenvectors:
     [[ 0.04624786  0.06401074  0.11730043  0.09299022  0.09299694  0.09120063
       0.07372392  0.09786299  0.01512684 -0.01713051  0.16921811 -0.05926616
      -0.02310793  0.11213095  0.01673349  0.07798021  0.1269096   0.15151783
      -0.06918188  0.10654086  0.08170021  0.17885166  0.17780088  0.17876113
       0.17850649  0.10542514 -0.07564688  0.06401074  0.09299022  0.09299694
       0.09120063  0.09841392  0.12084989  0.12085026  0.12277979  0.11275531
       0.00103245  0.11495356  0.1046365   0.1174044   0.08711012  0.10211757
       0.07989779  0.16699835  0.15772115  0.16783159  0.09162475  0.16101733
       0.12630416  0.16870152  0.15683575  0.16808267  0.09078346  0.1637686
       0.12508048  0.1697506   0.15160474  0.16762697  0.08945239  0.16603868
       0.12065576  0.16477714  0.15713743  0.16719893  0.09225796  0.15773129
       0.12620047]
     [-0.0821315  -0.13091573 -0.11872687 -0.20280257 -0.20280043 -0.20054308
      -0.20367444 -0.05393915  0.03692533  0.00910396  0.08568103  0.17597577
       0.02160706 -0.03400746  0.01806353 -0.21431969 -0.03542262 -0.03329781
       0.02211243 -0.03506049  0.04974342  0.09155511  0.09066832  0.09136398
       0.09151112 -0.05203247 -0.07811373 -0.13091573 -0.20280257 -0.20280043
      -0.20054308 -0.24551599 -0.13817084 -0.13826251 -0.13124677 -0.13467023
      -0.00148804 -0.0044496  -0.13528629 -0.00708575 -0.16116366 -0.01589906
      -0.16785406  0.13297528  0.07427218  0.08150582 -0.15123189  0.1453508
       0.07651585  0.13112133  0.06903663  0.08320845 -0.15240094  0.14245983
       0.07238344  0.12739483  0.06058219  0.08524205 -0.15409075  0.13686613
       0.06693914  0.13411125  0.07864188  0.07976376 -0.15016172  0.14722757
       0.07981327]]

     Eigenvalues:
     [24.70531734 11.02176121]

     Projected Data:
     [[-2.66211742 -0.63254068]
     [ 0.65100075 -0.78207399]
     [-0.32065728  1.13780784]
     ...
     [ 2.53683445 -4.21461477]
     [-3.75831891 -1.50580651]
     [-7.95564964  1.49572966]]

```
len(B)
```

    64535

```
PC1 = pd.DataFrame(B)
```

```
plt.scatter(PC1[0], PC1[1])
plt.title("Data After PCA")
plt.xlabel('PC1')
plt.show()
```

# Stretch Goals

## 1) Perform further data exploration on the HADS national dataset (the version before we one-hot encoded it) Make scatterplots and see if you can see any resemblance between the original scatterplots and the plot of the principal components that you made in 7.1.

(You may or may not not see very much resemblance depending on the variables you choose, and that's ok!)

## 2) Study "Scree Plots" and then try and make one for your PCA dataset. How many principal conponents do you need to retain in order for your PCs to contain 90% of the explained variance?

We will present this topic formally at the beginning of tomorrow's lecture, so if you figure this stretch goal out, you're ahead of the game.

## 3) Explore further the intuition behind eigenvalues and eigenvectors by creating your very own eigenfaces:

Prioritize self-study over this stretch goal if you are not semi-comfortable with the topics of PCA, Eigenvalues, and Eigenvectors.

You don't necessarily have to use this resource, but this will get you started:
[Eigenface Tutorial](https://sandipanweb.wordpress.com/2018/01/06/eigenfaces-and-a-simple-face-detector-with-pca-svd-in-python/)


# License
https://raw.githubusercontent.com/computational-sediment-hyd/a-rudimentary-knowledge-of-river-bed-variation/master/LICENSE

# import module

```python
import numpy as np
import pandas as pd
import numba
import sys
import os
import json
import glob as glob
```

# Governing Equation of River flow

\begin{align}
    \frac{\partial A}{\partial t}+\frac{\partial Q}{\partial x} = 0 \\
\end{align}

\begin{align}
    \frac{\partial Q}{\partial t}+\frac{\partial}{\partial x}\left(\frac{ Q^2}{ A}\right) + gA\frac{\partial H}{\partial x}+gAi_e = 0
\end{align}

```python
@numba.jit(nopython=True, parallel=False)
def flowmodel(A, Q, Adown, Qup, dAb, zb, B, dx, dt, g, manning):
    imax = len(A)
    Anew, Qnew = np.zeros(imax), np.zeros(imax)

# continuous equation
    for i in numba.prange(1, imax-1) : Anew[i] = A[i] - dt * ( Q[i] - Q[i-1] ) / dx

    Anew[0], Anew[-1] = Anew[1], Adown

# moumentum equation
    for i in numba.prange(1,imax-1):
        ip, ic, im = (i+1, i, i-1)
        Cr1 = 0.5*( Q[ic]/A[ic] + Q[ip]/A[ip] )*dt/dx
        Cr2 = 0.5*( Q[ic]/A[ic] + Q[im]/A[im] )*dt/dx
        dHdx1 = ( Anew[ip]/B[ip] + zb[ip] + dAb[ip]/B[ip] - Anew[ic]/B[ic] - zb[ic] - dAb[ic]/B[ic] ) / dx
        dHdx2 = ( Anew[ic]/B[ic] + zb[ic] + dAb[ic]/B[ic] - Anew[im]/B[im] - zb[im] - dAb[im]/B[im] ) / dx
        dHdx = (1.0 - Cr1) * dHdx1 + Cr2 * dHdx2

        Qnew[ic] = Q[ic] - dt * ( Q[ic]**2/A[ic] - Q[im]**2/A[im] ) / dx  \
                   - dt * g * Anew[ic] * dHdx \
                   - dt * g * A[ic] * manning**2 * Q[ic]**2 / B[ic]**2 / ( A[ic]/B[ic] )**(10.0/3.0)

    Qnew[0], Qnew[-1] = Qup, Qnew[-2]

    return Anew, Qnew
```

# Governing Equation of River Bed : Hirano model

\begin{align}
    (1-\lambda)\frac{\partial A_{b}}{\partial t}+\frac{\partial }{\partial x} \sum_{i=1}^n ( Q_{bi}P_i)= 0 \\
\end{align}

\begin{align}
    \frac{\partial P_i}{\partial t} &= - \frac{1}{(1-\lambda)E_dB} \frac{\partial (Q_{bi}P_i)}{\partial x} - \frac{P_{si}}{E_d B}\frac{\partial A_b}{\partial t} \\
    &= - \frac{1}{E_d B}\left(\frac{\partial A_{bi}}{\partial t} + P_{si}\frac{\partial A_b}{\partial t}\right)
\end{align}

\begin{align}
&P_{si} =
\left\{ \begin{array}{ll}
    P_i \mbox{ in exchange layer} & \left(\dfrac{\partial A_b}{\partial t} > 0 \right) \\
    P_i \mbox{ under exchange layer} & \left(\dfrac{\partial A_b}{\partial t} < 0 \right) \\
\end{array} \right. \\
\end{align}

$Q_{bi}$ : Ashida-Michiue Eq. and  Egiazaroff Eq.

$E_d$ : thickness of exchange layer

\begin{align}
    \frac{\tau_{*c i}}{\tau_{*cm}} &= 0.85 \frac{D_m}{D_i} \qquad & \left(\frac{D_i}{D_m} < 0.4 \right) \\
    \dfrac{\tau_{*ci}}{\tau_{*cm}} &= \left( \dfrac{ \displaystyle \log_e 19 }{ \displaystyle \log_e \left(19\dfrac{D_i}{D_m} \right) } \right)^2 \qquad & \left(\frac{D_i}{D_m} \geq 0.4 \right)
\end{align}

```python
@numba.jit(nopython=True, parallel=False)
def sedimentmodel(dAb, dratio, A, Q, B, dx, dt, g, manning, dsize, dratioStandard, hExlayer, Qbup):
    rhosw = 1.65 # grain specific gravity in water
    porosity = 0.4
    tscAve = 0.05 # critical tractive force of average grain size

    dAbnew = np.zeros_like( dAb )
    drationew = np.zeros_like( dratio )
    Qbsub = np.zeros_like( dratio )
    dAbsub = np.zeros_like( dratio )

    imax, lmax = len(dratio), len(dratio[0])

    for i in numba.prange(imax):
        Ap, Qp, dr, Bp = A[i], Q[i], dratio[i], B[i]
        dAve = np.sum(dsize * dr)
        us = np.sqrt(g * Ap/Bp) * Qp/Ap * manning / (Ap/Bp)**(2/3)
        tsAve = us**2.0/rhosw/g/dAve
        use = Qp/Ap / ( 6.0 + 2.5 * np.log( Ap/Bp/dAve/( 1.0+2.0*tsAve) ) )
        Kc=1.0

        for l in range(lmax):
            dri, dsi = dr[l], dsize[l]
            ts  = us**2.0 /rhosw/g/dsi
            tse = use**2.0/rhosw/g/dsi

# Egiazaroff Eq.
            x = dsi/dAve
            tscbytscm = (np.log(19)/np.log(19*x))**2 if x > 0.4 else 0.85/x
            tsc = Kc * tscbytscm * tscAve

            if ts > tsc :
# Ashida-Michiue Eq.
                Qbsub[i][l] = np.sqrt(rhosw * g * dsi**3.0) \
                        * 17.0 * tse**1.5 * ( 1.0 - tsc / ts) * ( 1.0 - np.sqrt(tsc / ts) ) \
                        * dri * B[i]
            else:
                Qbsub[i][l] = 0.0

    for i in numba.prange(imax):
        Qbs = Qbsub[i]
        if i == 0:
            if np.min(Qbup) < 0.0 : # equilibrium sand supply
                dAbsub[i][:] = 0.0
            else:
                for l in range(lmax):
                    dAbsub[i][l] = - dt / (1.0 - porosity) * ( Qbsub[i][l]-Qbup[l] )/dx
                    if np.abs(dAbsub[i][l]) > hExlayer :
                        print(i) ; print('dzsub-error')
        else:
            for l in range(lmax):
                dAbsub[i][l] = - dt / (1.0 - porosity) * ( Qbsub[i][l] - Qbsub[i-1][l] )/dx
                if np.abs(dAbsub[i][l]) > hExlayer :
                    print(i) ; print('dzsub-error')

    for i in numba.prange(imax):
        Qbs = Qbsub[i]
# update dAb
        dAball = np.sum( dAbsub[i] )
        if np.abs(dAball) > hExlayer:
            print(i) ; print('dz-error')

        dAbnew[i] = dAb[i] + dAball

# update dratio
        dratioIn = dratioStandard[i][:] if dAball < 0.0 else dratio[i][:]

        for l in range(lmax):
            drationew[i][l] = dratio[i][l] + ( dAbsub[i][l] - dAball * dratioIn[l] ) /hExlayer/B[i]
            if drationew[i][l] < 0.0 : drationew[i][l] = 0.0

# correct ration so that sum of ration become 100%
        sumdr = np.sum( drationew[i] )
        drationew[i] /= sumdr

    return dAbnew, drationew, Qbsub
```

# main

```python
def bedvariation(
dx,dt,manning,totalTime,outTimeStep,RunUpTime
,dsize ,dratioStandard ,dratio ,hExlayer ,A ,Q ,B ,zb ,dAb ,Qup ,Adown
,outputfilename, screenclass
):

    g = 9.8
    Qbup = np.full( ( len(dsize) ), -9999.0 ) # when equilibrium sand supply, set qbup to minus value

# run-up calculation
    for i in range(int(RunUpTime/dt)):
        ib = ( ( dAb[-2]/B[-2] + zb[-2] ) - ( dAb[-1]/B[-1] + zb[-1] ) )/dx
        Adownp = Adown(0.0, Q[-1], dAb[-1]/B[-1], ib)
        Qupp = Qup(0.0)
        A, Q = flowmodel(A, Q, Adownp, Qupp, dAb, zb, B, dx, dt, g, manning)

# main calculation
    for i in range( int(totalTime/dt) ):

# cal bed variation
        dAb, dratio, Qbsub = sedimentmodel(dAb, dratio, A, Q, B, dx, dt, g, manning, dsize, dratioStandard, hExlayer, Qbup)

# cal water profile
        ib = ( ( dAb[-2]/B[-2] + zb[-2] ) - ( dAb[-1]/B[-1] + zb[-1] ) )/dx
        Adownp = Adown(i*dt, Q[-1], dAb[-1]/B[-1], ib)
        Qupp = Qup(i*dt)
        A, Q = flowmodel(A, Q, Adownp, Qupp, dAb, zb, B, dx, dt, g, manning)

# output
        if( int(i*dt) % int(outTimeStep) ) == 0 :
            print( str( int(i*dt) ) + ' second')
            profile = []
            for ii, (ap, qp, z, r, q) in enumerate(zip(A, Q, dAb, dratio, Qbsub)) :
                profile.append({'distance' : int(ii*dx), 'A':ap, 'Q':qp, 'dAb':z, 'ratio':list(r), 'Qb':list(q)})
            json.dump( {'time':int(i*dt), 'profile' : profile}, open('%010d' % int(i*dt) + 'sec.json', 'w') )

# join json files
    d = [ json.load( open(f, 'r') ) for f in glob.glob('*sec.json') ]
# delete json files
    for f in glob.glob('*sec.json') : os.remove(f)

    d.sort(key=lambda x: x['time'])

    cond = {'width':list(B), 'elevation':list(zb), 'screenclass':list(screenclass), 'dsize':list(dsize) }
    json.dump( {'condition':cond, 'output':d}, open(outputfilename, 'w') )
```


# Question repository

A list of open questions and possibly ambiguous stuff encountered throughout the material.

TODO: Tag exam-related ones appropriately, to differentiate them from (exclusively) curiosity-related ones.

**Note:** An alternative design would consist of adding a questions section to every notebook, tagging it appropriately using IPython metadata, and then using something like a Python/shell script to print all open questions in a centralized way. However...

## 2. Approximate retrieval
* Why perform first step of hashing if we only have a small number of features (e.g. 100)? If many features, why not just do a PCA first?
    - might be because we want the shingle representation (0s and 1s) for the nice properties that Jaccard similarity offers us

## 3. Classification
 * When transitioning from the first SVM formulation (with slack variables), to the second one aren't we loosening any constraint by fixing $\xi$?
     * (tentative) It seems we're not, since we're taking multiple cases into consideration and merging them together into a single formulation using max.
 * Slide 04:18: Is the first (primal) SVM formulation a (ii)-type one (since it has a minimization and its constraint as separate equation), or is it not eligible for this categorization?
 * Slide 06:15: How do we go from step 1 to 2? Isn't the $\lambda \| w \|_2^2$ term outside the sum?
  - yes it is, but the sum has a convenient $\frac{1}{T}$ in front of it, so we're safe to add the regularization term into the sum.

 * Why do some SVM OCP implementations *always* regularize, even when the model was not updated at that stage.

## 4. Non-linear classification
 * How exactly is the Lagrangian dual reformulation step (SVMs) different from the first time we reformulated the SVM problem statement to get rid of the slack variables?
     - it's different because we changed the objective! We no longer have $\min_w$ or $\min_{w, \xi}$, it's now a maximization of the Lagrance coefficients: $\max_\alpha$; it's not a *reformulation*, but an *equivalent problem*

## 5. Active learning
 * When doing active learning based on uncertainty sampling, how exactly do we know when we can safely infer some labels?

## 6. Clustering
 * Homework 5 solution, 2.2: Why is:

\begin{equation}
    \operatorname{Var}_{\hat{x}_i \sim q}\left [ \frac{1}{m} \sum_{i=1}^{m} \frac{d(\hat{x}_i; \mu)}{q(\hat{x}_i)} \right ] = \frac{1}{m^2} \sum_{i=1}^m \operatorname{Var}_{x_i \sim q} \left[ \frac{d(x_i; \mu)}{q(x_i)} \right]
\end{equation}

 * And why doe we still have the $i$ subscript in the variance formulation? Can't we just write $x \tilde{} q$?
 * Have to discuss this with friends!

## 7. Bandits

## 8. Exam-specific (and/or for review session on Jan 20)

 * Exam 2014 Problem 6 (Submodular functions)
     - solved by Syd in The Notes. Yay!

```python

```