#  パッケージ導入

```python
#@title
pip install econml[tf]
```

    Requirement already satisfied: econml[tf] in /usr/local/lib/python3.7/dist-packages (0.12.0)
    Requirement already satisfied: shap<0.40.0,>=0.38.1 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (0.39.0)
    Requirement already satisfied: statsmodels>=0.10 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (0.10.2)
    Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (1.1.5)
    Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (1.18.5)
    Requirement already satisfied: dowhy in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (0.6)
    Requirement already satisfied: sparse in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (0.13.0)
    Requirement already satisfied: lightgbm in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (2.2.3)
    Requirement already satisfied: scikit-learn>0.22.0 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (1.0.1)
    Requirement already satisfied: numba!=0.42.1 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (0.51.2)
    Requirement already satisfied: joblib>=0.13.0 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (1.1.0)
    Requirement already satisfied: scipy>1.4.0 in /usr/local/lib/python3.7/dist-packages (from econml[tf]) (1.4.1)
    Collecting tensorflow<2.3,>1.10
      Using cached tensorflow-2.2.3-cp37-cp37m-manylinux2010_x86_64.whl (516.4 MB)
    Collecting keras<2.4
      Using cached Keras-2.3.1-py2.py3-none-any.whl (377 kB)
    Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/dist-packages (from keras<2.4->econml[tf]) (1.0.8)
    Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras<2.4->econml[tf]) (3.13)
    Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from keras<2.4->econml[tf]) (1.1.2)
    Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras<2.4->econml[tf]) (1.15.0)
    Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras<2.4->econml[tf]) (2.10.0)
    Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba!=0.42.1->econml[tf]) (0.34.0)
    Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba!=0.42.1->econml[tf]) (57.4.0)
    Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>0.22.0->econml[tf]) (3.0.0)
    Requirement already satisfied: slicer==0.0.7 in /usr/local/lib/python3.7/dist-packages (from shap<0.40.0,>=0.38.1->econml[tf]) (0.0.7)
    Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from shap<0.40.0,>=0.38.1->econml[tf]) (1.3.0)
    Requirement already satisfied: tqdm>4.25.0 in /usr/local/lib/python3.7/dist-packages (from shap<0.40.0,>=0.38.1->econml[tf]) (4.62.3)
    Requirement already satisfied: patsy>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from statsmodels>=0.10->econml[tf]) (0.5.2)
    Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->econml[tf]) (2.8.2)
    Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->econml[tf]) (2018.9)
    Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (0.37.0)
    Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (1.42.0)
    Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (0.2.0)
    Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (1.1.0)
    Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (0.12.0)
    Collecting tensorflow-estimator<2.3.0,>=2.2.0
      Using cached tensorflow_estimator-2.2.0-py2.py3-none-any.whl (454 kB)
    Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (1.13.3)
    Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (0.3.3)
    Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (1.6.3)
    Collecting tensorboard<2.3.0,>=2.2.0
      Using cached tensorboard-2.2.2-py3-none-any.whl (3.0 MB)
    Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (3.3.0)
    Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.3,>1.10->econml[tf]) (3.17.3)
    Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (0.4.6)
    Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (1.8.0)
    Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (1.0.1)
    Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (2.23.0)
    Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (1.35.0)
    Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (3.3.6)
    Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (0.2.8)
    Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (4.2.4)
    Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (4.8)
    Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (1.3.0)
    Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (4.8.2)
    Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (3.10.0.2)
    Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (3.6.0)
    Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (0.4.8)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (3.0.4)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (1.24.3)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (2021.10.8)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (2.10)
    Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow<2.3,>1.10->econml[tf]) (3.1.1)
    Requirement already satisfied: sympy>=1.4 in /usr/local/lib/python3.7/dist-packages (from dowhy->econml[tf]) (1.7.1)
    Requirement already satisfied: pydot>=1.4 in /usr/local/lib/python3.7/dist-packages (from dowhy->econml[tf]) (1.4.2)
    Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from dowhy->econml[tf]) (2.6.3)
    Requirement already satisfied: pyparsing>=2.1.4 in /usr/local/lib/python3.7/dist-packages (from pydot>=1.4->dowhy->econml[tf]) (3.0.6)
    Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.7/dist-packages (from sympy>=1.4->dowhy->econml[tf]) (1.2.1)
    Installing collected packages: tensorflow-estimator, tensorboard, tensorflow, keras
      Attempting uninstall: tensorflow-estimator
        Found existing installation: tensorflow-estimator 2.7.0
        Uninstalling tensorflow-estimator-2.7.0:
          Successfully uninstalled tensorflow-estimator-2.7.0
      Attempting uninstall: tensorboard
        Found existing installation: tensorboard 2.7.0
        Uninstalling tensorboard-2.7.0:
          Successfully uninstalled tensorboard-2.7.0
      Attempting uninstall: tensorflow
        Found existing installation: tensorflow 2.7.0
        Uninstalling tensorflow-2.7.0:
          Successfully uninstalled tensorflow-2.7.0
      Attempting uninstall: keras
        Found existing installation: keras 2.7.0
        Uninstalling keras-2.7.0:
          Successfully uninstalled keras-2.7.0
    Successfully installed keras-2.3.1 tensorboard-2.2.2 tensorflow-2.2.3 tensorflow-estimator-2.2.0

```python
pip install keras==2.0.9
```

    Collecting keras==2.0.9
      Using cached Keras-2.0.9-py2.py3-none-any.whl (299 kB)
    Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras==2.0.9) (3.13)
    Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from keras==2.0.9) (1.15.0)
    Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras==2.0.9) (1.4.1)
    Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras==2.0.9) (1.18.5)
    Installing collected packages: keras
      Attempting uninstall: keras
        Found existing installation: Keras 2.3.1
        Uninstalling Keras-2.3.1:
          Successfully uninstalled Keras-2.3.1
    Successfully installed keras-2.0.9

```python
pip install tensorflow==2.7.0
```

    Collecting tensorflow==2.7.0
      Using cached tensorflow-2.7.0-cp37-cp37m-manylinux2010_x86_64.whl (489.6 MB)
    Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (3.10.0.2)
    Collecting keras<2.8,>=2.7.0rc0
      Using cached keras-2.7.0-py2.py3-none-any.whl (1.3 MB)
    Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.15.0)
    Collecting tensorboard~=2.6
      Using cached tensorboard-2.7.0-py3-none-any.whl (5.8 MB)
    Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.1.2)
    Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (2.10.0)
    Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (0.22.0)
    Requirement already satisfied: wheel<1.0,>=0.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (0.37.0)
    Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (0.2.0)
    Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (12.0.0)
    Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (3.3.0)
    Requirement already satisfied: gast<0.5.0,>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (0.3.3)
    Requirement already satisfied: numpy>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.18.5)
    Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (3.17.3)
    Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.13.3)
    Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.6.3)
    Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (0.12.0)
    Requirement already satisfied: flatbuffers<3.0,>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (2.0)
    Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.42.0)
    Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.7.0) (1.1.0)
    Collecting tensorflow-estimator<2.8,~=2.7.0rc0
      Using cached tensorflow_estimator-2.7.0-py2.py3-none-any.whl (463 kB)
    Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (1.35.0)
    Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (0.4.6)
    Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (57.4.0)
    Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (1.0.1)
    Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (2.23.0)
    Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (0.6.1)
    Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (1.8.0)
    Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow==2.7.0) (3.3.6)
    Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.0) (4.8)
    Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.0) (4.2.4)
    Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.0) (0.2.8)
    Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.7.0) (1.3.0)
    Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.7.0) (4.8.2)
    Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.7.0) (3.6.0)
    Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.0) (0.4.8)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.0) (2.10)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.0) (2021.10.8)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.0) (3.0.4)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.0) (1.24.3)
    Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.7.0) (3.1.1)
    Installing collected packages: tensorflow-estimator, tensorboard, keras, tensorflow
      Attempting uninstall: tensorflow-estimator
        Found existing installation: tensorflow-estimator 2.2.0
        Uninstalling tensorflow-estimator-2.2.0:
          Successfully uninstalled tensorflow-estimator-2.2.0
      Attempting uninstall: tensorboard
        Found existing installation: tensorboard 2.2.2
        Uninstalling tensorboard-2.2.2:
          Successfully uninstalled tensorboard-2.2.2
      Attempting uninstall: keras
        Found existing installation: Keras 2.0.9
        Uninstalling Keras-2.0.9:
          Successfully uninstalled Keras-2.0.9
      Attempting uninstall: tensorflow
        Found existing installation: tensorflow 2.2.3
        Uninstalling tensorflow-2.2.3:
          Successfully uninstalled tensorflow-2.2.3
    Successfully installed keras-2.7.0 tensorboard-2.7.0 tensorflow-2.7.0 tensorflow-estimator-2.7.0

```python
#@title
from econml.iv.nnet import DeepIV
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn import metrics
from sklearn import linear_model
import keras
import tensorflow as tf
from sklearn.neighbors import KNeighborsRegressor
from sklearn.preprocessing import PolynomialFeatures
from mpl_toolkits.mplot3d import Axes3D
import folium
from folium.plugins import HeatMap
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
```

```python
print(keras.__version__)
```

    2.7.0

```python
#@title
evaluation = pd.DataFrame({'Model': [],
                           'Details':[],
                           'Root Mean Squared Error (RMSE)':[],
                           'R-squared (training)':[],
                           'Adjusted R-squared (training)':[],
                           'R-squared (test)':[],
                           'Adjusted R-squared (test)':[],
                           '5-Fold Cross Validation':[]})
```

#  変数説明

id - Unique ID for each home sold

date - Date of the home sale

price - Price of each home sold

bedrooms - Number of bedrooms

bathrooms - Number of bathrooms, where .5 accounts for a room with a toilet but no shower

sqft_living - Square footage of the apartments interior living space

sqft_lot - Square footage of the land space

floors - Number of floors

waterfront - A dummy variable for
whether the apartment was overlooking the waterfront or not

view - An index from 0 to 4 of how good the view of the property was

condition - An index from 1 to 5 on the condition of the apartment,

grade - An index from 1 to 13, where 1-3 falls short of building
construction and design, 7 has an average level of construction and design, and 11-13 have a high quality level of construction and design.

sqft_above - The square footage of the interior housing space that is above ground level

sqft_basement - The square footage of the interior housing space that is below ground level

yr_built - The year the house was initially built

yr_renovated - The year of the house’s last renovation

zipcode - What zipcode area the house is in

lat - Lattitude

long - Longitude

sqft_living15 - The square footage of interior housing living space for the nearest 15 neighbors

sqft_lot15 - The square footage of the land lots of the nearest 15 neighbors

```python
#@title
df = pd.read_csv("kc_house_data.csv")
```

```python
df.head()
```

      id
      date
      price
      bedrooms
      bathrooms
      sqft_living
      sqft_lot
      floors
      waterfront
      view
      condition
      grade
      sqft_above
      sqft_basement
      yr_built
      yr_renovated
      zipcode
      lat
      long
      sqft_living15
      sqft_lot15

      0
      7129300520
      20141013T000000
      221900.0
      3
      1.00
      1180
      5650
      1.0
      0
      0
      3
      7
      1180
      0
      1955
      0
      98178
      47.5112
      -122.257
      1340
      5650

      1
      6414100192
      20141209T000000
      538000.0
      3
      2.25
      2570
      7242
      2.0
      0
      0
      3
      7
      2170
      400
      1951
      1991
      98125
      47.7210
      -122.319
      1690
      7639

      2
      5631500400
      20150225T000000
      180000.0
      2
      1.00
      770
      10000
      1.0
      0
      0
      3
      6
      770
      0
      1933
      0
      98028
      47.7379
      -122.233
      2720
      8062

      3
      2487200875
      20141209T000000
      604000.0
      4
      3.00
      1960
      5000
      1.0
      0
      0
      5
      7
      1050
      910
      1965
      0
      98136
      47.5208
      -122.393
      1360
      5000

      4
      1954400510
      20150218T000000
      510000.0
      3
      2.00
      1680
      8080
      1.0
      0
      0
      3
      8
      1680
      0
      1987
      0
      98074
      47.6168
      -122.045
      1800
      7503

#  決定係数$R^{2}$

説明変数増えると決定係数$R^{2}$も大きくなるので、自由度修正済決定係数を以下の式より計算できる：

$\bar{R^{2}}=R^{2}-\frac{k-1}{n-k}(1-R^{2})$

n、kはそれぞれ説明変数とパラメーターの数である。

```python
#@title
def adjustedR2(r2,n,k):
    return r2-(k-1)/(n-k)*(1-r2)
```

# 変数の相関関係

説明変数の数は少ないので、まずお互いのヒストグラムから、説明変数の分布を書いてみる。

```python
#@title
df1 = df[['price', 'bedrooms', 'bathrooms', 'sqft_living',
    'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade',
    'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'zipcode',
    'lat', 'long', 'sqft_living15', 'sqft_lot15']]
h = df1.hist(bins=25,figsize=(16,16),xlabelsize='10',ylabelsize='10',xrot=-15)
sns.despine(left=True, bottom=True)
[x.title.set_size(12) for x in h.ravel()];
[x.yaxis.tick_left() for x in h.ravel()];
```

非説明変数Priceの分布から見ると、sqft_living、sqft_above、bedroom、bathroomなど変数の分布と似ている。さらに変数の相関関係のヒートマップを描く。

```python
#@title
features = ['price','bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','yr_built','yr_renovated',
            'zipcode','lat','long','sqft_living15','sqft_lot15']

mask = np.zeros_like(df[features].corr(), dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

f, ax = plt.subplots(figsize=(16, 12))
plt.title('Pearson Correlation Matrix',fontsize=25)

sns.heatmap(df[features].corr(),linewidths=0.25,vmax=0.7,square=True,cmap="BuGn", #"BuGn_r" to reverse
            linecolor='w',annot=True,annot_kws={"size":8},mask=mask,cbar_kws={"shrink": .9});
```

説明変数の数が増えると${R}^2$は1に近くなるが、サンプルデータ（traning data）以外の新しデータセットに予測の精度は悪くなる。いわゆるOverfittingの問題も生じる。従って、余計な説明変数を避けた方がいい。あと二つ共相関関係強い説明変数ならモデルの中で存在するととoverfittingの可能性も高くなる。

もう一つ強い相関関係の変数存在する場合に説明変数と誤差相関の時、観測不可の変数が存在するので、計量経済学に内生性問題が生じる。

# データの前処理

前処理したデータは本データより高い精度を取れ、ある特定の結果に辿りやすくなるかもしえない。正規化やdata miningなど前処理方法はよく使われている。

```python
#@title
df_dm=df.copy()
df_dm.describe()
```

      id
      price
      bedrooms
      bathrooms
      sqft_living
      sqft_lot
      floors
      waterfront
      view
      condition
      grade
      sqft_above
      sqft_basement
      yr_built
      yr_renovated
      zipcode
      lat
      long
      sqft_living15
      sqft_lot15

      count
      2.161300e+04
      2.161300e+04
      21613.000000
      21613.000000
      21613.000000
      2.161300e+04
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000
      21613.000000

      mean
      4.580302e+09
      5.401822e+05
      3.370842
      2.114757
      2079.899736
      1.510697e+04
      1.494309
      0.007542
      0.234303
      3.409430
      7.656873
      1788.390691
      291.509045
      1971.005136
      84.402258
      98077.939805
      47.560053
      -122.213896
      1986.552492
      12768.455652

      std
      2.876566e+09
      3.673622e+05
      0.930062
      0.770163
      918.440897
      4.142051e+04
      0.539989
      0.086517
      0.766318
      0.650743
      1.175459
      828.090978
      442.575043
      29.373411
      401.679240
      53.505026
      0.138564
      0.140828
      685.391304
      27304.179631

      min
      1.000102e+06
      7.500000e+04
      0.000000
      0.000000
      290.000000
      5.200000e+02
      1.000000
      0.000000
      0.000000
      1.000000
      1.000000
      290.000000
      0.000000
      1900.000000
      0.000000
      98001.000000
      47.155900
      -122.519000
      399.000000
      651.000000

      25%
      2.123049e+09
      3.219500e+05
      3.000000
      1.750000
      1427.000000
      5.040000e+03
      1.000000
      0.000000
      0.000000
      3.000000
      7.000000
      1190.000000
      0.000000
      1951.000000
      0.000000
      98033.000000
      47.471000
      -122.328000
      1490.000000
      5100.000000

      50%
      3.904930e+09
      4.500000e+05
      3.000000
      2.250000
      1910.000000
      7.618000e+03
      1.500000
      0.000000
      0.000000
      3.000000
      7.000000
      1560.000000
      0.000000
      1975.000000
      0.000000
      98065.000000
      47.571800
      -122.230000
      1840.000000
      7620.000000

      75%
      7.308900e+09
      6.450000e+05
      4.000000
      2.500000
      2550.000000
      1.068800e+04
      2.000000
      0.000000
      0.000000
      4.000000
      8.000000
      2210.000000
      560.000000
      1997.000000
      0.000000
      98118.000000
      47.678000
      -122.125000
      2360.000000
      10083.000000

      max
      9.900000e+09
      7.700000e+06
      33.000000
      8.000000
      13540.000000
      1.651359e+06
      3.500000
      1.000000
      4.000000
      5.000000
      13.000000
      9410.000000
      4820.000000
      2015.000000
      2015.000000
      98199.000000
      47.777600
      -121.315000
      6210.000000
      871200.000000

Binningはデータ前処理の一つ手法として、観測の数が少ないデータからの影響を減らす手法である。今回のデータセットにyr_builtとyr_renovated二つ説明変数にbinningを使う。

```python
#@title
# just take the year from the date column
df_dm['sales_yr']=df_dm['date'].astype(str).str[:4]

# add the age of the buildings when the houses were sold as a new column
df_dm['age']=df_dm['sales_yr'].astype(int)-df_dm['yr_built']
# add the age of the renovation when the houses were sold as a new column
df_dm['age_rnv']=0
df_dm['age_rnv']=df_dm['sales_yr'][df_dm['yr_renovated']!=0].astype(int)-df_dm['yr_renovated'][df_dm['yr_renovated']!=0]
df_dm['age_rnv'][df_dm['age_rnv'].isnull()]=0

# partition the age into bins
bins = [-2,0,5,10,25,50,75,100,100000]
labels = ['<1','1-5','6-10','11-25','26-50','51-75','76-100','>100']
df_dm['age_binned'] = pd.cut(df_dm['age'], bins=bins, labels=labels)
# partition the age_rnv into bins
bins = [-2,0,5,10,25,50,75,100000]
labels = ['<1','1-5','6-10','11-25','26-50','51-75','>75']
df_dm['age_rnv_binned'] = pd.cut(df_dm['age_rnv'], bins=bins, labels=labels)

# histograms for the binned columns
f, axes = plt.subplots(1, 2,figsize=(15,5))
p1=sns.countplot(df_dm['age_binned'],ax=axes[0])
for p in p1.patches:
    height = p.get_height()
    p1.text(p.get_x()+p.get_width()/2,height + 50,height,ha="center")
p2=sns.countplot(df_dm['age_rnv_binned'],ax=axes[1])
sns.despine(left=True, bottom=True)
for p in p2.patches:
    height = p.get_height()
    p2.text(p.get_x()+p.get_width()/2,height + 200,height,ha="center")

axes[0].set(xlabel='Age')
axes[0].yaxis.tick_left()
axes[1].yaxis.set_label_position("right")
axes[1].yaxis.tick_right()
axes[1].set(xlabel='Renovation Age');

# transform the factor values to be able to use in the model
df_dm = pd.get_dummies(df_dm, columns=['age_binned','age_rnv_binned'])
```

# 線型モデル−1

一部前処理したデータを使い、一番簡単の線型モデルから始まる。

$$h_{\theta}(X)=\theta_{0}+\theta_{1}x_{1}+\theta_{2}x_{2}+...+\theta_{n}x_{n}$$

```python
#@title
train_data,test_data = train_test_split(df,train_size = 0.8,random_state=3)
train_data_dm,test_data_dm = train_test_split(df_dm,train_size = 0.8,random_state=3)

features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront','view',
             'grade','age_binned_<1', 'age_binned_1-5', 'age_binned_6-10','age_binned_11-25',
             'age_binned_26-50', 'age_binned_51-75','age_binned_76-100', 'age_binned_>100',
             'zipcode']
complex_model_1 = linear_model.LinearRegression()
complex_model_1.fit(train_data_dm[features],train_data_dm['price'])

print('Intercept: {}'.format(complex_model_1.intercept_))
print('Coefficients: {}'.format(complex_model_1.coef_))

pred = complex_model_1.predict(test_data_dm[features])
rmsecm = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred)),'.3f'))
rtrcm = float(format(complex_model_1.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm = float(format(adjustedR2(complex_model_1.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm = float(format(complex_model_1.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm = float(format(adjustedR2(complex_model_1.score(test_data_dm[features],test_data['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv = float(format(cross_val_score(complex_model_1,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Multiple Regression-1','selected features',rmsecm,rtrcm,artrcm,rtecm,artecm,cv]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

    Intercept: 14914402.737082087
    Coefficients: [-3.74862318e+04  4.83935681e+04  1.71784897e+02 -2.31321046e-01
      1.03548284e+04  5.56414363e+05  4.79049661e+04  1.24241940e+05
     -8.88973485e+04 -1.05897212e+05 -1.04783195e+05 -1.35973525e+05
     -5.37512681e+04  8.41812470e+04  1.84256092e+05  2.20865209e+05
     -1.59875071e+02]

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

# 線型モデル−2

次は説明変数を増やし、同じモデルを使う。

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront','view',
             'grade','age_binned_<1', 'age_binned_1-5', 'age_binned_6-10','age_binned_11-25',
             'age_binned_26-50', 'age_binned_51-75','age_binned_76-100', 'age_binned_>100',
             'zipcode']
complex_model_2 = linear_model.LinearRegression()
complex_model_2.fit(train_data_dm[features],train_data_dm['price'])

print('Intercept: {}'.format(complex_model_2.intercept_))
print('Coefficients: {}'.format(complex_model_2.coef_))

pred = complex_model_2.predict(test_data_dm[features])
rmsecm = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred)),'.3f'))
rtrcm = float(format(complex_model_2.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm = float(format(adjustedR2(complex_model_2.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm = float(format(complex_model_2.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm = float(format(adjustedR2(complex_model_2.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv = float(format(cross_val_score(complex_model_2,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Multiple Regression-2','selected features',rmsecm,rtrcm,artrcm,rtecm,artecm,cv]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

    Intercept: 14914402.737082087
    Coefficients: [-3.74862318e+04  4.83935681e+04  1.71784897e+02 -2.31321046e-01
      1.03548284e+04  5.56414363e+05  4.79049661e+04  1.24241940e+05
     -8.88973485e+04 -1.05897212e+05 -1.04783195e+05 -1.35973525e+05
     -5.37512681e+04  8.41812470e+04  1.84256092e+05  2.20865209e+05
     -1.59875071e+02]

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

# 線型モデル−3

今回は前処理されないデータを使う場合。

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront','view',
            'condition','grade','sqft_above','sqft_basement','yr_built','yr_renovated',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
complex_model_3 = linear_model.LinearRegression()
complex_model_3.fit(train_data[features],train_data['price'])

print('Intercept: {}'.format(complex_model_3.intercept_))
print('Coefficients: {}'.format(complex_model_3.coef_))

pred = complex_model_3.predict(test_data[features])
rmsecm = float(format(np.sqrt(metrics.mean_squared_error(test_data['price'],pred)),'.3f'))
rtrcm = float(format(complex_model_3.score(train_data[features],train_data['price']),'.3f'))
artrcm = float(format(adjustedR2(complex_model_3.score(train_data[features],train_data['price']),train_data.shape[0],len(features)),'.3f'))
rtecm = float(format(complex_model_3.score(test_data[features],test_data['price']),'.3f'))
artecm = float(format(adjustedR2(complex_model_3.score(test_data[features],test_data['price']),test_data.shape[0],len(features)),'.3f'))
cv = float(format(cross_val_score(complex_model_3,df[features],df['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Multiple Regression-3','all features, no preprocessing',rmsecm,rtrcm,artrcm,rtecm,artecm,cv]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

    Intercept: 7561473.2988707395
    Coefficients: [-3.51672008e+04  4.26213118e+04  1.10776801e+02  1.13367200e-01
      6.82764823e+03  5.61916155e+05  5.28811479e+04  2.49198871e+04
      9.58660245e+04  7.02387720e+01  4.05380293e+01 -2.71165435e+03
      2.26743708e+01 -5.80529156e+02  5.98722304e+05 -2.09107231e+05
      2.32792585e+01 -3.75114599e-01]

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      2
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

# 線型モデル−4

最後全部の変数を説明変数として、前処理のデータを使う。

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
complex_model_4 = linear_model.LinearRegression()
complex_model_4.fit(train_data_dm[features],train_data_dm['price'])

print('Intercept: {}'.format(complex_model_4.intercept_))
print('Coefficients: {}'.format(complex_model_4.coef_))

pred = complex_model_4.predict(test_data_dm[features])
rmsecm = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred)),'.3f'))
rtrcm = float(format(complex_model_4.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm = float(format(adjustedR2(complex_model_4.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm = float(format(complex_model_4.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm = float(format(adjustedR2(complex_model_4.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv = float(format(cross_val_score(complex_model_4,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Multiple Regression-4','all features',rmsecm,rtrcm,artrcm,rtecm,artecm,cv]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

    Intercept: 8725167.441148559
    Coefficients: [-3.33833154e+04  3.76917785e+04  1.10788334e+02  1.22619381e-01
     -1.26814291e+04  5.69937691e+05  5.42029509e+04  3.17597396e+04
      9.53241661e+04  7.05755192e+01  4.02128697e+01 -4.47502487e+04
     -5.44967425e+04 -7.11653741e+04 -8.94061644e+04 -5.92815623e+04
      3.71466307e+04  1.22907186e+05  1.59046276e+05 -1.97647293e+04
      9.43944386e+04  8.73267990e+04  4.14586225e+04 -1.50055169e+04
     -1.06929583e+05 -8.14800309e+04 -6.50179892e+02  6.03430775e+05
     -2.10272751e+05  2.42326810e+01 -3.08412010e-01]

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      3
      Multiple Regression-4
      all features
      192003.134
      0.701
      0.700
      0.713
      0.711
      0.698

      2
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

# 正則化

機械学習において、一番生じやすい問題はは過学習(overfitting)と学習不足(underfitting)
である。正則化(Regularization)は過学習と学習不足を抑制するための一つ方法である。

過学習は機械学習モデルは高い分散を持っている。モデルは複雑しすぎて余計な変数や強い相関関係ある変数を使うので、元の訓練データ(training data)に高い精度で予測できるが、新しいデータセットの予測パフォーマンスは低下である。

その一方、学習不足は分散が低いが、バイアスが高い。モデルは簡単すぎて、良い予測結果を得られない。

Ridge回帰モデルとLassoモデルは二つよく使われている正則化モデルである。場合による、どちらか使うにより、機械学習モデルの過学習・学習不足の問題を改善できる。

# Ridge回帰モデル

Ridge回帰(L2 regularization)はSGD(sochastic gradient decent)の最適化過程の中でペナルティをつけ、以下の式を得られる。

$$RSS_{RIDGE} = \sum_{i=1}^{m}(h_{\theta}(x_{i})-y_{i})^{2} + \alpha \sum_{j=1}^{n}\theta^{2}_{j}$$

$\alpha$はペナルティーパラメーターで、正則化の効果に直接影響を与える。

以下のセルで違う$\alpha$の値と正則化してない線型モデルの結果を比較する。

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
complex_model_R = linear_model.Ridge(alpha=1)
complex_model_R.fit(train_data_dm[features],train_data_dm['price'])

pred1 = complex_model_R.predict(test_data_dm[features])
rmsecm1 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred1)),'.3f'))
rtrcm1 = float(format(complex_model_R.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm1 = float(format(adjustedR2(complex_model_R.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm1 = float(format(complex_model_R.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm1 = float(format(adjustedR2(complex_model_R.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv1 = float(format(cross_val_score(complex_model_R,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

complex_model_R = linear_model.Ridge(alpha=100)
complex_model_R.fit(train_data_dm[features],train_data_dm['price'])

pred2 = complex_model_R.predict(test_data_dm[features])
rmsecm2 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred2)),'.3f'))
rtrcm2 = float(format(complex_model_R.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm2 = float(format(adjustedR2(complex_model_R.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm2 = float(format(complex_model_R.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm2 = float(format(adjustedR2(complex_model_R.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv2 = float(format(cross_val_score(complex_model_R,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

complex_model_R = linear_model.Ridge(alpha=1000)
complex_model_R.fit(train_data_dm[features],train_data_dm['price'])

pred3 = complex_model_R.predict(test_data_dm[features])
rmsecm3 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred3)),'.3f'))
rtrcm3 = float(format(complex_model_R.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm3 = float(format(adjustedR2(complex_model_R.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm3 = float(format(complex_model_R.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm3 = float(format(adjustedR2(complex_model_R.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv3 = float(format(cross_val_score(complex_model_R,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Ridge Regression','alpha=1, all features',rmsecm1,rtrcm1,artrcm1,rtecm1,artecm1,cv1]
evaluation.loc[r+1] = ['Ridge Regression','alpha=100, all features',rmsecm2,rtrcm2,artrcm2,rtecm2,artecm2,cv2]
evaluation.loc[r+2] = ['Ridge Regression','alpha=1000, all features',rmsecm3,rtrcm3,artrcm3,rtecm3,artecm3,cv3]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      3
      Multiple Regression-4
      all features
      192003.134
      0.701
      0.700
      0.713
      0.711
      0.698

      4
      Ridge Regression
      alpha=1, all features
      192027.091
      0.701
      0.700
      0.713
      0.711
      0.698

      2
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      5
      Ridge Regression
      alpha=100, all features
      195492.878
      0.694
      0.693
      0.703
      0.701
      0.691

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      6
      Ridge Regression
      alpha=1000, all features
      209743.123
      0.651
      0.651
      0.658
      0.655
      0.648

# Lasso回帰モデル

Lassoモデル(L1 regularization)はRidgeと似ている。ただ、ペナルティーの項は違う。

$$RSS_{LASSO} = \sum_{i=1}^{m}(h_{\theta}(x_{i})-y_{i})^{2} + \alpha \sum_{j=1}^{n}|\theta_{j}|$$

同じように、違うペナルティーパラメーターと正則化なしモデルを比較する。

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
complex_model_L = linear_model.Lasso(alpha=1)
complex_model_L.fit(train_data_dm[features],train_data_dm['price'])

pred1 = complex_model_L.predict(test_data_dm[features])
rmsecm1 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred1)),'.3f'))
rtrcm1 = float(format(complex_model_L.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm1 = float(format(adjustedR2(complex_model_L.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm1 = float(format(complex_model_L.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm1 = float(format(adjustedR2(complex_model_L.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv1 = float(format(cross_val_score(complex_model_L,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

complex_model_L = linear_model.Lasso(alpha=100)
complex_model_L.fit(train_data_dm[features],train_data_dm['price'])

pred2 = complex_model_L.predict(test_data_dm[features])
rmsecm2 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred2)),'.3f'))
rtrcm2 = float(format(complex_model_L.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm2 = float(format(adjustedR2(complex_model_L.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm2 = float(format(complex_model_L.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm2 = float(format(adjustedR2(complex_model_L.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv2 = float(format(cross_val_score(complex_model_L,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

complex_model_L = linear_model.Lasso(alpha=1000)
complex_model_L.fit(train_data_dm[features],train_data_dm['price'])

pred3 = complex_model_L.predict(test_data_dm[features])
rmsecm3 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred3)),'.3f'))
rtrcm3 = float(format(complex_model_L.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrcm3 = float(format(adjustedR2(complex_model_L.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rtecm3 = float(format(complex_model_L.score(test_data_dm[features],test_data_dm['price']),'.3f'))
artecm3 = float(format(adjustedR2(complex_model_L.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv3 = float(format(cross_val_score(complex_model_L,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['Lasso Regression','alpha=1, all features',rmsecm1,rtrcm1,artrcm1,rtecm1,artecm1,cv1]
evaluation.loc[r+1] = ['Lasso Regression','alpha=100, all features',rmsecm2,rtrcm2,artrcm2,rtecm2,artecm2,cv2]
evaluation.loc[r+2] = ['Lasso Regression','alpha=1000, all features',rmsecm3,rtrcm3,artrcm3,rtecm3,artecm3,cv3]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      3
      Multiple Regression-4
      all features
      192003.134
      0.701
      0.700
      0.713
      0.711
      0.698

      4
      Ridge Regression
      alpha=1, all features
      192027.091
      0.701
      0.700
      0.713
      0.711
      0.698

      7
      Lasso Regression
      alpha=1, all features
      192004.507
      0.701
      0.700
      0.713
      0.711
      0.698

      8
      Lasso Regression
      alpha=100, all features
      192183.917
      0.701
      0.700
      0.713
      0.711
      0.698

      2
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      9
      Lasso Regression
      alpha=1000, all features
      193710.209
      0.697
      0.697
      0.708
      0.706
      0.695

      5
      Ridge Regression
      alpha=100, all features
      195492.878
      0.694
      0.693
      0.703
      0.701
      0.691

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      6
      Ridge Regression
      alpha=1000, all features
      209743.123
      0.651
      0.651
      0.658
      0.655
      0.648

# 多項式回帰

多項式回帰は線型モデルより高い次元のモデルを仮説する。以下の式通り表せる。

$$h_{\theta}(X)=\theta_{0}+\theta_{1}x+\theta_{2}x^{2}+...+\theta_{n}x^{n}$$

一方、多項式モデルの次元高いほど、過学習の問題も生じやすくなるので、注意すべきはずだ。

```python
#@title
evaluation_poly = pd.DataFrame({'Model': [],
                                'Details':[],
                                'Root Mean Squared Error (RMSE)':[],
                                'R-squared (training)':[],
                                'Adjusted R-squared (training)':[],
                                'R-squared (test)':[],
                                'Adjusted R-squared (test)':[],
                                '5-Fold Cross Validation':[]})

features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront','view',
             'grade','yr_built','zipcode']
polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df[features])
X_trainpoly = polyfeat.fit_transform(train_data[features])
X_testpoly = polyfeat.fit_transform(test_data[features])
poly = linear_model.LinearRegression().fit(X_trainpoly, train_data['price'])

pred1 = poly.predict(X_testpoly)
rmsepoly1 = float(format(np.sqrt(metrics.mean_squared_error(test_data['price'],pred1)),'.3f'))
rtrpoly1 = float(format(poly.score(X_trainpoly,train_data['price']),'.3f'))
rtepoly1 = float(format(poly.score(X_testpoly,test_data['price']),'.3f'))
cv1 = float(format(cross_val_score(linear_model.LinearRegression(),X_allpoly,df['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=3)
X_allpoly = polyfeat.fit_transform(df[features])
X_trainpoly = polyfeat.fit_transform(train_data[features])
X_testpoly = polyfeat.fit_transform(test_data[features])
poly = linear_model.LinearRegression().fit(X_trainpoly, train_data['price'])

pred2 = poly.predict(X_testpoly)
rmsepoly2 = float(format(np.sqrt(metrics.mean_squared_error(test_data['price'],pred2)),'.3f'))
rtrpoly2 = float(format(poly.score(X_trainpoly,train_data['price']),'.3f'))
rtepoly2 = float(format(poly.score(X_testpoly,test_data['price']),'.3f'))
cv2 = float(format(cross_val_score(linear_model.LinearRegression(),X_allpoly,df['price'],cv=5).mean(),'.3f'))

features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront','view',
            'condition','grade','sqft_above','sqft_basement','yr_built','yr_renovated',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df[features])
X_trainpoly = polyfeat.fit_transform(train_data[features])
X_testpoly = polyfeat.fit_transform(test_data[features])
poly = linear_model.LinearRegression().fit(X_trainpoly, train_data['price'])

pred3 = poly.predict(X_testpoly)
rmsepoly3 = float(format(np.sqrt(metrics.mean_squared_error(test_data['price'],pred3)),'.3f'))
rtrpoly3 = float(format(poly.score(X_trainpoly,train_data['price']),'.3f'))
rtepoly3 = float(format(poly.score(X_testpoly,test_data['price']),'.3f'))
cv3 = float(format(cross_val_score(linear_model.LinearRegression(),X_allpoly,df['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=3)
X_allpoly = polyfeat.fit_transform(df[features])
X_trainpoly = polyfeat.fit_transform(train_data[features])
X_testpoly = polyfeat.fit_transform(test_data[features])
poly = linear_model.LinearRegression().fit(X_trainpoly, train_data['price'])

pred4 = poly.predict(X_testpoly)
rmsepoly4 = float(format(np.sqrt(metrics.mean_squared_error(test_data['price'],pred4)),'.3f'))
rtrpoly4 = float(format(poly.score(X_trainpoly,train_data['price']),'.3f'))
rtepoly4 = float(format(poly.score(X_testpoly,test_data['price']),'.3f'))
cv4 = float(format(cross_val_score(linear_model.LinearRegression(),X_allpoly,df['price'],cv=5).mean(),'.3f'))

features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df_dm[features])
X_trainpoly = polyfeat.fit_transform(train_data_dm[features])
X_testpoly = polyfeat.fit_transform(test_data_dm[features])
poly = linear_model.LinearRegression().fit(X_trainpoly, train_data['price'])

pred5 = poly.predict(X_testpoly)
rmsepoly5 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred5)),'.3f'))
rtrpoly5 = float(format(poly.score(X_trainpoly,train_data_dm['price']),'.3f'))
rtepoly5 = float(format(poly.score(X_testpoly,test_data_dm['price']),'.3f'))
cv5 = float(format(cross_val_score(linear_model.LinearRegression(),X_allpoly,df_dm['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df_dm[features])
X_trainpoly = polyfeat.fit_transform(train_data_dm[features])
X_testpoly = polyfeat.fit_transform(test_data_dm[features])
poly = linear_model.Ridge(alpha=1).fit(X_trainpoly, train_data['price'])

pred6 = poly.predict(X_testpoly)
rmsepoly6 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred6)),'.3f'))
rtrpoly6 = float(format(poly.score(X_trainpoly,train_data_dm['price']),'.3f'))
rtepoly6 = float(format(poly.score(X_testpoly,test_data_dm['price']),'.3f'))
cv6 = float(format(cross_val_score(linear_model.Ridge(alpha=1),X_allpoly,df_dm['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df_dm[features])
X_trainpoly = polyfeat.fit_transform(train_data_dm[features])
X_testpoly = polyfeat.fit_transform(test_data_dm[features])
poly = linear_model.Ridge(alpha=50000).fit(X_trainpoly, train_data['price'])

pred7 = poly.predict(X_testpoly)
rmsepoly7 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred7)),'.3f'))
rtrpoly7 = float(format(poly.score(X_trainpoly,train_data_dm['price']),'.3f'))
rtepoly7 = float(format(poly.score(X_testpoly,test_data_dm['price']),'.3f'))
cv7 = float(format(cross_val_score(linear_model.Ridge(alpha=50000),X_allpoly,df_dm['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df_dm[features])
X_trainpoly = polyfeat.fit_transform(train_data_dm[features])
X_testpoly = polyfeat.fit_transform(test_data_dm[features])
poly = linear_model.Lasso(alpha=1).fit(X_trainpoly, train_data['price'])

pred8 = poly.predict(X_testpoly)
rmsepoly8 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred8)),'.3f'))
rtrpoly8 = float(format(poly.score(X_trainpoly,train_data_dm['price']),'.3f'))
rtepoly8 = float(format(poly.score(X_testpoly,test_data_dm['price']),'.3f'))
cv8 = float(format(cross_val_score(linear_model.Lasso(alpha=1),X_allpoly,df_dm['price'],cv=5).mean(),'.3f'))

polyfeat = PolynomialFeatures(degree=2)
X_allpoly = polyfeat.fit_transform(df_dm[features])
X_trainpoly = polyfeat.fit_transform(train_data_dm[features])
X_testpoly = polyfeat.fit_transform(test_data_dm[features])
poly = linear_model.Lasso(alpha=50000).fit(X_trainpoly, train_data['price'])

pred9 = poly.predict(X_testpoly)
rmsepoly9 = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred9)),'.3f'))
rtrpoly9 = float(format(poly.score(X_trainpoly,train_data_dm['price']),'.3f'))
rtepoly9 = float(format(poly.score(X_testpoly,test_data_dm['price']),'.3f'))
cv9 = float(format(cross_val_score(linear_model.Lasso(alpha=50000),X_allpoly,df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation_poly.shape[0]
evaluation_poly.loc[r] = ['Polynomial Regression','degree=2, selected features, no preprocessing',rmsepoly1,rtrpoly1,'-',rtepoly1,'-',cv1]
evaluation_poly.loc[r+1] = ['Polynomial Regression','degree=3, selected features, no preprocessing',rmsepoly2,rtrpoly2,'-',rtepoly2,'-',cv2]
evaluation_poly.loc[r+2] = ['Polynomial Regression','degree=2, all features, no preprocessing',rmsepoly3,rtrpoly3,'-',rtepoly3,'-',cv3]
evaluation_poly.loc[r+3] = ['Polynomial Regression','degree=3, all features, no preprocessing',rmsepoly4,rtrpoly4,'-',rtepoly4,'-',cv4]
evaluation_poly.loc[r+4] = ['Polynomial Ridge Regression','alpha=1, degree=2, all features',rmsepoly6,rtrpoly6,'-',rtepoly6,'-',cv6]
evaluation_poly.loc[r+5] = ['Polynomial Ridge Regression','alpha=50000, degree=2, all features',rmsepoly7,rtrpoly7,'-',rtepoly7,'-',cv7]
evaluation_poly.loc[r+6] = ['Polynomial Lasso Regression','alpha=1, degree=2, all features',rmsepoly8,rtrpoly8,'-',rtepoly8,'-',cv8]
evaluation_poly.loc[r+7] = ['Polynomial Lasso Regression','alpha=50000, degree=2, all features',rmsepoly9,rtrpoly9,'-',rtepoly9,'-',cv9]
evaluation_poly_temp = evaluation_poly[['Model','Details','Root Mean Squared Error (RMSE)','R-squared (training)','R-squared (test)','5-Fold Cross Validation']]
evaluation_poly_temp.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      R-squared (test)
      5-Fold Cross Validation

      2
      Polynomial Regression
      degree=2, all features, no preprocessing
      151270.666
      0.830
      0.822
      0.813

      5
      Polynomial Ridge Regression
      alpha=50000, degree=2, all features
      159953.227
      0.810
      0.801
      0.791

      7
      Polynomial Lasso Regression
      alpha=50000, degree=2, all features
      166104.824
      0.797
      0.785
      0.779

      6
      Polynomial Lasso Regression
      alpha=1, degree=2, all features
      166274.577
      0.807
      0.785
      0.778

      0
      Polynomial Regression
      degree=2, selected features, no preprocessing
      191155.470
      0.730
      0.716
      0.714

      1
      Polynomial Regression
      degree=3, selected features, no preprocessing
      188927.434
      0.749
      0.722
      0.541

      3
      Polynomial Regression
      degree=3, all features, no preprocessing
      188654.029
      0.874
      0.723
      -0.923

      4
      Polynomial Ridge Regression
      alpha=1, degree=2, all features
      150247.266
      0.838
      0.824
      -3203.429

# k-NN

k最近傍法（k-Nearest Neighbor: kNN）とは，教師あり学習の一手法であり，クラス未知のサンプルのクラスラベルを近傍サンプルk
個による多数決で決めようというシンプルな手法である。kは自由に決めることができるハイパーパラメータです．

```python
#@title
features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
knnreg = KNeighborsRegressor(n_neighbors=15)
knnreg.fit(train_data_dm[features],train_data_dm['price'])
pred = knnreg.predict(test_data_dm[features])
y_test = np.array(test_data['price'], dtype=pd.Series)

rmseknn1 = float(format(np.sqrt(metrics.mean_squared_error(y_test,pred)),'.3f'))
rtrknn1 = float(format(knnreg.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrknn1 = float(format(adjustedR2(knnreg.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rteknn1 = float(format(knnreg.score(test_data_dm[features],test_data_dm['price']),'.3f'))
arteknn1 = float(format(adjustedR2(knnreg.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv1 = float(format(cross_val_score(knnreg,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

knnreg = KNeighborsRegressor(n_neighbors=25)
knnreg.fit(train_data_dm[features],train_data_dm['price'])
pred = knnreg.predict(test_data_dm[features])

rmseknn2 = float(format(np.sqrt(metrics.mean_squared_error(y_test,pred)),'.3f'))
rtrknn2 = float(format(knnreg.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrknn2 = float(format(adjustedR2(knnreg.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rteknn2 = float(format(knnreg.score(test_data_dm[features],test_data_dm['price']),'.3f'))
arteknn2 = float(format(adjustedR2(knnreg.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv2 = float(format(cross_val_score(knnreg,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

knnreg = KNeighborsRegressor(n_neighbors=27)
knnreg.fit(train_data_dm[features],train_data_dm['price'])
pred = knnreg.predict(test_data_dm[features])

rmseknn3 = float(format(np.sqrt(metrics.mean_squared_error(y_test,pred)),'.3f'))
rtrknn3 = float(format(knnreg.score(train_data_dm[features],train_data_dm['price']),'.3f'))
artrknn3 = float(format(adjustedR2(knnreg.score(train_data_dm[features],train_data_dm['price']),train_data_dm.shape[0],len(features)),'.3f'))
rteknn3 = float(format(knnreg.score(test_data_dm[features],test_data_dm['price']),'.3f'))
arteknn3 = float(format(adjustedR2(knnreg.score(test_data_dm[features],test_data_dm['price']),test_data_dm.shape[0],len(features)),'.3f'))
cv3 = float(format(cross_val_score(knnreg,df_dm[features],df_dm['price'],cv=5).mean(),'.3f'))

r = evaluation.shape[0]
evaluation.loc[r] = ['KNN Regression','k=15, all features',rmseknn1,rtrknn1,artrknn1,rteknn1,arteknn1,cv1]
evaluation.loc[r+1] = ['KNN Regression','k=25, all features',rmseknn2,rtrknn2,artrknn2,rteknn2,arteknn2,cv2]
evaluation.loc[r+2] = ['KNN Regression','k=27, all features',rmseknn3,rtrknn3,artrknn3,rteknn3,arteknn3,cv3]
evaluation.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      3
      Multiple Regression-4
      all features
      192003.134
      0.701
      0.700
      0.713
      0.711
      0.698

      4
      Ridge Regression
      alpha=1, all features
      192027.091
      0.701
      0.700
      0.713
      0.711
      0.698

      7
      Lasso Regression
      alpha=1, all features
      192004.507
      0.701
      0.700
      0.713
      0.711
      0.698

      8
      Lasso Regression
      alpha=100, all features
      192183.917
      0.701
      0.700
      0.713
      0.711
      0.698

      2
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      9
      Lasso Regression
      alpha=1000, all features
      193710.209
      0.697
      0.697
      0.708
      0.706
      0.695

      5
      Ridge Regression
      alpha=100, all features
      195492.878
      0.694
      0.693
      0.703
      0.701
      0.691

      0
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      1
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      6
      Ridge Regression
      alpha=1000, all features
      209743.123
      0.651
      0.651
      0.658
      0.655
      0.648

      10
      KNN Regression
      k=15, all features
      242951.663
      0.562
      0.561
      0.541
      0.537
      0.496

      11
      KNN Regression
      k=25, all features
      247151.425
      0.529
      0.528
      0.525
      0.521
      0.487

      12
      KNN Regression
      k=27, all features
      247534.500
      0.523
      0.522
      0.523
      0.520
      0.486

# Deep IV

Deep IVは二段階の操作変数法(IV)モデルに深層神経ネットワーク(deep neural networks)を使っている。説明変数X以外に、操作変数Z(Instruments)と操作変数直接影響する変数T(treatments)がある。Zは非説明変数Yに影響する方法はXに通うのみだ。二段階は以下の通り行う。

1.一段階に特定のZとXで、Tの分布を混合のニューラルネットワークによって推定する。。
2.二段階に一段階のTとXによってYを推定・予測する。

以上の二つ推定はニューラルネットワークを使って行う。

[参考リンク](https://econml.azurewebsites.net/spec/estimation/deepiv.html)

今回の相関関係強い変数sqft_aboveとsqft_livingそれぞれ操作変数とトリートメント変数として、sqft_above(地上面積)はsqft_living(生活面積)を通ってpriceに影響する。

```python
#@title
Y = df["price"]
Y.shape
```

```python
#@title
features = ['bedrooms','bathrooms','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_basement','yr_built','yr_renovated',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
```

```python
X = df[features]
X.shape
```

```python
df["sqft_treatment"] = df["sqft_living"]
T = df["sqft_treatment"]
T.shape
```

```python
#house_df["room"] = house_df["bedrooms"] + house_df["bathrooms"]
Z = df["sqft_above"]
Z.shape
```

```python
#@title
# Show the marginal distribution of t
plt.hist(T)
plt.xlabel("T")
plt.show()
```

```python
plt.scatter(Z,T)
```

```python
plt.scatter(T,Y)
```

```python
#@title
X_train, X_test, Y_train, Y_test, T_train, T_test, Z_train, Z_test = train_test_split(X, Y, T, Z, train_size = 0.8,random_state=3)

#define simple treatment and response models using the Keras Sequential model built up of a series of layers.
treatment_model = keras.Sequential([keras.layers.Dense(128, activation='relu', input_shape=(17,)),
                                    keras.layers.Dropout(0.17),
                                    keras.layers.Dense(64, activation='relu'),
                                    keras.layers.Dropout(0.17),
                                    keras.layers.Dense(32, activation='relu'),
                                    keras.layers.Dropout(0.17)])

response_model = keras.Sequential([keras.layers.Dense(128, activation='relu', input_shape=(17,)),
                                   keras.layers.Dropout(0.17),
                                   keras.layers.Dense(64, activation='relu'),
                                   keras.layers.Dropout(0.17),
                                   keras.layers.Dense(32, activation='relu'),
                                   keras.layers.Dropout(0.17),
                                   keras.layers.Dense(1)])

#instantiate the DeepIV class
#Define the response model outside of the lambda passed into constructor
keras_fit_options = { "epochs": 30,
                      "validation_split": 0.1,
                      "callbacks": [tf.keras.callbacks.EarlyStopping(patience=2, restore_best_weights=True)]}

deepIvEst = DeepIV(n_components = 10, # number of gaussians in our mixture density network
                   m = lambda z, x : treatment_model(keras.layers.concatenate([z,x])), # treatment model
                   h = lambda t, x : response_model(keras.layers.concatenate([t,x])),  # response model
                   n_samples = 1, # number of samples to use to estimate the response
                   use_upper_bound_loss = False, # whether to use an approximation to the true loss
                   n_gradient_samples = 16, # number of samples to use in second estimate of the response (to make loss estimate unbiased)
                   optimizer='adam', # Keras optimizer to use for training - see https://keras.io/optimizers/
                   first_stage_options = keras_fit_options, # options for training treatment model
                   second_stage_options = keras_fit_options) # options for training response model

#fit deepiv model to the data
deepIvEst.fit(Y=Y_train,T=T_train,X=X_train,Z=Z_train)

Y_pred = deepIvEst.predict(T_test, X_test)

```

    Epoch 1/30
    487/487 [==============================] - 3s 3ms/step - loss: nan - val_loss: nan
    Epoch 2/30
    487/487 [==============================] - 1s 3ms/step - loss: nan - val_loss: nan
    Epoch 1/30
    487/487 [==============================] - 23s 23ms/step - loss: nan - val_loss: nan
    Epoch 2/30
    487/487 [==============================] - 8s 16ms/step - loss: nan - val_loss: nan
    WARNING:tensorflow:
    The following Variables were used a Lambda layer's call (lambda_23), but
    are not present in its tracked objects:
      <tf.Variable 'dense_3/kernel:0' shape=(17, 128) dtype=float32>
      <tf.Variable 'dense_3/bias:0' shape=(128,) dtype=float32>
      <tf.Variable 'dense_4/kernel:0' shape=(128, 64) dtype=float32>
      <tf.Variable 'dense_4/bias:0' shape=(64,) dtype=float32>
      <tf.Variable 'dense_5/kernel:0' shape=(64, 32) dtype=float32>
      <tf.Variable 'dense_5/bias:0' shape=(32,) dtype=float32>
      <tf.Variable 'dense_6/kernel:0' shape=(32, 1) dtype=float32>
      <tf.Variable 'dense_6/bias:0' shape=(1,) dtype=float32>
    It is possible that this is intended behavior, but it is more likely
    an omission. This is a strong indication that this layer should be
    formulated as a subclassed Layer rather than a Lambda layer.

```python
#@title
evaluation_deepiv = pd.DataFrame({'Model': [],
                                'Details':[],
                                'Root Mean Squared Error (RMSE)':[],
                                'R-squared (training)':[],
                                'Adjusted R-squared (training)':[],
                                'R-squared (test)':[],
                                'Adjusted R-squared (test)':[],
                                '5-Fold Cross Validation':[]})

features = ['bedrooms','bathrooms','sqft_living','sqft_lot','floors','waterfront',
            'view','condition','grade','sqft_above','sqft_basement','age_binned_<1',
            'age_binned_1-5', 'age_binned_6-10','age_binned_11-25', 'age_binned_26-50',
            'age_binned_51-75','age_binned_76-100', 'age_binned_>100','age_rnv_binned_<1',
            'age_rnv_binned_1-5', 'age_rnv_binned_6-10', 'age_rnv_binned_11-25',
            'age_rnv_binned_26-50', 'age_rnv_binned_51-75', 'age_rnv_binned_>75',
            'zipcode','lat','long','sqft_living15','sqft_lot15']
deepivfeat = PolynomialFeatures(degree=2)
X_alldeepiv = deepivfeat.fit_transform(df_dm[features])
X_traindeepiv = deepivfeat.fit_transform(train_data_dm[features])
X_testdeepiv = deepivfeat.fit_transform(test_data_dm[features])
deepiv = linear_model.LinearRegression().fit(X_traindeepiv, train_data['price'])

pred = deepiv.predict(X_testdeepiv)
rmsedeepiv = float(format(np.sqrt(metrics.mean_squared_error(test_data_dm['price'],pred5)),'.3f'))
rtrdeepiv = float(format(deepiv.score(X_traindeepiv,train_data_dm['price']),'.3f'))
rtedeepiv = float(format(deepiv.score(X_testdeepiv,test_data_dm['price']),'.3f'))
cv5 = float(format(cross_val_score(linear_model.LinearRegression(),X_alldeepiv,df_dm['price'],cv=5).mean(),'.3f'))

artrcm = float(format(adjustedR2(deepiv.score(X_traindeepiv,train_data['price']),X_traindeepiv.shape[0],len(features)),'.3f'))
artecm = float(format(adjustedR2(deepiv.score(X_testdeepiv,test_data['price']),X_testdeepiv.shape[0],len(features)),'.3f'))

# deepivfeat = PolynomialFeatures(degree=2)
# X_alldeepiv = deepivfeat.fit_transform(df_dm[features])
# X_traindeepiv = deepivfeat.fit_transform(train_data_dm[features])
# X_testdeepiv = deepivfeat.fit_transform(test_data_dm[features])
# deepiv = linear_model.Ridge(alpha=1).fit(X_traindeepiv, train_data['price'])

r = evaluation_deepiv.shape[0]
evaluation_deepiv.loc[r] = ['Deep IV','all features',rmsedeepiv,rtrdeepiv,artrcm,rtedeepiv,artecm,cv1]
evaluation_deepiv_temp = evaluation_deepiv[['Model','Details','Root Mean Squared Error (RMSE)','R-squared (training)','Adjusted R-squared (training)','R-squared (test)','Adjusted R-squared (test)','5-Fold Cross Validation']]
evaluation_deepiv_temp.sort_values(by = '5-Fold Cross Validation', ascending=False)
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      0
      Deep IV
      all features
      151671.998
      0.84
      0.84
      0.821
      0.82
      0.496

# 結果比較

以下の結果により、DNNは一般の線型モデルよりいいパフォーマンスを得られる。高い次元において、多項式回帰の方が高い${R}^2$を得られるが、cross-validationはマイナスになる可能性が高くなるので、Deep Neural Networkの方が複雑のモデル構造できると考えている。

```python
#@title
evaluation_temp=evaluation.append(evaluation_poly)
evaluation_temp=evaluation.append(evaluation_deepiv)
evaluation_temp1=evaluation_temp.sort_values(by = '5-Fold Cross Validation', ascending=False)
evaluation_temp2=evaluation_temp1.reset_index()
evaluation_f=evaluation_temp2.iloc[:,1:]
evaluation_f
```

      Model
      Details
      Root Mean Squared Error (RMSE)
      R-squared (training)
      Adjusted R-squared (training)
      R-squared (test)
      Adjusted R-squared (test)
      5-Fold Cross Validation

      0
      Multiple Regression-4
      all features
      192003.134
      0.701
      0.700
      0.713
      0.711
      0.698

      1
      Ridge Regression
      alpha=1, all features
      192027.091
      0.701
      0.700
      0.713
      0.711
      0.698

      2
      Lasso Regression
      alpha=1, all features
      192004.507
      0.701
      0.700
      0.713
      0.711
      0.698

      3
      Lasso Regression
      alpha=100, all features
      192183.917
      0.701
      0.700
      0.713
      0.711
      0.698

      4
      Multiple Regression-3
      all features, no preprocessing
      193817.972
      0.698
      0.697
      0.708
      0.707
      0.695

      5
      Lasso Regression
      alpha=1000, all features
      193710.209
      0.697
      0.697
      0.708
      0.706
      0.695

      6
      Ridge Regression
      alpha=100, all features
      195492.878
      0.694
      0.693
      0.703
      0.701
      0.691

      7
      Multiple Regression-1
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      8
      Multiple Regression-2
      selected features
      209832.557
      0.652
      0.652
      0.657
      0.656
      0.648

      9
      Ridge Regression
      alpha=1000, all features
      209743.123
      0.651
      0.651
      0.658
      0.655
      0.648

      10
      KNN Regression
      k=15, all features
      242951.663
      0.562
      0.561
      0.541
      0.537
      0.496

      11
      Deep IV
      all features
      151671.998
      0.840
      0.840
      0.821
      0.820
      0.496

      12
      KNN Regression
      k=25, all features
      247151.425
      0.529
      0.528
      0.525
      0.521
      0.487

      13
      KNN Regression
      k=27, all features
      247534.500
      0.523
      0.522
      0.523
      0.520
      0.486

```python

```


```python
#we may need some code in the ../python directory and/or matplotlib styles
import sys
import os
sys.path.append('../python/')

#set up matplotlib
os.environ['MPLCONFIGDIR'] = '../mplstyles'
print(os.environ['MPLCONFIGDIR'])
import matplotlib as mpl
from matplotlib import pyplot as plt
#got smarter about the mpl config: see mplstyles/ directory
plt.style.use('standard')
print(mpl.__version__)
print(mpl.get_configdir())

#fonts
# Set the font dictionaries (for plot title and axis titles)
title_font = {'fontname':'Arial', 'size':'16', 'color':'black', 'weight':'normal',
              'verticalalignment':'bottom'} # Bottom vertical alignment for more space
axis_font = {'fontname':'Arial', 'size':'32'}
legend_font = {'fontname':'Arial', 'size':'22'}

#fonts global settings
mpl.rc('font',family=legend_font['fontname'])

#set up numpy
import numpy as np
```

    ../mplstyles
    3.0.3
    /home/phys/villaa/analysis/misc/nrFano_Constraint/mplstyles

# Summary

In the notebook `stat_uncertainty_C.ipynb` we extracted the fit uncertainties for both the EDELWEISS measured ionization yield widths and the widths that result from _only_ the inclusion of multiple-scatter events in the nuclear recoil band.

Since the EDELWEISS measured widths are still significantly larger than those for the multiple-scatter event inclusion, it implies that there is another factor that serves to widen the ionization yield distribution at a given measured recoil energy. We assume that factor is the effective nuclear recoil Fano factor.

# Extracting the Effective Fano Factor

To extract the effective Fano factor for the nuclear recoils we need to come up with a parameter C$_F$ which is a function of recoil energy and is a corrected version of the measured "widening" parameter C from the EDELWEISS data and the widening parameter C$^{\prime}$ from the effect of multiple-scattering.

The corrected parameter C$_F$ is assumed to be due to the effective Fano factor for nuclear recoils and is given by:

\begin{equation}
C_F = \sqrt{C^2 - C^{\prime 2}}.
\end{equation}

This parameter can be used to extract the effective Fano factor at a given recoil energy by applying our $\tilde{E}_r$-Q plane model (from `QEr_2D_joint.ipynb`), with an arbitrary Fano factor, until the correct ionization yield (Q) width is obtained (see `Qwidth_confirm.ipynb`). Mathematically this corresponds to adjusting F$_n$ until the following equality is satisfied:

\begin{equation}
\tilde{\sigma}_{Q}^0(E_r;F_n) = \sqrt{\left(\tilde{\sigma}_{Q}^0(E_r)\right)^2 + C_F^2}.
\end{equation}

## Uncertainties on F$_n$

Since both C and C$^{\prime}$ have uncertainty it is necessary to propagate that uncertainty to F$_n$. If we call the uncertainty (1$\sigma$) on C $\sigma$ and on C$^{\prime}$ $\sigma^{\prime}$, then the uncertainty on C$_F$ is given by:

\begin{equation}
\sigma_{C_F} = \frac{1}{\sqrt{C^2 - C^{\prime 2}}} \sqrt{C^2 \sigma^2 + C^{\prime 2} \sigma^{\prime 2}}.
\end{equation}

These uncertainties are propagated to the extracted F$_n$ by solving the following equality for F$^+_n$ and F$^-_n$ which represent the corresponding upper and lower boundaries on F$_n$.

\begin{equation}
\begin{aligned}
\tilde{\sigma}_{Q}^0(E_r;F^+_n) &= \sqrt{\left(\tilde{\sigma}_{Q}^0(E_r)\right)^2 + \left(C_F + \sigma_{C_F}\right)^2} \\
\tilde{\sigma}_{Q}^0(E_r;F^-_n) &= \sqrt{\left(\tilde{\sigma}_{Q}^0(E_r)\right)^2 + \left(C_F - \sigma_{C_F}\right)^2}
\end{aligned}
\end{equation}

```python
import fano_calc as fc

(Er,F,Fup,Fdn) = fc.RWCalcFMCMC('data/mcmc_fano.h5')
```

    GGA3/4.0/5.556E-02/0.0381/
    True

```python
#set up a plot

fig,axes = plt.subplots(1,1,figsize=(9.0,8.0),sharex=True)
ax1 = axes

xmax=10

#ax1.errorbar(ddata_e,ddata_fluct_F,yerr=[ddata_fluct_F_err,ddata_fluct_F_err], marker='o', markersize=8, \
#             ecolor='k',color='k', linestyle='none', label='Dougherty eff. F', linewidth=2)

#ax1.plot (X, diff, 'm-', label='Thomas-Fermi (newgrad)')
#ax1.plot (Esi(epr), 100*np.sqrt(f_Omega2_eta2(epr))*ylindv(1000*Esi(epr)), 'g-', label='$\Omega/\epsilon$ (NAC III approx. D)')
ax1.plot (Er, F, 'k-', label='extracted Ge eff. Fano')
ax1.plot (Er, Fup, 'b', label='')
ax1.plot (Er, Fdn, 'b', label='')

blue = '#118DFA'
ax1.fill_between(Er,Fdn,Fup,facecolor=blue,alpha=0.5,label='1$\sigma$ statistical region')

ax1.set_yscale('linear')
ax1.set_xscale('linear')
ax1.set_xlim(10, 200)
ax1.set_ylim(6,300)
ax1.set_xlabel('recoil energy ($E_r$) [keV]',**axis_font)
ax1.set_ylabel('effective Fano factor',**axis_font)
#ax1.grid(True)
#ax1.xaxis.grid(True,which='minor',linestyle='--')
ax1.legend(loc=4,prop={'size':22})

for axis in ['top','bottom','left','right']:
  ax1.spines[axis].set_linewidth(2)

plt.tight_layout()
#plt.savefig('figures/figure.png')
plt.show()
```

```python
#set up a plot

fig,axes = plt.subplots(1,1,figsize=(9.0,8.0),sharex=True)
ax1 = axes

xmax=10

#ax1.errorbar(ddata_e,ddata_fluct_F,yerr=[ddata_fluct_F_err,ddata_fluct_F_err], marker='o', markersize=8, \
#             ecolor='k',color='k', linestyle='none', label='Dougherty eff. F', linewidth=2)

#ax1.plot (X, diff, 'm-', label='Thomas-Fermi (newgrad)')
#ax1.plot (Esi(epr), 100*np.sqrt(f_Omega2_eta2(epr))*ylindv(1000*Esi(epr)), 'g-', label='$\Omega/\epsilon$ (NAC III approx. D)')
ax1.plot (Er, F, 'k-', label='extracted Ge eff. Fano')
ax1.plot (Er, Fup, 'b', label='')
ax1.plot (Er, Fdn, 'b', label='')

blue = '#118DFA'
ax1.fill_between(Er,Fdn,Fup,facecolor=blue,alpha=0.5,label='1$\sigma$ statistical region')

ax1.set_yscale('linear')
ax1.set_xscale('linear')
ax1.set_xlim(7, 20)
ax1.set_ylim(0,20)
ax1.set_xlabel('recoil energy ($E_r$) [keV]',**axis_font)
ax1.set_ylabel('effective Fano factor',**axis_font)
#ax1.grid(True)
#ax1.xaxis.grid(True,which='minor',linestyle='--')
ax1.legend(loc=4,prop={'size':22})

for axis in ['top','bottom','left','right']:
  ax1.spines[axis].set_linewidth(2)

plt.tight_layout()
#plt.savefig('figures/figure.png')
plt.show()
```

```python

```


# The gradient regularized asymptotic correction.
## I. The ionization potenital issue

Often with density functional theory one would like to do something interesting with the resulting wavefunction; however a number of issues exsist with convention DFT computations.

Let us first compute Ionization Potential at several different levels of the theory and then compare the results to the NIST database (http://physics.nist.gov).

```python
import psi4
import numpy as np
import ks_helper as ks
%matplotlib inline
import matplotlib.pyplot as plt

from pkg_resources import parse_version
if parse_version(psi4.__version__) >= parse_version('1.3a1'):
    build_superfunctional = psi4.driver.dft.build_superfunctional
else:
    build_superfunctional = psi4.driver.dft_funcs.build_superfunctional

mol = psi4.geometry("""
Ne
symmetry c1
""")
options = {'BASIS':     'aug-cc-pVDZ',
           'NAT_ORBS':  True,
           'NUM_ROOTS': 2,
           'QC_MODULE': 'DETCI'}
psi4.set_options(options)

scf_e, scf_wfn = psi4.energy("SCF", return_wfn=True)
scf_e, dft_wfn = psi4.energy("PBE0", return_wfn=True)
scf_e, ci_wfn =  psi4.energy("CISD", return_wfn=True)
```

Now that we have the Wavefunctions we can then compute the ionization potenital. For SCF theories this is the the energy of the HOMO while for more advanced theories, it is the differente between the ground and excited states. We can obtain the IP for each level of theory in the following way:

```python
scf_ip = -1 * scf_wfn.epsilon_a().np[scf_wfn.nalpha()-1] * psi4.constants.hartree2ev
dft_ip = -1 * dft_wfn.epsilon_a().np[dft_wfn.nalpha()-1] * psi4.constants.hartree2ev
ci_ip = -1 * (psi4.core.variable("CI ROOT 0 TOTAL ENERGY") - psi4.core.variable("CI ROOT 1 TOTAL ENERGY")) * psi4.constants.hartree2ev

print("Neon Ionization Potential (eV)")
print("SCF     %10.4f" % scf_ip)
print("PBE0    %10.4f" % dft_ip)
print("CISD    %10.4f" % ci_ip)
print("NIST    %10.4f" % 21.5645)
```

    Neon Ionization Potential (eV)
    SCF        23.2125
    PBE0       16.0337
    CISD       22.2776
    NIST       21.5645

## I. Density shapes
If we think carefully about the ionization potenital and what this really means it can be summed up susinctly as "How hard is it for the electron to leave the system" or "How deep is our potenital well"? A easy way to view the shape of the potenital for a given method is to inspect the density as a function of spatial extent. A weaker potential would mean that the density extends further than it should and our IP would be too small. While a stronger potenital would mean that the density is more concentrated and the IP is too large. As we only have a single atom we can project our density into grid space and then sort the results as a function of distance from the nucleous.

```python
def one_electron_radial(matrices, Vpot):
    """
    For every matrix in matrices, computes the radial density using the incoming Vpot object.
    """

    results = [[] for x in matrices]
    Rlist = []

    points_func = Vpot.properties()[0]
    superfunc = Vpot.functional()

    xc_e = 0.0
    vv10_e = 0.0

    # First loop over the outer set of blocks
    for block in range(Vpot.nblocks()):

        # Obtain general grid information
        grid = Vpot.get_block(block)
        x = np.array(grid.x())
        y = np.array(grid.y())
        z = np.array(grid.z())
        npoints = z.shape[0]

        # Get radial information
        R = ((x ** 2) + (y ** 2) + (z ** 2)) ** 0.5
        Rlist.append(R)

        # Loop over and compute the density for each matrix
        for num, mat in enumerate(matrices):
            points_func.set_pointers(mat)
            points_func.compute_points(grid)
            results[num].append(np.array(points_func.point_values()["RHO_A"])[:npoints])

    # Stack R
    Rlist = np.hstack(Rlist)

    # Get the R unique and sort
    unique, uidx = np.unique(Rlist, return_index=True)
    Rlist = Rlist[uidx]

    # Numpy is magic, it would be worthwhile to consider excatly what this does
    results = [np.hstack(x)[uidx] for x in results]

    return (Rlist, results)

# Compute the radial densities
sup = build_superfunctional("SVWN", True)[0]
Vpot = psi4.core.VBase.build(scf_wfn.basisset(), sup, "RV")
Vpot.initialize()

R, data = one_electron_radial([scf_wfn.Da(), dft_wfn.Da(), ci_wfn.Da()], Vpot)
```

Now that we have the radial densities, we build  a function that will plot the density as a ratio between
the method a benchmark. Here we will select the CISD densities as the benchmark and plot the SCF and DFT quanties against them.

```python
def plot_scatter_ratio(dictionary, bench, x_values):
    fig1 = plt.figure()
    ax1 = fig1.add_subplot(111)

    for k, v in dictionary.items():
        ax1.plot(x_values, v / bench, label=k)

    ax1.legend(loc='upper left')
    ax1.set_xlim(0, 7)
    ax1.set_xlabel('R')
    ax1.set_ylabel('rho(r) / rho(r)_bench')

    plt.plot([0, 7], [1.0, 1.0], 'k-')
```

Plotting this data we note that the SCF density falls off too rapidly, meaning that potential is too deep while the DFT density falls off far too slowly indicating the potenital is too shallow. For example many theories examine the overlap between densities. From this chart we can surmise issues that would arise with both DFT and HF theories.

```python
data_dict = {"SCF": data[0], "DFT": data[1]}
plot_scatter_ratio(data_dict, data[2], R)
```

## 2. The GRAC Correction
The question then arises, what if we could correct the DFT potential asymptotic so that it recovers the correct density and therefore the correction ionization potenital. Effectively we want to accomplish:

$$v_{\rm{xc}}({\bf r}) = -\frac{1}{r} + v_{\rm{xc}}(\infty)$$

However, for correct asympotics this would require the exact $v_{\rm{xc}}$ potenital. As we operate completely in the Density Funcitonal Approximation domain, this is not a possible task. Instead, the GRAC correction requires that we know our Ionization Potential and we can replace the complex DFT term with the following:

$$v_{\rm{xc}}({\bf r}) = -\frac{1}{r} + (I_{\bf P} + \epsilon_N)$$

Where $\epsilon_N$ is our HOMO energy. This is quite clever as we now only need a functional that correctly goes to the $\frac{1}{r}$ asymptotic. As no general functional accomplishes this task a special functional, "LB94", was built with exactly this property. The downside is that this functionals characteristics are quite poor in the "bulk" region or the region near the atom itself. To generalize this a switching function is introduced:

$$
\begin{align}
v_{\rm{xc}}^{GRAC}(\rho({\bf r})) &= [1 - f[x({\bf r})]]v_{\rm{XC}}^{\rm bulk}+ f[x({\bf r})]v_{\rm{XC}}^{\rm Asym.}  \\ \\
f[x({\bf r})] &= \frac{1}{1 + e^{-\alpha[x({\bf r}) - \beta]}}\\\\
x({\bf r}) &= \frac{| \nabla \rho({\bf r}) | }{\rho^\frac{4}{3}({\bf r})}
\end{align}
$$

Thus, in such a way we seemlessly switch between the bulk and asymptotic potenitals. In addition, such a formulation allows us to apply the GRAC correction to any underlying functional in a general way.

```python
# Build a LB 94 functional
lb94_func = psi4.core.SuperFunctional.blank()
lb94_func.set_max_points(500000)
lb94_func.set_deriv(1)

# Add member functionals
tmp_func = psi4.core.LibXCFunctional('XC_GGA_X_LB', True)
# We need to scale this for the amount of exact exchange is in the functional
tmp_func.set_alpha(0.75)
lb94_func.add_x_functional(tmp_func)

# Dont forget VWN correlation!
tmp_func = psi4.core.LibXCFunctional('XC_LDA_C_VWN', True)
lb94_func.add_c_functional(tmp_func)

# Allocate the functional
lb94_func.allocate()

grac_shift = (21.5645 - dft_ip) /  psi4.constants.hartree2ev

def compute_V_GRAC(D, Vpot):
    """
    Computes the XC energy and the V_xc Fock term
    """

    grac_alpha = 0.5
    grac_beta = 40.0

    nbf = D.shape[0]
    Varr = np.zeros((nbf, nbf))

    points_func = Vpot.properties()[0]
    superfunc = Vpot.functional()

    total_e = 0.0

    for x in range(Vpot.nblocks()):

        # Grid data
        grid = Vpot.get_block(x)
        w = np.array(grid.w())
        npoints = w.shape[0]

        points_func.compute_points(grid)
        lpos = np.array(grid.functions_local_to_global())
        nfunctions = lpos.shape[0]

        phi = np.array(points_func.basis_values()["PHI"])[:npoints, :nfunctions]

        rho = np.array(points_func.point_values()["RHO_A"])[:npoints]
        gamma = np.array(points_func.point_values()["GAMMA_AA"])[:npoints]

        grac_x = (gamma ** 0.5) / (rho ** (4.0/3.0))
        grac_fx = 1.0 / (1.0 + np.exp(-grac_alpha * (grac_x - grac_beta)))

        ret = superfunc.compute_functional(points_func.point_values(), -1)
        lbret = lb94_func.compute_functional(points_func.point_values(), -1)

        total_e += np.vdot(np.array(ret["V"])[:npoints], w)

        v_rho_a  = ( 1 - grac_fx ) * (np.array(ret["V_RHO_A"])[:npoints] - grac_shift)
        v_rho_a += grac_fx * np.array(lbret["V_RHO_A"])[:npoints]
        v_rho_a *= 0.5 * w

        Vtmp = np.einsum('pb,p,pa->ab', phi, v_rho_a, phi, optimize=True)

        # GGA
        rho_x = np.array(points_func.point_values()["RHO_AX"])[:npoints]
        rho_y = np.array(points_func.point_values()["RHO_AY"])[:npoints]
        rho_z = np.array(points_func.point_values()["RHO_AZ"])[:npoints]

        phi_x = np.array(points_func.basis_values()["PHI_X"])[:npoints, :nfunctions]
        phi_y = np.array(points_func.basis_values()["PHI_Y"])[:npoints, :nfunctions]
        phi_z = np.array(points_func.basis_values()["PHI_Z"])[:npoints, :nfunctions]

        v_gamma  = (1 - grac_fx) * np.array(ret["V_GAMMA_AA"])[:npoints]
        v_gamma *= 2.0 * w
        Vtmp += np.einsum('pb,p,p,pa->ab', phi, v_gamma, rho_x, phi_x, optimize=True)
        Vtmp += np.einsum('pb,p,p,pa->ab', phi, v_gamma, rho_y, phi_y, optimize=True)
        Vtmp += np.einsum('pb,p,p,pa->ab', phi, v_gamma, rho_z, phi_z, optimize=True)

        Varr[(lpos[:, None], lpos)] += Vtmp + Vtmp.T

    return total_e, Varr

# Test our function the the handy ks solver.
grac_e, grac_data = ks.ks_solver("PBE0", mol, options, compute_V_GRAC)
```

    Number of occupied orbitals: 5
    Number of basis functions:   23

    Total time taken for setup: 0.105 seconds

    Starting SCF iterations:

        Iter            Energy             XC E         Delta E        D RMS

    SCF Iter  1: -122.77276807253172   -11.5514842   -1.22773E+02   1.54426E-01
    SCF Iter  2: -117.44913553130895    -6.4570001    5.32363E+00   1.90436E-01
    SCF Iter  3: -128.70194843644396    -9.6941063   -1.12528E+01   2.73924E-02
    SCF Iter  4: -128.80625192857761    -9.4817965   -1.04303E-01   1.01038E-02
    SCF Iter  5: -128.82004266191009    -9.3867465   -1.37907E-02   1.31926E-03
    SCF Iter  6: -128.82045018472292    -9.3709337   -4.07523E-04   5.00298E-05
    SCF Iter  7: -128.82045599552373    -9.3713087   -5.81080E-06   1.95107E-06
    SCF Iter  8: -128.82045594675131    -9.3713278    4.87724E-08   6.32213E-08

    Total time for SCF iterations: 7.182 seconds

    Final SCF energy: -128.82045595 hartree

We can now check against the Psi4 reference implementation. Note that we set the GRAC shift on either side of the computation so that we do not contaminate other computations.

```python
psi4.set_options({"dft_grac_shift": grac_shift})
print(psi4.energy("PBE0"))
psi4.set_options({"dft_grac_shift": 0.0})
```

    -128.820455952288

## 3. Inspecting the outcome
Now that we applied the GRAC shift we can observe the density and see how well we do.

```python
grac_ip = -1 * grac_data["eigenvalues"].np[dft_wfn.nalpha()-1] * psi4.constants.hartree2ev

print("Neon Ionization Potential (eV)")
print("SCF     %10.4f" % scf_ip)
print("PBE0    %10.4f" % dft_ip)
print("PBE0-AC %10.4f" % grac_ip)
print("CISD    %10.4f" % ci_ip)
print("NIST    %10.4f" % 21.5645)
```

    Neon Ionization Potential (eV)
    SCF        23.2125
    PBE0       16.0337
    PBE0-AC    21.4446
    CISD       22.2776
    NIST       21.5645

We observe that our ionization potenital is now much closer to the correct ionization potenital and while the density is not perfect, it is much closer to the CISD density.

```python
R, data = one_electron_radial([scf_wfn.Da(), dft_wfn.Da(), grac_data["Da"], ci_wfn.Da()], Vpot)
data_dict = {"SCF": data[0], "DFT": data[1], "DFT-AC": data[2]}
plot_scatter_ratio(data_dict, data[3], R)
```

Refs:
 - M. Gruning, O. V. Gritsenko, S. J. A. van Gisbergen, and E. J. BaerendsJ. Chem. Phys., **2001**, *114*, 652


```python
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential

print(tf.__version__)
#cool this works with git
```

    1.13.0-dev20181221

\begin{align}
\dot{x} & = \sigma(y-x) \\
\dot{y} & = \rho x - y - xz \\
\dot{z} & = -\beta z + xy
\end{align}

```python
model = Sequential()
```

```python
from tensorflow.keras.layers import Dense

model.add(keras.layers.Flatten(input_shape=(28, 28)))
model.add(Dense(units=64, activation='relu', input_dim=100))
model.add(Dense(units=10, activation='softmax'))
```

```python
model.compile(loss='sparse_categorical_crossentropy',
             optimizer='sgd',
             metrics=['accuracy'])
```

```python
fashion_mnist = keras.datasets.fashion_mnist

(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
train_images = train_images / 255.0

test_images = test_images / 255.0

model.fit(train_images,train_labels, epochs=5,batch_size=32)
```

    Epoch 1/5
    60000/60000 [==============================] - 3s 50us/sample - loss: 0.7521 - acc: 0.7588
    Epoch 2/5
    60000/60000 [==============================] - 3s 48us/sample - loss: 0.5149 - acc: 0.8263
    Epoch 3/5
    60000/60000 [==============================] - 3s 49us/sample - loss: 0.4692 - acc: 0.8389
    Epoch 4/5
    60000/60000 [==============================] - 3s 49us/sample - loss: 0.4442 - acc: 0.8472
    Epoch 5/5
    60000/60000 [==============================] - 3s 49us/sample - loss: 0.4255 - acc: 0.8531

    <tensorflow.python.keras.callbacks.History at 0x7fafe7afbba8>

```python
test_loss, test_acc = model.evaluate(test_images, test_labels)
```

     9376/10000 [===========================>..] - ETA: 0s - loss: 0.4510 - acc: 0.8431

```python
predictions = model.predict(test_images)
```

```python
predictions[0]
#sum(predictions[0])
```

    array([3.1326774e-05, 2.6694561e-05, 8.0051126e-05, 3.1632266e-05,
           1.4969780e-05, 8.5115217e-02, 1.3452196e-04, 1.9154643e-01,
           4.0055118e-03, 7.1901357e-01], dtype=float32)

```python
test_labels[0]

```

    9

```python
def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])

  plt.imshow(img, cmap=plt.cm.binary)

  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label:
    color = 'blue'
  else:
    color = 'red'

  plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
  predictions_array, true_label = predictions_array[i], true_label[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])
  thisplot = plt.bar(range(10), predictions_array, color="#777777")
  plt.ylim([0, 1])
  predicted_label = np.argmax(predictions_array)

  thisplot[predicted_label].set_color('red')
  thisplot[true_label].set_color('blue')
```

```python
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
import numpy as np
i = 0
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions, test_labels, test_images)
plt.subplot(1,2,2)
plot_value_array(i, predictions,  test_labels)
```

```python

```