last edited on May 24, 2019, updated and cleaned up on June 24, 2019

# Get Summary Stats of Simulations
this gets summary stats of simulations, csv is also saved to personal computer, a little messy but does work!

```python
#!/bin/env python

#SBATCH --job-name=stat_full
#SBATCH --output=statplt_%j.out
#SBATCH --time=24:05:00
#SBATCH --partition=bigmem2
#SBATCH --nodes=1
#SBATCH --mem=0
# #SBATCH --exclusive
# this one did work

# import packages
import numpy as np
from scipy.signal import get_window, csd
from scipy.signal.windows import hann, hanning, nuttall, flattop
from scipy.fftpack import fft, ifft, fftfreq, fftshift, ifftshift

import matplotlib.pyplot as plt

import scipy.integrate as sciint
import pandas as pd
import datetime
import matplotlib.cm as cm
from math import pi
import matplotlib.ticker as tck
import datetime
from sympy import solve, Poly, Eq, Function, exp, re, im
from netCDF4 import Dataset, num2date # This is to read .nc files and time array
from scipy.optimize import fsolve
from decimal import Decimal
import pickle
import multiprocessing as mp
from joblib import Parallel, delayed
import matplotlib.colors as colors
from seaborn import cubehelix_palette #for contour plot colors
import seaborn as sns
from decimal import Decimal
import numpy.ma as ma
from scipy.stats import skew

# get file names and collect sims
from os import walk
import pickle

flabs = []
for (dirpath, dirnames, filenames) in walk('gphfiles/'):
    flabs.extend(filenames)
    break

f = []
for (dirpath, dirnames, filenames) in walk('/scratch/midway2/clairev/from_home/01_full_sims/'):
    f.extend(filenames)
    break

jjj = 0

```

```python
f = []
for (dirpath, dirnames, filenames) in walk('scratch-midway2/enso_sims/'):
    f.extend(filenames)
    break
```

```python
seasons = ["winter", "spring", "summer", "fall"]
ens = ["nino", "nina", "neutral"]
d2_names = [enso + " " + part for part in seasons for enso in ens]
```

for wantfile in range(len(flabs)):
    index = str(flabs[wantfile][-10:-5])

    # get detrend
    ring = 'detrended/new_detrend_' + str(index) + '.h5'

    data_store = pd.HDFStore(ring)

    # Retrieve data using key
    untrend_df = data_store['untrend_geopot']
    data_store.close()

    seasons = ["winter","spring","summer","fall"]
    # write flatten function

    untrend_df["seasonmean"] = untrend_df.groupby(by=['year','season'])['adj_z'].transform('mean')
    untrend_df["diff_mean"] = untrend_df["adj_z"] - untrend_df["seasonmean"]

```python
storage_list[0]
```

    ['58.5Nnino winter',
     509.8705139015072,
     (-0.15950769881999285-0.016854110781558046j)]

```python

for wantfile in range(1,len(flabs)):
    index = str(flabs[wantfile][-10:-5])
# go through sims to get the correct ones
    sims = []
    for name in f:
        if name[8:13] == flabs[wantfile][-10:-5]:
            print(name)
            file_pickle = open("scratch-midway2/enso_sims/" + name, "rb")
            sims1 = pickle.load(file_pickle)

            sims.append(sims1)

    flatten = lambda l: [item for sublist in l for item in sublist]
    #flatten each
    flat_sims = [[flatten(entry) for entry in sublist] for sublist in sims]

    for j in range(12):
        flat_all = []
        for k in range(len(flat_sims)):
            flat_tested = flat_sims[k]
            flat_all.append(flat_tested[j])

        flat_all = flatten(flat_all)

        act_var = np.var(flat_all)
        act_skew = skew(flat_all, axis = None)

        storage_list.append([str(index) + str(d2_names[j]), act_var, act_skew])

```

```python
import csv

with open('enso_stat', 'wb') as myfile:
    wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
    wr.writerow(storage_list)

```

```python
skew_list = np.real([entry[2] for entry in storage_list])
var_list = np.real([entry[1] for entry in storage_list])
name_list = np.real([entry[0] for entry in storage_list])
```

```python
todo = {"name":name_list, "var": var_list, "skew":skew_list}
```

```python
import pandas
df = pandas.DataFrame(todo)
df.to_csv("enso_stat_2.csv", sep=',',index=False)
```

```python

```

      name
      skew
      var

      0
      58.5Nnino winter
      -0.159508
      509.870514

      1
      58.5Nnina winter
      -0.429642
      502.625568

      2
      58.5Nneutral winter
      -0.321109
      542.798842

      3
      58.5Nnino spring
      -1.050231
      384.394635

      4
      58.5Nnina spring
      0.290616
      367.455810

      5
      58.5Nneutral spring
      0.805822
      352.839923

      6
      58.5Nnino summer
      1.522569
      234.396411

      7
      58.5Nnina summer
      -5.972210
      230.384690

      8
      58.5Nneutral summer
      1.357973
      217.620277

      9
      58.5Nnino fall
      0.321617
      379.411534

      10
      58.5Nnina fall
      0.175340
      383.907261

      11
      58.5Nneutral fall
      0.182353
      388.583098

      12
      31.5Snino winter
      0.027010
      58.041642

      13
      31.5Snina winter
      0.109466
      55.632681

      14
      31.5Sneutral winter
      -0.275419
      57.901194

      15
      31.5Snino spring
      0.609615
      87.539932

      16
      31.5Snina spring
      -2.630782
      74.322073

      17
      31.5Sneutral spring
      0.033434
      85.978238

      18
      31.5Snino summer
      -0.437406
      147.481417

      19
      31.5Snina summer
      -0.539583
      138.915356

      20
      31.5Sneutral summer
      -0.492165
      151.736997

      21
      31.5Snino fall
      -0.190992
      114.306934

      22
      31.5Snina fall
      -0.216047
      99.021668

      23
      31.5Sneutral fall
      -0.186384
      111.372161

      24
      45.0Nnino winter
      -0.232637
      499.747142

      25
      45.0Nnina winter
      0.150164
      552.809552

      26
      45.0Nneutral winter
      -0.073994
      499.719317

      27
      45.0Nnino spring
      -0.490483
      309.758611

      28
      45.0Nnina spring
      -1.143344
      304.283364

      29
      45.0Nneutral spring
      -0.450829
      313.763816

      ...
      ...
      ...
      ...

      90
      36.0Snino summer
      -0.627353
      225.510244

      91
      36.0Snina summer
      -1.102747
      207.091323

      92
      36.0Sneutral summer
      -0.383494
      214.111217

      93
      36.0Snino fall
      -0.144428
      165.460370

      94
      36.0Snina fall
      0.289449
      160.876670

      95
      36.0Sneutral fall
      0.066111
      165.697685

      96
      54.0Nnino winter
      0.205682
      606.117537

      97
      54.0Nnina winter
      0.071486
      538.887158

      98
      54.0Nneutral winter
      -0.106673
      563.885113

      99
      54.0Nnino spring
      -0.109757
      390.306675

      100
      54.0Nnina spring
      0.200169
      374.811904

      101
      54.0Nneutral spring
      0.697484
      356.445052

      102
      54.0Nnino summer
      -1.497659
      217.906911

      103
      54.0Nnina summer
      0.145333
      207.391855

      104
      54.0Nneutral summer
      0.270825
      203.926838

      105
      54.0Nnino fall
      -0.164452
      387.115716

      106
      54.0Nnina fall
      0.085704
      377.418075

      107
      54.0Nneutral fall
      0.588370
      401.328652

      108
      40.5Snino winter
      -0.971277
      182.436143

      109
      40.5Snina winter
      -1.402490
      162.535970

      110
      40.5Sneutral winter
      1.385926
      172.695624

      111
      40.5Snino spring
      0.645377
      221.221906

      112
      40.5Snina spring
      0.336784
      198.469901

      113
      40.5Sneutral spring
      5.613228
      201.947995

      114
      40.5Snino summer
      -1.483635
      278.766618

      115
      40.5Snina summer
      -0.352202
      253.691625

      116
      40.5Sneutral summer
      -0.166809
      277.222459

      117
      40.5Snino fall
      -39.275798
      208.107787

      118
      40.5Snina fall
      -0.476596
      218.581430

      119
      40.5Sneutral fall
      1.310123
      216.021168

120 rows × 3 columns

```python
storage_list = []
for wantfile in flabs:
    index = str(wantfile[-10:-5])
    ring = 'detrended/new_detrend_' + str(index) + '.h5'
    data_store = pd.HDFStore(ring)

    # Retrieve data using key
    untrend_df = data_store.select("untrend_geopot")
    untrend_df['adj_z'] = untrend_df['adj_z'].astype(np.float)
    # get stats from each thing
    untrend_df["season_mean"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('mean')
    # get stats from each thing
    untrend_df["season_variance"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('var')

    # get stats from each thing
    untrend_df["diff_from_season"] = untrend_df["adj_z"] - untrend_df["season_mean"]

    #actual means
    act_skew = untrend_df.groupby(['season'])['diff_from_season'].skew()
    act_var = untrend_df.groupby(['season'])['diff_from_season'].var()

    storage_list.append([index,act_var,act_skew])
```

```python
seasons = ["winter", "spring", "summer", "fall"]
ens = ["nino", "nina", "neutral"]
d2_names = [enso + " " + part for part in seasons for enso in ens]
```

```python
flatten = lambda l: [item for sublist in l for item in sublist]
    #flatten each
flat_sims = [[flatten(entry) for entry in sublist] for sublist in sims]
```

```python
for j in range(4):
    plt.clf();
    plt.figure(figsize=(15, 5));
    plt.hist(x = np.real(untrend_df[untrend_df["season"] == seasons[j]]["diff_mean"]),
                 bins = 20, density = True,
                 alpha = 0.5, label = "reanalysis")

    flat_all = []
    for k in range(len(flat_sims)):
        flat_tested = flat_sims[k]
        flat_all.append(flat_tested[j])

    flat_all = flatten(flat_all)
    for k in range(3):
        #print("hi")
        plt.hist(x = np.real(flat_all[j*3 + k]), bins = 100,
                 density = True, alpha = 0.5, label = d2_names[j*3 + k])

    plt.ylabel("density")
    plt.legend()
    plt.xlabel("departure from mean geopotential height")
    plt.title(str(flabs[wantfile][-10:-5]) + " season: " +str(seasons[j]))

    # formatting
    sns.set_style("ticks")
    sns.set_context("poster")
    sns.despine()
    plt.show()

```

```python
 for j in range(4):

    flat_all = []
    for k in range(len(sims)):
        flat_tested = flat_sims[k]

        flat_all.append(flat_tested[j])
```

```python
len(flat_all[0])
```

    6606240

```python
wantfile = 0

index = str(flabs[wantfile][-10:-5])

# get detrend
ring = 'detrended/new_detrend_' + str(index) + '.h5'

data_store = pd.HDFStore(ring)
```

```python
storage_list = []
for wantfile in flabs:
    index = str(wantfile[-10:-5])
    ring = 'detrended/new_detrend_' + str(index) + '.h5'
    data_store = pd.HDFStore(ring)

    # Retrieve data using key
    untrend_df = data_store.select("untrend_geopot")
    untrend_df['adj_z'] = untrend_df['adj_z'].astype(np.float)
    # get stats from each thing
    untrend_df["season_mean"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('mean')
    # get stats from each thing
    untrend_df["season_variance"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('var')

    # get stats from each thing
    untrend_df["diff_from_season"] = untrend_df["adj_z"] - untrend_df["season_mean"]

    #actual means
    act_skew = untrend_df.groupby(['season'])['diff_from_season'].skew()
    act_var = untrend_df.groupby(['season'])['diff_from_season'].var()

    storage_list.append([index,act_var,act_skew])
```

    /home/clairev/anaconda3/lib/python3.7/site-packages/pandas/core/dtypes/cast.py:702: ComplexWarning: Casting complex values to real discards the imaginary part
      return arr.astype(dtype, copy=True)

```python
len(storage_list)
```

    13

```python
# Retrieve data using key
untrend_df = data_store.select("untrend_geopot")
untrend_df['adj_z'] = untrend_df['adj_z'].astype(np.float)
```

```python
# get stats from each thing
untrend_df["season_mean"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('mean')
# get stats from each thing
untrend_df["season_variance"] = untrend_df.groupby(['season', "year"])['adj_z'].transform('var')

# get stats from each thing
untrend_df["diff_from_season"] = untrend_df["adj_z"] - untrend_df["season_mean"]
```

```python

```

```python
#actual means
act_skew = untrend_df.groupby(['season'])['diff_from_season'].skew()
act_var = untrend_df.groupby(['season'])['diff_from_season'].var()
```

```python
season_df = untrend_df[untrend_df["season"] == "fall"]
```

```python
test_vals = season_df["diff_from_season"]
```

```python
#test_vals
```

```python
test_skew = skew(test_vals)
```

```python
act_skew
```

    season
    fall     -0.407854
    spring   -0.431184
    summer   -0.595719
    winter   -0.210709
    Name: diff_from_season, dtype: float64

```python
test_skew = untrend_df.groupby(['season',"year"])['diff_from_season'].skew()
```

```python
np.average(test_skew["fall"]) # honestly its pretty close so im going to use that..
```

    -0.40382844767999915

```python
act_var # have variance
```

    season
    fall      389.564229
    spring    358.382595
    summer    223.546377
    winter    532.385968
    Name: diff_from_season, dtype: float64

```python
act_skew # now repeat this many times
```

    season
    fall      0.256179
    spring    0.168065
    summer   -0.184236
    winter    0.302864
    Name: diff_from_season, dtype: float64

```python
# get stats from each thing
untrend_df["season_mean"] = df["sst_seasdiff"].sub(df.groupby('year_num')["sst_seasdiff"].transform('mean'))
```

```python
f[0:20]

```

    []

```python
seasons = ["winter","spring","summer","fall"]
# write flatten function
winter_means = list(untrend_df[untrend_df["season"]
                                   == seasons[0]].groupby(['lon'])["adj_z"].mean())
spring_means = list(untrend_df[untrend_df["season"]
                                   == seasons[1]].groupby(['lon'])["adj_z"].mean())
summer_means = list(untrend_df[untrend_df["season"]
                                   == seasons[2]].groupby(['lon'])["adj_z"].mean())
fall_means = list(untrend_df[untrend_df["season"]
                                 == seasons[3]].groupby(['lon'])["adj_z"].mean())

all_means = [winter_means, spring_means, summer_means, fall_means]

# go through sims to get the correct ones
sims = []
for name in f[0:20]:
    print(name)
    if name[4:9] == flabs[wantfile][-10:-5]:
        file_pickle = open("/scratch/midway2/clairev/from_home/01_full_sims/" + name, "rb")
        sims1 = pickle.load(file_pickle, encoding = "latin1", fix_imports=True)
        newsims = [[list(np.add(sims1[season][0][j],
                                    np.real(all_means[season][j])))
                        for j in range(len(sims1[season][0]))]
                       for season in range(4)]
        sims.append(newsims)

flatten = lambda l: [item for sublist in l for item in sublist]
    #flatten each
flat_sims = [[flatten(entry) for entry in sublist] for sublist in sims]
```

```python
sertest = (entry for entry in sims)
```

```python
# teststack = np.dstack((sims[0], sims[1]))
# flatten around last axis
# newstack = [[flatten(entry) for entry in season] for season in teststack]
```

```python
runnumber = len(sims)
entry_len = len(sims[0][0][0])

lon_skew = [[skew(entry) for entry in season] for season in newstack]

full_skew = [skew(season) for season in newstack]
```

```python
# now put everythig into a dataframe
for season in range(4):

    meds = lon_median[season]
    meds.append(full_median[season])

    skewed = lon_skew[season]
    skewed.append(full_skew[season])

    varr = lon_vars[season]
    varr.append(full_var[season])

    avgs = lon_avgs[season]
    avgs.append(full_avg[season])

    lon_list = [i*1.5 for i in range(240)]
    lon_list.append("all")

    runtimes = [runnumber for i in range(241)]
    entrylen = [entry_len for i in range(241)]
    version = [flabs[wantfile][-10:-5] for i in range(241)]

    #append all to a temp frame
    d = {"version" : version, "runtimes" : runtimes, "entrylen" : entrylen,
        "lon": lon_list, "median" : meds, "skew" : skewed, "variance":varr, "average" : avgs}

    df = pd.DataFrame(data=d)

    jjj = jjj + 1
```

```python
jjj
```

    5

```python
# see if dataframe saved alright -- and it did
unpickled_df = pd.read_pickle("sims1_stats.pkl")
```

```python
#unpickled_df[unpickled_df["version"] == "58.5S"]
```

```python
old_stats = pd.read_csv("sims1_stats.csv")
```

```python
old_stats = unpickled_df
```

```python
old_skew = old_stats[old_stats["lon"] == 500]["skew"]
```

```python
def get_skew(mean,median,variance):
    sd = np.sqrt(variance)
    return 3*(mean - median)/ sd
```

```python
import re
class TFConverter(dict):
    column_name_pattern = re.compile(r'_tf$')
    def __getitem__(self, k):
        if k in self:
            return TFConverter.convert
        else:
            raise KeyError(k)
    def __contains__(self,k):
        return self.column_name_pattern.search(k) is not None
    @staticmethod
    def convert(txt):
        return complex(txt.strip("()"))

def read_tf_csv(filename, **kwargs):
    return pd.read_csv(filename, converters = TFConverter(), **kwargs)
```

```python
old_stats["new_skew"] = get_skew(old_stats["average"], old_stats["variance"], old_stats["median"])
```

```python
# save it again as a CSV!
old_stats.to_csv("sims1_stats2.csv")
```

```python
storage_list
```

    [['58.5N', season
      fall      389.564229
      spring    358.382595
      summer    223.546377
      winter    532.385968
      Name: diff_from_season, dtype: float64, season
      fall      0.256179
      spring    0.168065
      summer   -0.184236
      winter    0.302864
      Name: diff_from_season, dtype: float64], ['31.5S', season
      fall      110.261088
      spring     82.412888
      summer    150.716904
      winter     58.867478
      Name: diff_from_season, dtype: float64, season
      fall     -0.420345
      spring   -0.538256
      summer   -0.415503
      winter   -0.605304
      Name: diff_from_season, dtype: float64], ['45.0N', season
      fall      284.486461
      spring    316.910050
      summer    134.797566
      winter    546.963363
      Name: diff_from_season, dtype: float64, season
      fall     -0.323167
      spring   -0.317565
      summer   -0.531423
      winter   -0.124741
      Name: diff_from_season, dtype: float64], ['45.0S', season
      fall      292.054864
      spring    309.494125
      summer    329.100689
      winter    301.643735
      Name: diff_from_season, dtype: float64, season
      fall     -0.305224
      spring   -0.504135
      summer   -0.210158
      winter   -0.469655
      Name: diff_from_season, dtype: float64], ['58.5S', season
      fall      436.235287
      spring    475.715979
      summer    491.687052
      winter    382.248331
      Name: diff_from_season, dtype: float64, season
      fall      0.212365
      spring    0.274388
      summer    0.142022
      winter    0.483158
      Name: diff_from_season, dtype: float64], ['54.0S', season
      fall      440.298301
      spring    499.109734
      summer    472.968543
      winter    453.733755
      Name: diff_from_season, dtype: float64, season
      fall     -0.021755
      spring   -0.021432
      summer   -0.051517
      winter    0.181555
      Name: diff_from_season, dtype: float64], ['49.5S', season
      fall      377.772146
      spring    429.692642
      summer    403.423016
      winter    421.470354
      Name: diff_from_season, dtype: float64, season
      fall     -0.211970
      spring   -0.305290
      summer   -0.174252
      winter   -0.159908
      Name: diff_from_season, dtype: float64], ['36.0S', season
      fall      163.421021
      spring    133.053730
      summer    218.234440
      winter     99.818143
      Name: diff_from_season, dtype: float64, season
      fall     -0.385543
      spring   -0.552024
      summer   -0.287168
      winter   -0.681203
      Name: diff_from_season, dtype: float64], ['54.0N', season
      fall      406.379124
      spring    370.509623
      summer    207.335775
      winter    570.541907
      Name: diff_from_season, dtype: float64, season
      fall      0.068552
      spring    0.032045
      summer   -0.314286
      winter    0.186250
      Name: diff_from_season, dtype: float64], ['40.5S', season
      fall      219.641575
      spring    205.514957
      summer    271.623321
      winter    177.566157
      Name: diff_from_season, dtype: float64, season
      fall     -0.342464
      spring   -0.577040
      summer   -0.219582
      winter   -0.662966
      Name: diff_from_season, dtype: float64], ['36.0N', season
      fall      112.316943
      spring    160.060376
      summer     57.161146
      winter    278.281757
      Name: diff_from_season, dtype: float64, season
      fall     -0.375912
      spring   -0.539909
      summer   -0.646068
      winter   -0.213900
      Name: diff_from_season, dtype: float64], ['49.5N', season
      fall      373.544178
      spring    360.704142
      summer    177.509422
      winter    586.046137
      Name: diff_from_season, dtype: float64, season
      fall     -0.152528
      spring   -0.148791
      summer   -0.451394
      winter    0.028529
      Name: diff_from_season, dtype: float64], ['40.5N', season
      fall      183.665350
      spring    243.803518
      summer     92.737145
      winter    433.055791
      Name: diff_from_season, dtype: float64, season
      fall     -0.407854
      spring   -0.431184
      summer   -0.595719
      winter   -0.210709
      Name: diff_from_season, dtype: float64]]

```python

```


## Code from Book

### 2. Getting Started

```python
! whoami
```

```bash
%%bash
echo $0
```

```python
! echo 'hello'
```

```python
import os
os.__file__
```

```python
%timeit 9999 in range(10000)
```

```javascript
%%javascript
var d = 10
alert("d = " + d)
```

### 3. Thinking in Binary

```python
! file my_image.jpg
```

```python
import magic
print magic.from_file("my_image.jpg")
```

```python
if magic.from_file("upload.jpg", mime=True) == "image/jpeg":
    continue_uploading("upload.jpg")
else:
    alert("Sorry! This file type is not allowed")
```

```python
import imghdr
print imghdr.what("path/to/my/file.ext")
```

```python
import binascii

def spoof_file(file, magic_number):
    magic_number = binascii.unhexlify(magic_number)
    with open(file, "r+b") as f:
        old = f.read()
        f.seek(0)
        f.write(magic_number + old)
```

```python
! xxd -b my_file.docx | less
```

```python
! du -h my_file.docx
```

```python
def to_ascii_bytes(string):
    return " ".join(format(ord(char), '08b') for char in string)
```

```python
string = "my ascii string"
"".join(hex(ord(char))[2:] for char in string)
```

```python
hex_string = "6d7920617363696920737472696e67"
print hex_string.decode("hex")
print "".join(chr(int(hex_string[i:i+2], 16)) for i in range(0, len(hex_string), 2))
```

```python
# adapted from https://code.activestate.com/recipes/142812-hex-dumper/
def hexdump(string, length=8):
    result = []
    digits = 4 if isinstance(string, unicode) else 2

    for i in xrange(0, len(string), length):
        s = string[i:i + length]
        hexa = "".join("{:0{}X}".format(ord(x), digits) for x in s)
        text = "".join(x if 0x20 <= ord(x) < 0x7F else '.' for x in s)
        result.append("{:04X}   {:{}}   {}".format(i, hexa, length * (digits + 1), text))

    return '\n'.join(result)
```

```python
print hexdump("The quick brown fox jumps over the lazy dog")
```

```python
import struct

num = 0x103e4
struct.pack("I", 0x103e4)
```

```python
string = '\xe4\x03\x01\x00'
struct.unpack("i", string)
```

```python
bytes = '\x01\xc2'
struct.pack("<h", struct.unpack(">h", bytes)[0])
```

```python
import base64

base64.b64encode('encodings are fun...')
```

```python
print base64.b64decode(_)
```

```python
string = "hello\x00"
binary_string = ' '.join('{:08b}'.format(ord(char)) for char in string)
" ".join(binary_string[i:i+6] for i in range(0, len(binary_string), 6))
```

```python
bin_string = '011010 000110 010101 101100 011011 000110 111100 000000'
[int(b, 2) for b in bin_string.split()]
```

```bash
%%bash
echo -n hello | base64
echo aGVsbG8= | base64 --decode && echo
```

```python
u'◑ \u2020'.encode('utf8')
```

```python
'\xe2\x97\x91 \xe2\x80\xa0'.decode('utf8')
```

```python
unicode('\xe2\x97\x91 \xe2\x80\xa0', encoding='utf8')
```

```python
utf8_string = 'Åêíòü'
utf8_string
```

```python
unicode_string = utf8_string.decode('utf8')
unicode_string
```

```python
unicode_string.encode('mac roman')
```

```python
'Åêíòü'.decode('utf8').encode('ascii') # Raises UnicodeEncodeError
```

```python
! chardetect uni.txt another_file.txt
```

```python
file = """潍楪慢敫椠⁳桴慧扲敬整瑸琠慨⁴獩琠敨爠獥汵⁴景琠硥⁴敢湩敤潣敤獵湩湡甠楮瑮湥敤档
牡捡整⁲湥潣楤杮楷桴挠浯汰瑥汥⁹湵敲慬整湯獥景整牦浯愠搠晩敦敲瑮眠楲楴杮猠獹整⹭‧⠊慔敫
牦浯攠⹮楷楫数楤⹡牯⥧"""

print file.decode('utf8').encode('utf16')
```

```python
import ftfy
ftfy.fix_text(u"â€œMojibakeâ€œ can be fixed.")
```

```python
x = 0b1111
y = 0b1010
bin(int("{:b}{:b}".format(x, y), 2))
```

```python
bin(x << 4 | y)
```

### 4. Cryptography

```python
import random
import string

r = random.SystemRandom()

# Get a random integer between 0 and 20
r.randint(0, 20)

# Get a random number between 0 and 1
r.random()

# Generate a random 40-bit number
r.getrandbits(40)

# Choose a random item from a string or list
chars = string.printable
r.choice(chars)

# Randomize the order of a sequence
seq = ['a', 'b', 'c', 'd', 'e']
r.shuffle(seq)
```

```python
"ALLIGATOR".encode('rot13')
```

```python
"NYYVTNGBE".encode('rot13')
```

```python
plaintext = "A secret-ish message!"
"".join(chr((ord(c) + 20) % 256) for c in plaintext)
```

```python
ciphertext = 'U4\x87yw\x86y\x88A}\x87|4\x81y\x87\x87u{y5'
"".join(chr((ord(c) - 20) % 256) for c in ciphertext)
```

```python
plaintext = 0b110100001101001
one_time_pad = 0b110000011100001
bin(plaintext ^ one_time_pad)
```

```python
decrypted = 0b100010001000 ^ one_time_pad
format(decrypted, 'x').decode('hex')
```

```python
import os
import binascii

# ASCII-encoded plaintext
plaintext = "this is a secret message"
plaintext_bits = int(binascii.hexlify(plaintext), 16)

print "plaintext (ascii):", plaintext
print "plaintext (hex):", plaintext_bits

# Generate the one-time pad
onetime_pad = int(binascii.hexlify(os.urandom(len(plaintext))), 16)

print "one-time pad: (hex):", onetime_pad

# Encrypt plaintext using XOR operation with one-time pad
ciphertext_bits = plaintext_bits ^ onetime_pad

print "encrypted text (hex):", ciphertext_bits

# Decrypt using XOR operation with one-time pad
decrypted_text = ciphertext_bits ^ onetime_pad
decrypted_text = binascii.unhexlify(hex(decrypted_text)[2:-1])

print "decrypted text (ascii):", decrypted_text
```

```python
import random
import binascii

p1 = "this is the part where you run away"
p2 = "from bad cryptography practices."

# pad plaintexts with spaces to ensure equal length
p1 = p1.ljust(len(p2))
p2 = p2.ljust(len(p1))

p1 = int(binascii.hexlify(p1), 16)
p2 = int(binascii.hexlify(p2), 16)

# get random one-time pad
otp = random.SystemRandom().getrandbits(p1.bit_length())

# encrypt
c1 = p1 ^ otp
c2 = p2 ^ otp  # otp reuse...not good!

print "c1 ^ c2 == p1 ^ p2 ?", c1 ^ c2 == p1 ^ p2
print "c1 ^ c2 =", hex(c1 ^ c2)

# the crib
crib = " the "
crib = int(binascii.hexlify(crib), 16)

xored = c1 ^ c2

print "crib =", hex(crib)

cbl = crib.bit_length()
xbl = xored.bit_length()

print
mask = (2**(cbl + 1) - 1)
fill = len(str(xbl / 8))

# crib dragging
for s in range(0, xbl - cbl + 8, 8):
    xor = (xored ^ (crib << s)) & (mask << s)
    out = binascii.unhexlify(hex(xor)[2:-1])

    print "{:>{}}   {}".format(s/8, fill, out)
```

```python
from cryptography.fernet import Fernet
key = Fernet.generate_key()
f = Fernet(key)
ciphertext = f.encrypt("this is my plaintext")
decrypted = f.decrypt(ciphertext)
print decrypted
```

```python
import os
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

pt = "my plaintext"

backend = default_backend()
key = os.urandom(32)
iv = os.urandom(16)

padder = padding.PKCS7(128).padder()
pt = padder.update(pt) + padder.finalize()

cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend)
encryptor = cipher.encryptor()
ct = encryptor.update(pt) + encryptor.finalize()
decryptor = cipher.decryptor()
out = decryptor.update(ct) + decryptor.finalize()

unpadder = padding.PKCS7(128).unpadder()
out = unpadder.update(out) + unpadder.finalize()
print out

```

```python
nonce = os.urandom(64/8)
nonce
```

```python
import hashlib
hashlib.md5("hash me please").hexdigest()
```

```python
! md5 -s 'hash me please'
```

```python
hashlib.sha1("hash me please").hexdigest()
```

```python
! echo 'hash me please' | openssl dgst -sha1
```

```python
m1 = binascii.unhexlify("d131dd02c5e6eec4693d9a0698aff95c2fcab58712467eab4004583eb8fb7f8955ad340609f4b30283e488832571415a085125e8f7cdc99fd91dbdf280373c5bd8823e3156348f5bae6dacd436c919c6dd53e2b487da03fd02396306d248cda0e99f33420f577ee8ce54b67080a80d1ec69821bcb6a8839396f9652b6ff72a70")

m2 = binascii.unhexlify("d131dd02c5e6eec4693d9a0698aff95c2fcab50712467eab4004583eb8fb7f8955ad340609f4b30283e4888325f1415a085125e8f7cdc99fd91dbd7280373c5bd8823e3156348f5bae6dacd436c919c6dd53e23487da03fd02396306d248cda0e99f33420f577ee8ce54b67080280d1ec69821bcb6a8839396f965ab6ff72a70")

hashlib.md5(m1).hexdigest() == hashlib.md5(m1).hexdigest()
```

```python
import os
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
from cryptography.hazmat.backends import default_backend

backend = default_backend()
salt = os.urandom(16)

kdf = Scrypt(salt=salt, length=64, n=2**14, r=8, p=1, backend=backend)
key = kdf.derive("your favorite password")
key
```

```python
kdf = Scrypt(salt=salt, length=64, n=2**14, r=8, p=1, backend=backend)
kdf.verify("your favorite password", key)
```

```python
import hmac
import hashlib

secret_key = "my secret key"
ciphertext = "my ciphertext"

# generate HMAC
h = hmac.new(key=secret_key, msg=ciphertext, digestmod=hashlib.sha256)
print h.hexdigest()

# verify HMAC
hmac.compare_digest(h.hexdigest(), h.hexdigest())
```

```python
p = 9576890767
q = 1299827
n = p * q
print n
```

```python
e = 65537
phi = (p - 1) * (q - 1)
phi % e != 0
```

```python
import sympy

d = sympy.numbers.igcdex(e, phi)[0]
print d
```

```python
m = 12345
c = pow(m, e, n)
print c
```

```python
pow(c, d, n)
```

```python
m = 0
while pow(m, e, n) != c:
    m += 1
print m
```

```python
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization

private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend())

public_key = private_key.public_key()

private_pem = private_key.private_bytes(encoding=serialization.Encoding.PEM,
                                        format=serialization.PrivateFormat.PKCS8,
                                        encryption_algorithm=serialization.BestAvailableEncryption('your password here'))

public_pem = public_key.public_bytes(encoding=serialization.Encoding.PEM,
                                     format=serialization.PublicFormat.SubjectPublicKeyInfo)

print public_pem
print private_pem
```

```python
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
import base64

with open("path/to/public_key.pem", "rb") as key_file:
    public_key = serialization.load_pem_public_key(key_file.read(),
                                                   backend=default_backend())

message = "your secret message"
ciphertext = public_key.encrypt(message,
                                padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()),
                                             algorithm=hashes.SHA256(),
                                             label=None))
b64_ciphertext = base64.urlsafe_b64encode(ciphertext)
print b64_ciphertext

```

```python
plaintext = private_key.decrypt(ciphertext,
                                padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()),
                                             algorithm=hashes.SHA256(),
                                             label=None))
print plaintext
```

```python
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding

signer = private_key.signer(padding.PSS(mgf=padding.MGF1(hashes.SHA256()),
                                        salt_length=padding.PSS.MAX_LENGTH),
                            hashes.SHA256())
message = "A message of arbitrary length"
signer.update(message)
signature = signer.finalize()
signature
```

```python
public_key = private_key.public_key()
verifier = public_key.verifier(signature,
                               padding.PSS(mgf=padding.MGF1(hashes.SHA256()),
                                           salt_length=padding.PSS.MAX_LENGTH),
                               hashes.SHA256())
verifier.update(message)
verifier.verify()
```

### 5. Networking

```python
import requests
r = requests.get('https://www.google.com/imghp')
r.content[:200]
```

```python
r.status_code
```

```python
r.headers
```

```python
len(r.content)
```

```python
r.apparent_encoding
```

```python
r.elapsed
```

```python
r.request.headers
```

```python
custom_headers = {"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"}
r = requests.get("https://www.google.com/imghp", headers=custom_headers)
r.request.headers
```

```python
import requests
import logging
import http.client

# Enable logging
http.client.HTTPConnection.debuglevel = 1

logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
r = requests.get('https://www.google.com/')
```

```python
import urlparse
simple_url = "http://www.example.com/path/to/my/page"
parsed = urlparse.urlparse(simple_url)
```

```python
parsed.scheme
```

```python
parsed.hostname
```

```python
parsed.path
```

```python
url_with_query = "http://www.example.com/?page=1&key=Anvn4mo24"
query = urlparse.urlparse(url_with_query).query
urlparse.parse_qs(query)
```

```python
import urllib
url = 'https://www.example.com/%5EA-url-with-%-and-%5E?page=page+with%20spaces'
urllib.unquote(url)
```

```python
chars = '!@#$%^%$#)'
urllib.quote(chars)
```

```python
urllib.unquote_plus(url)
```

```python
urllib.quote_plus('one two')
```

```python
import requests
from bs4 import BeautifulSoup

http_client.HTTPConnection.debuglevel = 0  # Logging off
r = requests.get("http://www.google.com")
soup = BeautifulSoup(r.content, "lxml")

soup.find_all('p')
```

```python
soup.find_all('a')
```

```python
for link in soup.find_all('a'):
    print link.text, link["href"]
```

```python
import dryscrape
from bs4 import BeautifulSoup
session = dryscrape.Session()
session.visit("http://www.google.com")
r = session.body()
soup = BeautifulSoup(r, "lxml")
```

```python
from selenium import webdriver
driver = webdriver.Chrome("/path/to/chromedriver")
driver.get("http://www.google.com")
html = driver.page_source
driver.save_screenshot("screenshot.png")
driver.quit()
```

```python
import smtplib

server = smtplib.SMTP('localhost', port=1025)
server.set_debuglevel(True)
server.sendmail("me@localhost", "you@localhost", "This is an email message")
server.quit()
```

```python
! host google.com
```

```python
! host 172.217.11.14
```


_Lambda School Data Science — Linear Models_

# Understanding Linear Regression

#### Objectives
- understand how ordinary least squares regression minimizes the sum of squared errors
- understand how linear algebra can solve ordinary least squares regression
- get and interpret coefficients of a linear model
- visualize a line of best fit in 2D, and hyperplane in 3D
- use regression metrics: MSE, RMSE, MAE, R^2

#### Extra Links
- [Statistics 101: Simple Linear Regression](https://www.youtube.com/watch?v=ZkjP5RJLQF4) (20 minute video)
- [_An Introduction to Statistical Learning_](http://www-bcf.usc.edu/~gareth/ISL/ISLR%20Seventh%20Printing.pdf), Chapter 3.1, Simple Linear Regression, & 3.2, Multiple Linear Regression
- Priceonomics, [The Discovery of Statistical Regression](https://priceonomics.com/the-discovery-of-statistical-regression/)
- Priceonomics, [Why the Father of Modern Statistics Didn’t Believe Smoking Caused Cancer](https://priceonomics.com/why-the-father-of-modern-statistics-didnt-believe/)
- Harvard Business Review, [When to Act on a Correlation, and When Not To](https://hbr.org/2014/03/when-to-act-on-a-correlation-and-when-not-to)
- [xkcd 552: Correlation](https://www.explainxkcd.com/wiki/index.php/552:_Correlation)
- [xkcd 1725: Linear Regression](https://www.explainxkcd.com/wiki/index.php/1725:_Linear_Regression)

## What is Linear Regression?

[Linear Regression](https://en.wikipedia.org/wiki/Linear_regression) is a statistical model that seeks to describe the relationship between some y variable and one or more x variables.

In the simplest case, linear regression seeks to fit a straight line through a cloud of points. This line is referred to as the "regression line" or "line of best fit." This line tries to summarize the relationship between our X and Y in a way that enables us to use the equation for that line to make predictions.

```python
%matplotlib inline
import seaborn as sns
x = [10,  8, 13,  9, 11, 14,  6,  4, 12,  7,  5]
y = [ 8,  6,  7,  8,  8,  9,  7,  4, 10,  4,  5]
sns.regplot(x, y);
```

### Synonyms for "y variable"
- Dependent Variable
- Response Variable
- Outcome Variable
- Predicted Variable
- Measured Variable
- Explained Variable
- Label
- Target

### Synonyms for "x variable"
- Independent Variable
- Explanatory Variable
- Regressor
- Covariate
- Feature

## Simple Linear Regresion

#### Making Predictions

Say that we were trying to create a model that captured the relationship between temperature outside and ice cream sales. In Machine Learning our goal is often different that of other flavors of Linear Regression Analysis, because we're trying to fit a model to this data with the intention of making **predictions** on new data (in the future) that we don't have yet.

#### What are we trying to predict?

So if we had measured ice cream sales and the temprature outside on 11 different days, at the end of our modeling **what would be the thing that we would want to predict? - Ice Cream Sales or Temperature?**

We would probably want to be measuring temperature with the intention of using that to **forecast** ice cream sales. If we were able to successfully forecast ice cream sales from temperature, this might help us know beforehand how much ice cream to make or how many cones to buy or on which days to open our store, etc. Being able to make predictions accurately has a lot of business implications. This is why making accurate predictions is so valuable (And in large part is why data scientists are paid so well).

#### y variable intuition

We want the thing that we're trying to predict to serve as our **y** variable. This is why it's sometimes called the "predicted variable." We call it the "dependent" variable because our prediction for how much ice cream we're going to sell "depends" on the temperature outside.

#### x variable intuition

All other variables that we use to predict our y variable (we're going to start off just using one) we call our **x** variables. These are called our "independent" variables because they don't *depend* on y, they "explain" y. Hence they are also referred to as our "explanatory" variables.

## Example: Predict presidential election voting

#### Douglas Hibbs, [Background Information on the ‘Bread and Peace’ Model of Voting in Postwar US Presidential Elections](https://douglas-hibbs.com/background-information-on-bread-and-peace-voting-in-us-presidential-elections/)

> Aggregate two-party vote shares going to candidates of the party holding the presidency during the postwar era are well explained by just two fundamental determinants:

> (1) Positively by weighted-average growth of per capita real disposable personal income over the term.
> (2) Negatively by cumulative US military fatalities (scaled to population) owing to unprovoked, hostile deployments of American armed forces in foreign wars.

#### Data sources
- 1952-2012: Douglas Hibbs, [2014 lecture at Deakin University Melbourne](http://www.douglas-hibbs.com/HibbsArticles/HIBBS-PRESVOTE-SLIDES-MELBOURNE-Part1-2014-02-26.pdf), Slide 40
- 2016, Vote Share: [The American Presidency Project](https://www.presidency.ucsb.edu/statistics/elections)
- 2016, Recent Growth in Personal Incomes: [The 2016 election economy: the "Bread and Peace" model final forecast](https://angrybearblog.com/2016/11/the-2016-election-economy-the-bread-and-peace-model-final-forecast.html)
- 2016, US Military Fatalities: Assumption that Afghanistan War fatalities in 2012-16 occured at the same rate as 2008-12

```python
import pandas as pd

columns = ['Year','Inc_Party_Candidate','Other_Candidate','Inc_Party_Vote']

data = [[1952,"Stevenson","Eisenhower",44.6],
        [1956,"Eisenhower","Stevenson",57.76],
        [1960,"Nixon","Kennedy",49.91],
        [1964,"Johnson","Goldwater",61.34],
        [1968,"Humphrey","Nixon",49.60],
        [1972,"Nixon","McGovern",61.79],
        [1976,"Ford","Carter",48.95],
        [1980,"Carter","Reagan",44.70],
        [1984,"Reagan","Mondale",59.17],
        [1988,"Bush, Sr.","Dukakis",53.94],
        [1992,"Bush, Sr.","Clinton",46.55],
        [1996,"Clinton","Dole",54.74],
        [2000,"Gore","Bush, Jr.",50.27],
        [2004,"Bush, Jr.","Kerry",51.24],
        [2008,"McCain","Obama",46.32],
        [2012,"Obama","Romney",52.00],
        [2016,"Clinton","Trump",48.2]]

votes = pd.DataFrame(data=data, columns=columns)
```

```python
columns = ['Year','Avg_Rec_Growth']

data = [[1952,2.40],
        [1956,2.89],
        [1960, .85],
        [1964,4.21],
        [1968,3.02],
        [1972,3.62],
        [1976,1.08],
        [1980,-.39],
        [1984,3.86],
        [1988,2.27],
        [1992, .38],
        [1996,1.04],
        [2000,2.36],
        [2004,1.72],
        [2008, .10],
        [2012, .95],
        [2016, .10]]

growth = pd.DataFrame(data=data, columns=columns)
```

```python
"""
Fatalities denotes the cumulative number of American military
fatalities per millions of US population the in Korea, Vietnam,
Iraq and Afghanistan wars during the presidential terms
preceding the 1952, 1964, 1968, 1976 and 2004, 2008 and
2012 elections.

http://www.douglas-hibbs.com/HibbsArticles/HIBBS-PRESVOTE-SLIDES-MELBOURNE-Part1-2014-02-26.pdf
"""

columns = ['Year','fatal_per_mil']

data = [[1952,190],
        [1956,  0],
        [1960,  0],
        [1964,  1],
        [1968,146],
        [1972,  0],
        [1976,  2],
        [1980,  0],
        [1984,  0],
        [1988,  0],
        [1992,  0],
        [1996,  0],
        [2000,  0],
        [2004,  4],
        [2008, 14],
        [2012,  5],
        [2016,  5]]

deaths = pd.DataFrame(data=data, columns=columns)
```

```python
df = votes.merge(growth).merge(deaths)
```

```python
df.head()
```

      Year
      Inc_Party_Candidate
      Other_Candidate
      Inc_Party_Vote
      Avg_Rec_Growth
      fatal_per_mil

      0
      1952
      Stevenson
      Eisenhower
      44.60
      2.40
      190

      1
      1956
      Eisenhower
      Stevenson
      57.76
      2.89
      0

      2
      1960
      Nixon
      Kennedy
      49.91
      0.85
      0

      3
      1964
      Johnson
      Goldwater
      61.34
      4.21
      1

      4
      1968
      Humphrey
      Nixon
      49.60
      3.02
      146

### Plot univariate correlations
[Seaborn tutorial: Visualizing linear relationships](https://seaborn.pydata.org/tutorial/regression.html)

```python
%matplotlib inline
import seaborn as sns

target = 'Inc_Party_Vote'
features = ['Avg_Rec_Growth',
            'fatal_per_mil']

for feature in features:
    sns.lmplot(x=feature, y=target, data=df)
```

We can see from the scatterplot that these data points seem to follow a somewhat linear relationship for the "Average Recent Growth in Personal Incomes" feature. This means that we could probably summarize their relationship well by fitting a line of best fit to these points. Lets do it.

## The Equation for a Line

A common equation for a line is:

\begin{align}
y = mx + b
\end{align}

Where $m$ is the slope of our line and $b$ is the y-intercept.

If we want to plot a line through our cloud of points we figure out what these two values should be. Linear Regression seeks to **estimate** the slope and intercept values that describe a line that best fits the data points.

## The Anatomy of Linear Regression

- Intercept: The $b$ value in our line equation $y=mx+b$
- Slope: The $m$ value in our line equation $y=mx+b$. These two values together define our regression line.
- $\hat{y}$ : A prediction
- Line of Best Fit (Regression Line)
- Predicted (fitted) Values: Points on our regression line
- Observed Values: Points from our dataset
- Error: The distance between predicted and observed values.

Ordinary Least Squares Regression is a way to solve for $m$ and $b$.

Let's start by seeing what would happen if we just guessed and checked some values for $m$ and $b$.

What's the line of "best" fit look like? What's the error?

```python
# TODO

x = df['Avg_Rec_Growth']
y = df['Inc_Party_Vote']

m = 0
b = y.mean()
y_pred = m * x + b
y_pred
```

    0     51.828235
    1     51.828235
    2     51.828235
    3     51.828235
    4     51.828235
    5     51.828235
    6     51.828235
    7     51.828235
    8     51.828235
    9     51.828235
    10    51.828235
    11    51.828235
    12    51.828235
    13    51.828235
    14    51.828235
    15    51.828235
    16    51.828235
    Name: Avg_Rec_Growth, dtype: float64

```python
import matplotlib.pyplot as plt
from sklearn.metrics import mean_absolute_error, r2_score

def plot_preds(x, y, y_pred):
    plt.scatter(x,y, label='y_true')
    plt.plot(x,y_pred, label='y_pred')
    plt.legend()
    mae = mean_absolute_error(y, y_pred)
    r2 = r2_score(y, y_pred)
    print(f'Mean Absolute Error: {mae}')
    print(f'R^2 Score: {r2}')
```

```python
plot_preds(x, y, y_pred)
```

```python
m = 4.1
b = 45
y_pred = m*x + b
plot_preds(x, y, y_pred)
plt.title('Guessing LR Values')
```

## R Squared:  $R^2$

One final attribute of linear regressions that we're going to talk about today is a measure of goodness of fit known as $R^2$ or R-squared. $R^2$ is a statistical measure of how close the data are fitted to our regression line. A helpful interpretation for the $R^2$ is the percentage of the dependent variable that is explained by the model.

In other words, the $R^2$ is the percentage of y that is explained by the x variables included in the model. For this reason the $R^2$ is also known as the "coefficient of determination," because it explains how much of y is explained (or determined) by our x varaibles. We won't go into the calculation of $R^2$ today, just know that a higher $R^2$ percentage is nearly always better and indicates a model that fits the data more closely.

## Residual Error

The residual error is the distance between points in our dataset and our regression line.

```python
def regression_residuals(df, feature, target, m, b):
    x = df[feature]
    y = df[target]
    y_pred = m*x + b

    plt.scatter(x, y, label='y_true')
    plt.plot(x, y_pred, label='y_pred')
    plt.legend()

    # Plot residual errors
    for x, y1, y2 in zip(x, y, y_pred):
        plt.plot((x, x), (y1, y2), color='grey')

    mae = mean_absolute_error(y, y_pred)
    r2 = r2_score(y, y_pred)
    print('Mean Absolute Error:', mae)
    print('R^2:', r2)

regression_residuals(df, feature='Avg_Rec_Growth',
                     target='Inc_Party_Vote', m=3, b=46)
```

```python
from ipywidgets import interact, fixed

interact(regression_residuals,
         df=fixed(df),
         feature=fixed('Avg_Rec_Growth'),
         target=fixed('Inc_Party_Vote'),
         m=(-10,10,0.5),
         b=(40,60,0.5));
```

    interactive(children=(FloatSlider(value=0.0, description='m', max=10.0, min=-10.0, step=0.5), FloatSlider(valu…

## More Formal Notation

We have talked about a line of regression being represented like a regular line $y=mx+b$ but as we get to more complicated versions we're going to need to extend this equation. So lets establish the proper terminology.

**X** - Independent Variable, predictor variable, explanatory variable, regressor, covariate

**Y** - Response variable, predicted variable, measured vairable, explained variable, outcome variable

$\beta_0$ - "Beta Naught" or "Beta Zero", the intercept value. This is how much of y would exist if X were zero. This is sometimes represented by the letter "a" but I hate that. So it's "Beta 0" during my lecture.

$\beta_1$ - "Beta One" The primary coefficient of interest. This values is the slope of the line that is estimated by "minimizing the sum of the squared errors/residuals" - We'll get to that.

$\epsilon$ - "Epsilon" The "error term", random noise, things outside of our model that affect y.

## Minimizing the Sum of the Squared Error

The most common method of estimating our $\beta$ parameters  is what's known as "Ordinary Least Squares" (OLS). (There are different methods of arriving at a line of best fit). OLS estimates the parameters that minimize the squared distance between each point in our dataset and our line of best fit.

\begin{align}
SSE = \sum(y_i - \hat{y})^2
\end{align}

```python
from matplotlib.patches import Rectangle
import numpy as np
from sklearn.metrics import mean_squared_error

def regression_squared_errors(df, feature, target, m, b):
    x = df[feature]
    y = df[target]
    y_pred = m*x + b

    fig = plt.figure(figsize=(7,7))
    ax = plt.axes()
    ax.scatter(x, y, label='y_true')
    ax.plot(x, y_pred, label='y_pred')
    ax.legend()

    # Plot square errors
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()
    scale = (xmax-xmin)/(ymax-ymin)
    for x, y1, y2 in zip(x, y, y_pred):
        bottom_left = (x, min(y1, y2))
        height = abs(y1 - y2)
        width = height * scale
        ax.add_patch(Rectangle(xy=bottom_left, width=width, height=height, alpha=0.1))

    mse = mean_squared_error(y, y_pred)
    rmse = np.sqrt(mse)
    mae = mean_absolute_error(y, y_pred)
    r2 = r2_score(y, y_pred)
    print('Mean Squared Error:', mse)
    print('Root Mean Squared Error:', rmse)
    print('Mean Absolute Error:', mae)
    print('R^2:', r2)

regression_squared_errors(df, feature='Avg_Rec_Growth',
           target='Inc_Party_Vote', m=3, b=46)
```

```python
interact(regression_squared_errors,
         df=fixed(df),
         feature=fixed('Avg_Rec_Growth'),
         target=fixed('Inc_Party_Vote'),
         m=(-10,10,0.5),
         b=(40,60,0.5));
```

    interactive(children=(FloatSlider(value=0.0, description='m', max=10.0, min=-10.0, step=0.5), FloatSlider(valu…

## Hypotheses

```python
b = 46
ms = np.arange(-10,10,0.5)
sses = []
feature = 'Avg_Rec_Growth'

for m in ms:
    predictions = m * df[feature] + b
    errors = df[target] - predictions
    square_errors = errors ** 2
    sse = square_errors.sum()
    sses.append(sse)

hypotheses = pd.DataFrame({'Slope': ms})
hypotheses['Intercept'] = b
hypotheses['Sum of Square Errors'] = sses

hypotheses
```

      Slope
      Intercept
      Sum of Square Errors

      0
      -10.0
      46
      15215.58740

      1
      -9.5
      46
      14095.52845

      2
      -9.0
      46
      13018.88500

      3
      -8.5
      46
      11985.65705

      4
      -8.0
      46
      10995.84460

      5
      -7.5
      46
      10049.44765

      6
      -7.0
      46
      9146.46620

      7
      -6.5
      46
      8286.90025

      8
      -6.0
      46
      7470.74980

      9
      -5.5
      46
      6698.01485

      10
      -5.0
      46
      5968.69540

      11
      -4.5
      46
      5282.79145

      12
      -4.0
      46
      4640.30300

      13
      -3.5
      46
      4041.23005

      14
      -3.0
      46
      3485.57260

      15
      -2.5
      46
      2973.33065

      16
      -2.0
      46
      2504.50420

      17
      -1.5
      46
      2079.09325

      18
      -1.0
      46
      1697.09780

      19
      -0.5
      46
      1358.51785

      20
      0.0
      46
      1063.35340

      21
      0.5
      46
      811.60445

      22
      1.0
      46
      603.27100

      23
      1.5
      46
      438.35305

      24
      2.0
      46
      316.85060

      25
      2.5
      46
      238.76365

      26
      3.0
      46
      204.09220

      27
      3.5
      46
      212.83625

      28
      4.0
      46
      264.99580

      29
      4.5
      46
      360.57085

      30
      5.0
      46
      499.56140

      31
      5.5
      46
      681.96745

      32
      6.0
      46
      907.78900

      33
      6.5
      46
      1177.02605

      34
      7.0
      46
      1489.67860

      35
      7.5
      46
      1845.74665

      36
      8.0
      46
      2245.23020

      37
      8.5
      46
      2688.12925

      38
      9.0
      46
      3174.44380

      39
      9.5
      46
      3704.17385

```python
hypotheses.plot(x='Slope', y='Sum of Square Errors',
                title=f'Intercept={b}');
```

## Scikit-learn

#### Jake VanderPlas, [Python Data Science Handbook, Chapter 5.2, Introducing Scikit-Learn](https://jakevdp.github.io/PythonDataScienceHandbook/05.02-introducing-scikit-learn.html), Scikit-Learn's Estimator API

> Most commonly, the steps in using the Scikit-Learn estimator API are as follows (we will step through a handful of detailed examples in the sections that follow).

> 1. Choose a class of model by importing the appropriate estimator class from Scikit-Learn.
> 2. Choose model hyperparameters by instantiating this class with desired values.
> 3. Arrange data into a features matrix and target vector following the discussion above.
> 4. Fit the model to your data by calling the `fit()` method of the model instance.
> 5. Apply the Model to new data: For supervised learning, often we predict labels for unknown data using the `predict()` method.

```python
# TODO
```

```python
from sklearn.linear_model import LinearRegression

model = LinearRegression()
features = ['Avg_Rec_Growth']
target = 'Inc_Party_Vote'

x = df[features]
y = df[target]

model.fit(x, y)
```

    LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,
             normalize=False)

```python
model.intercept_
```

    46.499209757741625

```python
model.coef_
```

    array([2.97417709])

```python
model.predict([[1]])
```

    array([49.47338685])

```python
model.predict([[4]])
```

    array([58.39591811])

```python
model.predict([[2]]) - model.predict([[1]])
```

    array([2.97417709])

```python
x
```

      Avg_Rec_Growth

      0
      2.40

      1
      2.89

      2
      0.85

      3
      4.21

      4
      3.02

      5
      3.62

      6
      1.08

      7
      -0.39

      8
      3.86

      9
      2.27

      10
      0.38

      11
      1.04

      12
      2.36

      13
      1.72

      14
      0.10

      15
      0.95

      16
      0.10

```python
m = model.coef_[0]
b = model.intercept_

y_pred = m*x + b
plot_preds(x,y,y_pred)
```

```python
y_pred = model.predict(x)
plot_preds(x,y, y_pred)
```

## Linear Algebra!

The same result that is found by minimizing the sum of the squared errors can be also found through a linear algebra process known as the "Least Squares Solution:"

\begin{align}
\hat{\beta} = (X^{T}X)^{-1}X^{T}y
\end{align}

Before we can work with this equation in its linear algebra form we have to understand how to set up the matrices that are involved in this equation.

### The $\beta$ vector

The $\beta$ vector represents all the parameters that we are trying to estimate, our $y$ vector and $X$ matrix values are full of data from our dataset. The $\beta$ vector holds the variables that we are solving for: $\beta_0$ and $\beta_1$

Now that we have all of the necessary parts we can set them up in the following equation:

\begin{align}
y = X \beta + \epsilon
\end{align}

Since our $\epsilon$ value represents **random** error we can assume that it will equal zero on average.

\begin{align}
y = X \beta
\end{align}

The objective now is to isolate the $\beta$ matrix. We can do this by pre-multiplying both sides by "X transpose" $X^{T}$.

\begin{align}
X^{T}y =  X^{T}X \beta
\end{align}

Since anything times its transpose will result in a square matrix, if that matrix is then an invertible matrix, then we should be able to multiply both sides by its inverse to remove it from the right hand side. (We'll talk tomorrow about situations that could lead to $X^{T}X$ not being invertible.)

\begin{align}
(X^{T}X)^{-1}X^{T}y =  (X^{T}X)^{-1}X^{T}X \beta
\end{align}

Since any matrix multiplied by its inverse results in the identity matrix, and anything multiplied by the identity matrix is itself, we are left with only $\beta$ on the right hand side:

\begin{align}
(X^{T}X)^{-1}X^{T}y = \hat{\beta}
\end{align}

We will now call it "beta hat" $\hat{\beta}$ because it now represents our estimated values for $\beta_0$ and $\beta_1$

### Lets calculate our $\beta$ coefficients with numpy!

```python
from statsmodels.api import add_constant

X = add_constant(df[feature].values)
print('X')
print(X)

y = df[target].values[:, np.newaxis]
print('y')
print(y)

X_transpose = X.T
print('X Transpose')
print(X_transpose)

X_transpose_X = X_transpose @ X
print('X Transpose X')
print(X_transpose_X)

X_transpose_X_inverse = np.linalg.inv(X_transpose_X)
print('X Transpose X Inverse')
print(X_transpose_X_inverse)

X_transpose_y = X_transpose @ y
print('X Transpose y')
print(X_transpose_y)

beta_hat = X_transpose_X_inverse @ X_transpose_y
print('Beta Hat')
print(beta_hat)
```

    X
    [[ 1.    2.4 ]
     [ 1.    2.89]
     [ 1.    0.85]
     [ 1.    4.21]
     [ 1.    3.02]
     [ 1.    3.62]
     [ 1.    1.08]
     [ 1.   -0.39]
     [ 1.    3.86]
     [ 1.    2.27]
     [ 1.    0.38]
     [ 1.    1.04]
     [ 1.    2.36]
     [ 1.    1.72]
     [ 1.    0.1 ]
     [ 1.    0.95]
     [ 1.    0.1 ]]
    y
    [[44.6 ]
     [57.76]
     [49.91]
     [61.34]
     [49.6 ]
     [61.79]
     [48.95]
     [44.7 ]
     [59.17]
     [53.94]
     [46.55]
     [54.74]
     [50.27]
     [51.24]
     [46.32]
     [52.  ]
     [48.2 ]]
    X Transpose
    [[ 1.    1.    1.    1.    1.    1.    1.    1.    1.    1.    1.    1.
       1.    1.    1.    1.    1.  ]
     [ 2.4   2.89  0.85  4.21  3.02  3.62  1.08 -0.39  3.86  2.27  0.38  1.04
       2.36  1.72  0.1   0.95  0.1 ]]
    X Transpose X
    [[17.    30.46 ]
     [30.46  86.831]]
    X Transpose X Inverse
    [[ 0.15835959 -0.05555197]
     [-0.05555197  0.03100405]]
    X Transpose y
    [[ 881.08  ]
     [1674.6167]]
    Beta Hat
    [[46.49920976]
     [ 2.97417709]]

# Multiple Regression

Simple or bivariate linear regression involves a single $x$ variable and a single $y$ variable. However, we can have many $x$ variables. A linear regression model that involves multiple x variables is known as **Multiple** Regression (NOT MULTIVARIATE).

\begin{align}
y = \beta_0 + \beta_1 X_1 + \beta_2 X_2 + ... + \beta_n X_n + \epsilon
\end{align}

```python
# TODO

model = LinearRegression()
features = [
    'Avg_Rec_Growth',
    'fatal_per_mil',
]

taget = 'Inc_Party_Vote'

x = df[features]
y = df[target]

model.fit(x,y)
```

    LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,
             normalize=False)

```python
pd.Series(model.coef_, features)
```

    Avg_Rec_Growth    3.406214
    fatal_per_mil    -0.053752
    dtype: float64

## Visualize hyperplane of best fit in 3D

```python
!pip install plotly
```

    Requirement already satisfied: plotly in c:\programdata\anaconda3\lib\site-packages (3.10.0)
    Requirement already satisfied: six in c:\programdata\anaconda3\lib\site-packages (from plotly) (1.12.0)
    Requirement already satisfied: requests in c:\programdata\anaconda3\lib\site-packages (from plotly) (2.21.0)
    Requirement already satisfied: pytz in c:\programdata\anaconda3\lib\site-packages (from plotly) (2018.9)
    Requirement already satisfied: retrying>=1.3.3 in c:\programdata\anaconda3\lib\site-packages (from plotly) (1.3.3)
    Requirement already satisfied: nbformat>=4.2 in c:\programdata\anaconda3\lib\site-packages (from plotly) (4.4.0)
    Requirement already satisfied: decorator>=4.0.6 in c:\programdata\anaconda3\lib\site-packages (from plotly) (4.4.0)
    Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\programdata\anaconda3\lib\site-packages (from requests->plotly) (3.0.4)
    Requirement already satisfied: certifi>=2017.4.17 in c:\programdata\anaconda3\lib\site-packages (from requests->plotly) (2019.3.9)
    Requirement already satisfied: urllib3<1.25,>=1.21.1 in c:\programdata\anaconda3\lib\site-packages (from requests->plotly) (1.24.1)
    Requirement already satisfied: idna<2.9,>=2.5 in c:\programdata\anaconda3\lib\site-packages (from requests->plotly) (2.8)
    Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in c:\programdata\anaconda3\lib\site-packages (from nbformat>=4.2->plotly) (3.0.1)
    Requirement already satisfied: ipython-genutils in c:\programdata\anaconda3\lib\site-packages (from nbformat>=4.2->plotly) (0.2.0)
    Requirement already satisfied: traitlets>=4.1 in c:\programdata\anaconda3\lib\site-packages (from nbformat>=4.2->plotly) (4.3.2)
    Requirement already satisfied: jupyter-core in c:\programdata\anaconda3\lib\site-packages (from nbformat>=4.2->plotly) (4.4.0)
    Requirement already satisfied: attrs>=17.4.0 in c:\programdata\anaconda3\lib\site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2->plotly) (19.1.0)
    Requirement already satisfied: pyrsistent>=0.14.0 in c:\programdata\anaconda3\lib\site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2->plotly) (0.14.11)
    Requirement already satisfied: setuptools in c:\programdata\anaconda3\lib\site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2->plotly) (40.8.0)

```python
# https://stackoverflow.com/a/47230966
# Plotly notebook mode with google colaboratory
# You need to define this function
# And call it in each offline plotting cell

def configure_plotly_browser_state():
    import IPython
    display(IPython.core.display.HTML('''

        '''))
```

```python
import itertools
import plotly.graph_objs as go
from plotly.offline import init_notebook_mode, iplot
init_notebook_mode(connected=True)

def viz3D(fitted_model, X, features, target='', num=100):
    """
    Visualize model predictions in 3D, for regression model fit on 2 features

    Parameters
    ----------
    fitted_model : scikit-learn model, already fitted
    X : pandas dataframe, which was used to fit model
    features : list of strings, column names of the 2 features
    target : string, name of target
    num : int, number of grid points for each feature

    References
    ----------
    https://plot.ly/python/3d-charts/
    """
    feature1, feature2 = features
    min1, max1 = X[feature1].min(), X[feature1].max()
    min2, max2 = X[feature2].min(), X[feature2].max()
    x1 = np.linspace(min1, max1, num)
    x2 = np.linspace(min2, max2, num)
    combos = list(itertools.product(x1, x2))
    Z = fitted_model.predict(combos).reshape(num, num)

    configure_plotly_browser_state()
    data = [go.Surface(x=x1, y=x2, z=Z)]
    layout = go.Layout(
        scene={'xaxis': {'title': feature1, 'range': [min1,max1], 'showticklabels': True},
               'yaxis': {'title': feature2, 'range': [min2,max2], 'showticklabels': True},
               'zaxis': {'title': target, 'showticklabels': True}},
    )
    fig = go.Figure(data=data, layout=layout)
    iplot(fig)
```

```python
# TODO

viz3D(model, x, features, target)
```

## Dimensionality in Linear Regression

Muliple Regression is simply an extension of the bivariate case. The reason why we see the bivariate case demonstrated so often is simply because it's easier to graph and all of the intuition from the bivariate case is the same as we keep on adding explanatory variables.

As we increase the number of $x$ values in our model we are simply fitting a n-1-dimensional plane to an n-dimensional cloud of points within an n-dimensional hypercube.

## Interpreting Coefficients

One of Linear Regression's strengths is that the parameters of the model (coefficients) are readily interpretable and useful. Not only do they describe the relationship between x and y but they put a number on just how much x is associated with y. We should be careful to not speak about this relationshiop in terms of causality because these coefficients are in fact correlative measures. We would need a host of additional techniques in order to estimate a causal effect using linear regression (econometrics).

\begin{align}
\hat{\beta} = \frac{Cov(x,y)}{Var(y)}
\end{align}

## Why is Linear Regression so Important?

### Popularity

Linear Regression is an extremely popular technique that every data scientist **needs** to understand. It's not the most advanced technique and there are supervised learning techniques that will obtain a higher accuracy, but where it lacks in accuracy it makes up for it in interpretability and simplicity.

### Interpretability

Few other models possess coefficients that are so directly linked to their variables with a such a clear interpretation. Tomorrow we're going to learn about ways to make them even easier to interpret.

### Simplicity

A linear regression model can be communicated just by writing out its equation. It's kind of incredible that such high dimensional relationships can be described from just a linear combination of variables and coefficients.

# Assignment
- Continue to predict New York City apartment rents. This is your last assignment with this dataset.
- You may select any number of features. You are encouraged to engineer new features.
- Get and plot your model's coefficients.
- Report your Root Mean Squared Error, Mean Absolute Error, and R^2 Score, for your Train and Test sets. Share your scores with your cohort on Slack!
- Fit a model with 2 features, and visualize the plane of best fit in 3D.
- Commit your notebook to your fork of the repo.

## Stretch Goals

Study more about Linear Regression. Here are two helpful links. If you find more links, share your favorites with your cohort on Slack.

1. Watch this 20 minute video that just hit 1 million views: Brandon Foltz, Statistics 101: Simple Linear Regression (https://www.youtube.com/watch?v=ZkjP5RJLQF4)
2. Skim _An Introduction to Statistical Learning_, Chapter 3.1, Simple Linear Regression, & Chapter 3.2, Multiple Linear Regression (http://www-bcf.usc.edu/~gareth/ISL/ISLR%20Seventh%20Printing.pdf)

In your 3D visualization, can you include the actual datapoints, like in [this notebook](https://nbviewer.jupyter.org/urls/s3.amazonaws.com/datarobotblog/notebooks/multiple_regression_in_python.ipynb)? Can you also include the residual lines from the datapoints to the plane of the best fit, like in _An Introduction to Statistical Learning?_ This would be hard to do, but awesome!

Can you get creative with feature engineering? Share with your cohort on Slack. We mentioned some feature ideas at the end of last lesson, but didn't demonstrate how to engineer them. So here are some example solutions:

```python
# Does apartment have a non-empty description?
df['description'] = df['description'].str.strip().fillna('')
df['has_description'] = df['description'] != ''

# How long is the description?
df['description_length'] = df['description'].str.len()

# How many total perks does each apartment have?
perk_cols = ['elevator', 'cats_allowed', 'hardwood_floors', 'dogs_allowed',
             'doorman', 'dishwasher', 'no_fee', 'laundry_in_building',
             'fitness_center', 'pre-war', 'laundry_in_unit', 'roof_deck',
             'outdoor_space', 'dining_room', 'high_speed_internet', 'balcony',
             'swimming_pool', 'new_construction', 'exclusive', 'terrace',
             'loft', 'garden_patio', 'common_outdoor_space',
             'wheelchair_access']
df['perk_count'] = df[perk_cols].sum(axis=1)

# Are pets allowed?
df['pets_allowed'] = (df['cats_allowed']==1) | (df['dogs_allowed']==1)
```


```python
import hublib
import pandas as pd
import ipywidgets as widgets
from IPython.display import clear_output
import matplotlib
%matplotlib notebook
import matplotlib.pyplot as plt
import scipy
from scipy import signal
import numpy as np
from decimal import Decimal
import unicodedata
from unicodedata import lookup as GL
import sympy as sy
from joblib import Parallel, delayed
from lmfit import Model
import warnings
import zipfile
from zipfile import ZipFile
import os
from hublib.ui import FileUpload, Download
from scipy import sparse
from scipy.sparse.linalg import spsolve
```

```python
class Spectrum:
    def __init__(self):
        self.x=0
        self.y=0
        self.I=[]
        self.W=[]
        self.If=[]
        self.PG=[]
        self.PGp=[]
        self.PD=[]
        self.IDfit=[]
        self.Q=0
        self.diffs=[]
        self.mdi=0
        self.md=0
        self.mf=[]
```

```python
global Specs
Specs=[]

global filelist
filelist = []

global cfl
```

```python
def mycb(w,fnames):
    global fnm
    fnm=fnames[0]
    fbase = os.path.basename(fnm)
    os.makedirs('data/' + os.path.splitext(fbase)[0])
    filelist.append(fbase)
    os.rename(fnm, 'data/raw/' + fbase)
    w.reset()

f = FileUpload("Please upload Raman spectra data file (CSV)",
               "Raman data files should be uploaded as 2 column CSV files",
               cb=mycb,
               maxsize=10000000)
f
```

```python
def errprint(code):
    errfile=pd.read_csv('errfile.txt',sep='\t',header=None)
    with errout:
        clear_output()
        print(errfile[0][code])
        fit_but.disabled=False
        errout
```

```python
def case_lookup(index):
    casefile=pd.read_csv('Case_List.txt',sep='\t',header=None)
    c=casefile[0][index]
    return c
```

```python
fit_but = widgets.Button(description='Do Fitting')

def fit_but_cb(change):
    global cfl
    fit_but.disabled=True
    param.disabled=True
    with plist:
        clear_output()
        print('Reading data files...')
    with errout:
        clear_output()
    with diffsplot:
        clear_output()
    with datplot:
        clear_output()

    for flnm in filelist:
        cfl = flnm
        if flnm[-3:]=='txt':
            sp='\s+'
        elif flnm[-3:]=='csv':
            sp=','
        else:
            errprint(0)
            return
        try:
            data = pd.read_csv('data/raw/' + flnm,sep=sp,header=None)
        except:
            sp='\t'
            data = pd.read_csv('data/raw/' + flnm,sep=sp,header=None)
        with plist:
            clear_output()
            print('Data file read')

        n=int(data.size/len(data)) #n determines the size of the data file

        global Specs
        Specs.clear()

        ##Single Spectra Data File, n=2
        if n==2:
            with plist:
                clear_output()
                print('Fitting single spectra data.')

            s=Spectrum()

            Spectra(s,data)
            Fit(s)

            dtplot(s)

            with diffsplot:
                clear_output()
                fig=plt.figure(figsize=(4,4))
                ax=fig.add_subplot(111)
                plt.plot(s.diffs,'kv')
                plt.plot(s.mdi,s.md,'gv')
                plt.annotate((round(Decimal(s.md),2)),xy=(s.mdi,1.2*s.md))
                plt.xticks(range(6),('1','2','3','4','5','Graphite'))
                plt.xlabel('# Layers')
                plt.ylabel('$\Delta$ [%]')
                plt.show()

            save_spec(s)
            zip_files('data')
            params_print(s)

        #Map files will be much larger than 2 points and need separate handling
        elif n > 2:
            Specs=[]
            Map(data)
        else:
            errprint(1)
            return
    fit_but.disabled=False

fit_but.on_click(fit_but_cb)
fit_but
```

```python
def Map(data):
    W=data[:][0:1]
    W=np.array(W)
    W=W[~np.isnan(W)]

    x=data[0]
    x=np.array(x)
    x=x[~np.isnan(x)]
    xu=np.unique(x)

    y=data[1]
    y=np.array(y)
    y=y[~np.isnan(y)]
    yu=np.unique(y)

    n=yu.size*xu.size

    s=Spectrum()

    Parallel(n_jobs=1)(delayed(maploop)(s,Specs,data,W,x,y,n,k) for k in range(n))

    wG=np.transpose(np.array([o.PG for o in Specs]))[2]

    Mplot(x,y,wG,'$\omega_G$ $[cm^{-1}]$','wG')
    Hplot(wG,'$\omega_G$ $[cm^{-1}]$','wG')

    with plist:
        clear_output()
        print('Fitting Finished')
    save_map(Specs)
    zip_files('data')
    with plist:
        clear_output()
    param.disabled=False
```

```python
def maploop(s,Specs,data,W,x,y,n,k):
    s=Spectrum()

    I_raw=np.array(data)[k+1][2:1026]
    tmp_min = np.min(I_raw)
    I_tmp = I_raw-tmp_min
    tmp_max = np.max(I_tmp)
    I=I_tmp/tmp_max
    #I=((I_raw-np.min(I_raw))/np.max(I_raw-np.min(I_raw)))
    s.I=I
    s.W=W
    s.x=x[k]
    s.y=y[k]
    Fit(s)
    Specs.append(s)

    pdone=100*(k+1)/n

    with plist:
        clear_output()
        print('Fitting map data. This may take some time...\n%1.2f%% Done'%(pdone))

    return Specs
```

```python
def Spectra(s,data):
    srow=0;
    if type(data[0][0])==str:
        srow=1

    W=data[0][srow:len(data)]
    W=np.array(W);W=W.astype(float)
    I_raw=data[1][srow:len(data)]
    I_raw=np.array(I_raw);I_raw=I_raw.astype(float)

    x = I_raw
    window_length = 7
    polyorder = 2

    I_raw = scipy.signal.savgol_filter(x, window_length, polyorder, deriv=0, delta=1.0, axis=- 1, mode='interp', cval=0.0)

# ---- Replace this part ---- #
    n=2

    polyx = np.array([W[0],W[int(len(W)/2)],W[len(W)-1]])
    polyy = np.array([I_raw[0],I_raw[int(len(W)/2)],I_raw[len(W)-1]])
    bkgfit = np.polyfit(W,I_raw,n)
    bkgpoly = 0
    for i in range(n):
        bkgpoly = bkgpoly + (bkgfit[i]*W**(n-i))
    I_raw = I_raw-bkgpoly

    m = (I_raw[len(W)-1]-I_raw[0])/(W[len(W)-1]-W[0])
    b = I_raw[len(W)-1]-m*W[len(W)-1]
    bkglin = m*W+b

    I_raw=I_raw-bkglin

    I=((I_raw-np.min(I_raw))/np.max(I_raw-np.min(I_raw)));

# ---- Upto here ---- #

    s.I=I

    s.W=W
    return s
```

```python
# scipy.signal.savgol_filter(x, window_length, polyorder, deriv=0, delta=1.0, axis=- 1, mode='interp', cval=0.0)
#x is the data to be filtered
#If x is not a single or double precision floating point array, it will be converted to type numpy.float64 before filtering
#If x has dimension greater than 1, axis determines the axis along which the filter is applied
#window_length must be a positive odd number; short paper says its the spectral channel; this will be equal to x_2 - x_1
#If mode is ‘interp’, window_length must be less than or equal to the size of x
#polyorder must be less than window_length
#delta is only used is deriv is greater than 0
```

```python
laser=np.random.randint(400,800)
print("laser: ", laser)
```

```python
def laser_wavelength(lam):
    peak=107.36*(1.24*10**3/lam)+2427.5
    print("peak: ", peak)
    return peak

def Fit(s):
    W=s.W
    I=s.I
    pG=[1.1*np.max(I), 50, 1581.6] #a w b
    pGp=[1.1*np.max(I), 50, laser_wavelength(laser)]

    L.set_bounds([1.5*np.max(I),60,2000],[0.3*np.max(I),33,1400])
    PGm=L.fit(s.W,s.I,pG)

    L.set_bounds([1.5*np.max(I),60,3000],[0.3*np.max(I),32,2000])
    PGpm=L.fit(s.W,s.I,pGp)

    PG=np.array(list(PGm.best_values.values()))
    PGp=np.array(list(PGpm.best_values.values()))

    PG[1]=np.absolute(PG[1]);PGp[1]=np.absolute(PGp[1]); #FWHM sometimes returns - bc always squared

    IGfit=Single_Lorentz(W,PG[0],PG[1],PG[2]);
    IGpfit=Single_Lorentz(W,PGp[0],PGp[1],PGp[2]);
    IGfit=IGfit
    IGpfit=IGpfit
    s.If=IGfit+IGpfit;

    s.PG=PG
    s.PGp=PGp

    pD=[0.1*np.max(I),5,1350]

    L.set_bounds([np.max(I),50,1400],[0,10,1300])
    PDm=L.fit(s.W,s.I,pD)
    PD=np.array(list(PDm.best_values.values()))
    PD[1]=np.absolute(PD[1]);
    IDfit=Single_Lorentz(W,PD[0],PD[1],PD[2]);
    s.IDfit=IDfit
    Q=1-(PD[0]/PG[0])
    s.Q=Decimal(Q)
    s.PD=PD

    Cdat=np.load('Cfits.npy')

    diffs_lin=[];diffs_Gp=[];
    diffs=[];diffs.clear()
    for d in range(6):

        LG=Cdat[d][0];LG=np.transpose(LG)[0]
        LGp=Cdat[d][1];LGp=np.transpose(LGp)[0]
        LGfit=Single_Lorentz(W,LG[0],LG[1],LG[2]);
        LGpfit=Single_Lorentz(W,LGp[0],LGp[1],LGp[2]);
        Lf=(LGfit+LGpfit)

        wts=[1.,1.,0.5]

        dfGp=np.average(np.absolute(100*(PGp-LGp)/LGp),weights=wts)
        dfG=np.average(np.absolute(100*(PG-LG)/LG),weights=wts)
        drat=np.absolute(100*(((PG[0]/PGp[0])-(LG[0]/LGp[0]))/(LG[0]/LGp[0])))
        df=np.average([dfGp,dfG,drat],weights=[0.5,0.5,1])
        diffs.append(df)

    s.diffs=diffs
    md=np.min(diffs)
    mdi=np.argmin(diffs)

    mG=Cdat[mdi][0];mGp=Cdat[mdi][1];
    mGfit=Single_Lorentz(W,mG[0],mG[1],mG[2]);
    mGpfit=Single_Lorentz(W,mGp[0],mGp[1],mGp[2]);
    mf=mGfit+mGpfit
    s.mf=mf
    s.md=md
    s.mdi=mdi
    return s
```

```python
def Single_Lorentz(x,a,w,b):
    return a*(((w/2)**2)/(((x-b)**2)+((w/2)**2)))
```

```python
def dtplot(s):
        with datplot:
            clear_output()
            fig1=plt.figure(figsize=(4,4))
            ax=fig1.add_subplot(111)
            plt.plot(s.W,s.I,'b',s.W,s.mf,'g',s.W,s.If+s.IDfit,'r')
            plt.xlabel('$\omega$ $[cm^{-1}]$')
            plt.ylabel('$I_{norm}$ [arb]')
            plt.legend(labels=['Raw','Test','Fit'])
            plt.annotate('Q=%1.2f' %round(s.Q,2) ,xy=(np.min(s.W),0.98))
            plt.annotate('D',xy=(0.85*s.PD[2],1.1*s.PD[0]))
            plt.annotate('G',xy=(0.9*s.PG[2],0.95*s.PG[0]))
            plt.annotate('G\'',xy=(0.94*s.PGp[2],0.95*s.PGp[0]))
            plt.show()
            plt.savefig('data/'+cfl[:-4]+'/specs.png')
```

```python
def dfplot(s):
    with diffsplot:
        clear_output()
        fig=plt.figure(figsize=(4,4))
        ax=fig.add_subplot(111)
        plt.plot(s.W,s.I,'b',s.W,s.mf,'g',s.W,s.If+s.IDfit,'r')
        plt.xlabel('$\omega$ $[cm^{-1}]$')
        plt.ylabel('$I_{norm}$ [arb]')
        plt.legend(labels=['Raw','Test','Fit'])
        plt.annotate('Q=%1.2f' %round(s.Q,2) ,xy=(np.min(s.W),0.98))
        plt.annotate('D',xy=(0.85*s.PD[2],1.1*s.PD[0]))
        plt.annotate('G',xy=(0.9*s.PG[2],0.95*s.PG[0]))
        plt.annotate('G\'',xy=(0.94*s.PGp[2],0.95*s.PGp[0]))
        plt.show()
    params_print(s)
```

```python
def Mplot(x,y,z,d,fn):
    global p,point,datax
    xi = np.linspace(min(x), max(x))
    yi = np.linspace(min(y), max(y))
    X, Y = np.meshgrid(xi, yi)

    with datplot:
        clear_output()
        Z=matplotlib.mlab.griddata(x, y, z, xi, yi, interp='linear')
        fig=plt.figure(figsize=(4,4))
        datax=fig.add_subplot(111)
        p,=datax.plot([],[],'o')
        point=pickPeaks(p)
        C=plt.contourf(X,Y,Z)
        plt.set_cmap('inferno')
        plt.xlabel('x [mm]')
        plt.ylabel('y [mm]')
        #datax.set_xlim(np.min(xi),np.max(xi))
        #datax.set_ylim(np.min(yi),np.max(yi))
        plt.title(d)
        plt.colorbar(C)
        plt.axis('off')
        #datax.autoscale
        plt.show()
        figname = 'data/'+ cfl[:-4]+'/' + fn + '_map.png'
        plt.savefig(figname)
        mout = open('data/'+ cfl[:-4]+'/' + fn + '_dat.map','w')
        mout.write(' '.join(str(n) for n in Z))
```

```python
def Hplot(z,d,fn):
    with plist:
        clear_output()
    with diffsplot:
        clear_output()
        fig=plt.figure(figsize=(4,4))
        ax=fig.add_subplot(111)
        plt.hist(z,bins='auto')
        plt.ylabel('Counts')
        plt.xlabel(d)
        plt.show()
        figname = 'data/'+ cfl[:-4]+'/' + fn + '_hist.png'
        plt.savefig(figname)
        hout = open('data/'+ cfl[:-4]+'/' + fn + '_dat.hist','w')
        hout.write(' '.join(str(n) for n in z))
```

```python
def params_print(s):
    with plist:
        clear_output()
        G=GL('GREEK CAPITAL LETTER GAMMA')
        o=GL('GREEK SMALL LETTER OMEGA')
        print('G Fitting Parameters:\n\tA=%1.2f\n\t%s=%1.2f\n\t%s=%1.2f\n'
              'G\' Fitting Parameters:\n\tA=%1.2f\n\t%s=%1.2f\n\t%s=%1.2f\n'
              'D Fitting Parameters:\n\tA=%1.2f\n\t%s=%1.2f\n\t%s=%1.2f\n'
              'Quality=%1.2f (Ratio of D to G)\n'
              'Best Case Match: %s'
              %(s.PG[0],G,s.PG[1],o,s.PG[2],s.PGp[0],G,s.PGp[1],o,s.PGp[2],s.PD[0],G,s.PD[1],o,s.PD[2],s.Q,case_lookup(s.mdi)))
        strain_doping(s.PG[2], s.PGp[2])
```

```python
def strain_doping(x_o, y_o):
    x_0,y_0 = (1581, 2692) #ideal point based on lamba = 532nm, the most common laser in literature
    strain_axis = 2.483*x_0 - 1233.62
    doping_axis = 0.553*x_0 + 1817.71
    x_e,y_e = (x_o, y_o) #fitted point; where do we find the fitted point?
    b = y_e - 2.483*x_e
    c = y_e - 0.553*x_e
    exp_strain_axis = 2.483*x_e + b
    exp_doping_axis = 0.553*x_e + c
    x_s = (c+1233.62)/1.93
    x_d = -1*(b-1817.71)/1.93
    y_s = 2.483*x_s - 1233.62
    y_d = 0.553*x_d + 1817.71
    #(x_s, y_s) = intersection of strain axis and experimental doping axis
    #(x_d, y_d) = intersection of doping axis and experimental strain axis
    strain_distance = ((x_s - x_0)**2 + (y_s - y_0)**2)**0.5
    doping_distance = ((x_d - x_0)**2 + (y_d - y_0)**2)**0.5
    strain_unit = 155.242
    doping_unit = 1.611
    strain_present = strain_distance / strain_unit
    doping_present = doping_distance / doping_unit
    print("Strain (%): ", strain_present)
    print("Doping (n): ", doping_present)
```

```python
datplot=widgets.Output();datplot
```

```python
diffsplot=widgets.Output();diffsplot
```

```python
errout=widgets.Output();errout
```

```python
plist=widgets.Output();plist
```

```python
o=GL('GREEK SMALL LETTER OMEGA')
G=GL('GREEK CAPITAL LETTER GAMMA')

def param_change(change):
    d = ['$I_G$ [arb]','$\Gamma_G$ $[cm^{-1}]$','$\omega_G$ $[cm^{-1}]$','$I_{G\'}$ [arb]','$\Gamma_{G\'}$ $[cm^{-1}]$','$\omega_{G\'}$ $[cm^{-1}]$','$I_D$ [arb]','$\Gamma_D$ $[cm^{-1}]$','$\omega_D$ $[cm^{-1}]$']
    D = ['IG','gG','wG','IGp','gGp','wGp','ID','gD','wD']
    N = d[param.value-1]
    n = D[param.value-1]
    if param.value in [1,4,7]:
        ind=0
    elif param.value in [2,5,8]:
        ind=1
    elif param.value in [3,6,9]:
        ind=2
    else:
        ind=0

    if param.value in [1,2,3]:
        z=np.transpose(np.array([o.PG for o in Specs]))[ind]
    elif param.value in [4,5,6]:
        z=np.transpose(np.array([o.PGp for o in Specs]))[ind]
    elif param.value in [7,8,9]:
        z=np.transpose(np.array([o.PD for o in Specs]))[ind]
    else:
        z=np.transpose(np.array([o.PG for o in Specs]))[2]
        N=d[2]
    xvals=np.transpose(np.array([o.x for o in Specs]))
    yvals=np.transpose(np.array([o.y for o in Specs]))
    Mplot(xvals,yvals,z,N,n)
    Hplot(z,N,n)

param=widgets.Dropdown(description='Parameter')
param.options=options={'Select': 0, 'I_G': 1, (G+'_G'): 2, (o+'_G'): 3, 'I_G\'': 4, (G+'_G\''): 5, (o+'_G\''): 6, 'I_D': 7, (G+'_D'): 8, (o+'_D'): 9}
param.observe(param_change,names='value')
param.disabled=True
param
```

```python
class pickPeaks:
    def __init__(self, line):
        self.line = line
        self.xs = line.get_xdata()
        self.ys = line.get_ydata()
        self.cid = line.figure.canvas.mpl_connect('button_press_event', self)

    def __call__(self, event):
        print('click', event)
        if event.inaxes!=self.line.axes: return
        self.xs=event.xdata
        self.ys=event.ydata

        Map_Spec_Plot()
        self.points.set_data(self.xs, self.ys)

    def __iter__(self):
        return zip(self.xs, self.ys)
```

```python
def Map_Spec_Plot():
    from scipy.spatial import cKDTree
    xvals=np.transpose(np.array([o.x for o in Specs]))
    yvals=np.transpose(np.array([o.y for o in Specs]))
    XY=np.zeros((len(Specs),2))
    XY[:,0]=xvals
    XY[:,1]=yvals
    tree = cKDTree(XY)
    dis, ind = tree.query([point.xs,point.ys], k=1)
    dfplot(Specs[ind])
```

```python
class Lorentz:
    def __init__(self):
        self.model=Model(Single_Lorentz)
    def set_bounds(self,ub,lb):
        self.model.set_param_hint('a',min=lb[0],max=ub[0])
        self.model.set_param_hint('w',min=lb[1],max=ub[1])
        self.model.set_param_hint('b',min=lb[2],max=ub[2])
    def fit(self,x,y,params):
        F=self.model.fit(data=y,x=x,a=params[0],w=params[1],b=params[2])
        return F
```

```python
L=Lorentz()
```

```python
def save_spec(s):
    with plist:
        clear_output()
        print('Generating plots\nThis may take a few seconds...')
    fout = open('data/'+cfl[:-4]+'/out.graft','w')
    fout.write(' '.join(str(n) for n in s.I)+'\n')
    fout.write(' '.join(str(n) for n in s.W)+'\n')
    fout.write(' '.join(str(n) for n in s.If)+'\n')
    fout.write(' '.join(str(n) for n in s.PG)+'\n')
    fout.write(' '.join(str(n) for n in s.PGp)+'\n')
    fout.write(' '.join(str(n) for n in s.PD)+'\n')
    fout.write(' '.join(str(n) for n in s.IDfit)+'\n')
    fout.write(str(s.Q)+'\n')
    fout.write(' '.join(str(n) for n in s.diffs)+'\n')
    fout.write(str(s.mdi)+'\n')
    fout.write(str(s.md)+'\n')
    fout.write(' '.join(str(n) for n in s.mf)+'\n')
    fout.close()

    fitfile = open('data/spec_fits.csv','a')
    fitfile.write(cfl[:-4]+',' + str(s.PD[2])+','+str(s.PD[0])+','+str(s.PD[1])+',' + str(s.PG[2])+','+str(s.PG[0])+','+str(s.PG[1])+',' + str(s.PGp[2])+','+str(s.PGp[0])+','+str(s.PGp[1])+'\n')
    fitfile.close()
```

```python
def save_map(Specs):
    with plist:
        clear_output()
        print('Generating plots\nThis may take a few seconds...')
    ## save all the data to a big output file that can be read in later
    fout = open('data/'+cfl[:-4]+'/out.graft','w')
    for o in Specs:
        fout.write(str(o.x)+'\n')
        fout.write(str(o.y)+'\n')
        fout.write(' '.join(str(n) for n in o.I)+'\n')
        fout.write(' '.join(str(n) for n in o.W)+'\n')
        fout.write(' '.join(str(n) for n in o.If)+'\n')
        fout.write(' '.join(str(n) for n in o.PG)+'\n')
        fout.write(' '.join(str(n) for n in o.PGp)+'\n')
        fout.write(' '.join(str(n) for n in o.PD)+'\n')
        fout.write(' '.join(str(n) for n in o.IDfit)+'\n')
        fout.write(str(o.Q)+'\n')
        fout.write(' '.join(str(n) for n in o.diffs)+'\n')
        fout.write(str(o.mdi)+'\n')
        fout.write(str(o.md)+'\n')
        fout.write(' '.join(str(n) for n in o.mf)+'\n')
    fout.close()

    ##save images to files as images and data files
    for i in [1,2,3,4,5,6,7,8,9]:
        param.value = i
```

```python
def get_all_file_paths(directory):

    # initializing empty file paths list
    file_paths = []

    # crawling through directory and subdirectories
    for root, directories, files in os.walk(directory):
        for filename in files:
            # join the two strings in order to form the full filepath.
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)

    # returning all file paths
    return file_paths

def zip_files(directory):
    # path to folder which needs to be zipped

    # calling function to get all file paths in the directory
    file_paths = get_all_file_paths(directory)

    # writing files to a zipfile
    with ZipFile('data.zip','w') as zip:
        # writing each file one by one
        for file in file_paths:
            zip.write(file, compress_type = zipfile.ZIP_DEFLATED)

def clear_data(directory):
    file_paths = get_all_file_paths(directory)
    for file in file_paths:
        os.remove(file)

    for directory in os.walk('data/'):
        if directory[0] == 'data/' or directory[0] == 'data/raw':
            i=1
        else:
            os.rmdir(directory[0])

```

```python
d = Download('data.zip', label='Download Data', icon='download', tooltip='DOWNLOAD FILE')
d
```

```python
clear_data('data')
```

```python

```

```python
# Testing for PyQt5 Tool
```

```python
# data=pd.read_table("./data/raw/20200630_2_SS.txt")
# cols=data.shape[1]
# rows=data.shape[0]

# if cols == 1:
#     #self.clearPast()
#     data=pd.DataFrame(data.iloc[0:rows/2,0],data.iloc[rows/2:rows,0])
#     #self.setSingle()
# elif cols == 2:
#     if type(data.iloc[0,0]) is str:
#         #self.clearPast()
#         data=data.iloc[1:rows,:]
#         #self.setSingle()
#     else:
#         #clearPast()
#         data=data
#         #setSingle()
# else:
#     del filelist[-1]
#     data = []
#     raise ValueError('Please use a single spectrum only')

# data
```

```python
# frequency = []
# intensity_norm = []
# I_BL = []

# frequency = np.array(data.iloc[:,0])
# intensity = np.array(data.iloc[:,1])
# length = len(frequency)
# a = 0
# for i in range(length):
#     if frequency[i]<=0:
#         a = a+1
#     else:
#         break
# frequency = frequency[a:]
# intensity = intensity[a:]

# # self.intensity_norm = []
# for i in intensity:
#     intensity_norm.append((i-np.min(intensity))/(np.max(intensity)-np.min(intensity)))

# #len(intensity_norm)
```

```python
# y = intensity_norm
# x = frequency

# degree = 3

# n = degree
# I_raw = np.array(y)
# W = np.array(x)

# polyx = np.array([W[0],W[int(len(W)/2)],W[len(W)-1]])
# polyy = np.array([I_raw[0],I_raw[int(len(W)/2)],I_raw[len(W)-1]])
# bkgfit = np.polyfit(W,I_raw,degree)
# bkgpoly = 0
# for i in range(n):
#     bkgpoly = bkgpoly + (bkgfit[i]*W**(n-i))
#     print('bkgpoly:', bkgpoly)
# I_raw = I_raw-bkgpoly

# m = (I_raw[len(W)-1]-I_raw[0])/(W[len(W)-1]-W[0])
# b = I_raw[len(W)-1]-m*W[len(W)-1]
# bkglin = m*W+b

# I_raw = I_raw-bkglin

# I_BL = []
# I_BL = ((I_raw-np.min(I_raw))/np.max(I_raw-np.min(I_raw)))

# print('polyx:', polyx)
# print('polyy:', polyy)
```

```python
# r = np.array([2,3,4])
# a = 5
# b = 3
# a*r**2 + b*r**1
```

```python
# %matplotlib inline
# plt.figure(0)
# plt.plot(frequency, intensity)
# plt.figure(1)
# plt.plot(frequency, intensity_norm)
# plt.figure(2)
# plt.plot(frequency, I_BL)
```

```python
# plt.plot(frequency, I_BL)
```

```python
# fpath = "C:/Users/Test/Documents/UIUC_Offline/Tool/Working/GSA-Raman\data\raw\spectest.csv"

# import os
# fpath = os.path.split(fpath)[-1]
# fpath
```