### 필요 모듈 및 데이터 로드

```python
import sys
import os

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
import itertools
```

```python
dacon_data_path = os.path.join("./data/dacon_data")
sys.path.insert(0, dacon_data_path)
```

```python
case = pd.read_csv("./Case.csv")
p_info = pd.read_csv("./PatientInfo.csv")
p_route = pd.read_csv("./PatientRoute.csv")
pol = pd.read_csv("./Policy.csv")
reg = pd.read_csv("./Region.csv")
st = pd.read_csv("./SearchTrend.csv")
sf = pd.read_csv("./SeoulFloating.csv")
ta = pd.read_csv("./TimeAge.csv")
time = pd.read_csv("./Time.csv")
tg = pd.read_csv("./TimeGender.csv")
tp = pd.read_csv("./TimeProvince.csv")
wt = pd.read_csv("./Weather.csv")
```

# utils

```python
def uniform_pdf(x):
    """ Uniform Distribution(균등 분포) 확률 밀도 함수 """
    return 1 if 0 <= x < 1 else 0
```

```python
def uniform_cdf(x):
    """ 균등 분포 누적 분포 함수 """
    if x < 0:
        return 0
    elif 0 <= x < 1:
        return x
    else:
        return 1
```

```python
def normal_pdf(x, mu=0.0, sigma=1.0):
    """평균이 mu이고, 표준편차가 sigma인 정규 분포(Normal Distribution)
    확률 밀도 함수
    """
    return math.exp(-(x - mu) ** 2 / (2 * sigma ** 2)) / (SQRT_TWO_PI * sigma)
```

```python
def normal_cdf(x, mu=0.0, sigma=1.0):
    """평균이 mu이고, 표준편차가 sigma인
    정규 분포(Normal Distribution)의 누적 분포 함수(Cumulative Distribution Function)
    math.erf() 함수(error function)를 이용해서 구현"""
    return (1 + math.erf((x - mu) / (math.sqrt(2) * sigma))) / 2
```

```python
def inverse_normal_cdf(p, mu=0.0, sigma=1.0, tolerance=0.00001):
    """누적 확률 p를 알고 있을 때 정규 분포 확률 변수 x = ?"""
    # 표준 정규 분포가 아니라면 표준 정규 분포로 변환
    if mu != 0.0 or sigma != 1.0:
        return mu + sigma * inverse_normal_cdf(p, tolerance=tolerance)
    low_z = -10.0 # 하한(lower bound)
    high_z = 10.0 # 상한(upper bound)
    while high_z - low_z > tolerance:
        mid_z = (low_z + high_z) / 2.0 # 중간 값
        mid_p = normal_cdf(mid_z) # 중간 값에서의 누적 확률
        if mid_p < p:
            low_z = mid_z
        else:
            high_z = mid_z
    return mid_z
```

### data EDA

```python
data_list = [case, p_info, p_route, pol, reg, st, sf, ta, time, tg, tp, wt]
data_str_list = ['case', 'p_info', 'p_route', 'pol', 'reg', 'st', 'sf', 'ta', 'time', 'tg', 'tp', 'wt']
```

```python
def data_shape_print(d_list, d_str_list):

    col_list = []

    for d_idx in range(len(d_list)):
        print(str(d_idx+1),'_',d_str_list[d_idx], ': ', d_list[d_idx].shape)
        print('columns : ', list(d_list[d_idx].columns))
        print()
        for c_idx in list(d_list[d_idx].columns):
            col_list.append(c_idx)

    return col_list

col_list = data_shape_print(data_list, data_str_list)
```

    1 _ case :  (112, 8)
    columns :  ['case_id', 'province', 'city', 'group', 'infection_case', 'confirmed', 'latitude', 'longitude']

    2 _ p_info :  (3388, 18)
    columns :  ['patient_id', 'global_num', 'sex', 'birth_year', 'age', 'country', 'province', 'city', 'disease', 'infection_case', 'infection_order', 'infected_by', 'contact_number', 'symptom_onset_date', 'confirmed_date', 'released_date', 'deceased_date', 'state']

    3 _ p_route :  (6714, 8)
    columns :  ['patient_id', 'global_num', 'date', 'province', 'city', 'type', 'latitude', 'longitude']

    4 _ pol :  (48, 7)
    columns :  ['policy_id', 'country', 'type', 'gov_policy', 'detail', 'start_date', 'end_date']

    5 _ reg :  (244, 12)
    columns :  ['code', 'province', 'city', 'latitude', 'longitude', 'elementary_school_count', 'kindergarten_count', 'university_count', 'academy_ratio', 'elderly_population_ratio', 'elderly_alone_ratio', 'nursing_home_count']

    6 _ st :  (1581, 5)
    columns :  ['date', 'cold', 'flu', 'pneumonia', 'coronavirus']

    7 _ sf :  (432000, 7)
    columns :  ['date', 'hour', 'birth_year', 'sex', 'province', 'city', 'fp_num']

    8 _ ta :  (540, 5)
    columns :  ['date', 'time', 'age', 'confirmed', 'deceased']

    9 _ time :  (102, 7)
    columns :  ['date', 'time', 'test', 'negative', 'confirmed', 'released', 'deceased']

    10 _ tg :  (120, 5)
    columns :  ['date', 'time', 'sex', 'confirmed', 'deceased']

    11 _ tp :  (1734, 6)
    columns :  ['date', 'time', 'province', 'confirmed', 'released', 'deceased']

    12 _ wt :  (25295, 10)
    columns :  ['code', 'province', 'date', 'avg_temp', 'min_temp', 'max_temp', 'precipitation', 'max_wind_speed', 'most_wind_direction', 'avg_relative_humidity']

```python
from collections import Counter

col_d = dict(sorted(dict(Counter(col_list)).items(), key=lambda item: item[1]))
col_d
```

    {'case_id': 1,
     'group': 1,
     'disease': 1,
     'infection_order': 1,
     'infected_by': 1,
     'contact_number': 1,
     'symptom_onset_date': 1,
     'confirmed_date': 1,
     'released_date': 1,
     'deceased_date': 1,
     'state': 1,
     'policy_id': 1,
     'gov_policy': 1,
     'detail': 1,
     'start_date': 1,
     'end_date': 1,
     'elementary_school_count': 1,
     'kindergarten_count': 1,
     'university_count': 1,
     'academy_ratio': 1,
     'elderly_population_ratio': 1,
     'elderly_alone_ratio': 1,
     'nursing_home_count': 1,
     'cold': 1,
     'flu': 1,
     'pneumonia': 1,
     'coronavirus': 1,
     'hour': 1,
     'fp_num': 1,
     'test': 1,
     'negative': 1,
     'avg_temp': 1,
     'min_temp': 1,
     'max_temp': 1,
     'precipitation': 1,
     'max_wind_speed': 1,
     'most_wind_direction': 1,
     'avg_relative_humidity': 1,
     'infection_case': 2,
     'patient_id': 2,
     'global_num': 2,
     'birth_year': 2,
     'age': 2,
     'country': 2,
     'type': 2,
     'code': 2,
     'released': 2,
     'latitude': 3,
     'longitude': 3,
     'sex': 3,
     'time': 4,
     'deceased': 4,
     'city': 5,
     'confirmed': 5,
     'province': 7,
     'date': 8}

### 사용할 데이터 추출

- 그룹화 할 수 있는 데이터(mapper?)
    + date
    + location(city, province)

```python
# 확진자 중 기저질환 보유 여부
_per = float(list(p_info.disease.value_counts())[0]/len(p_info))
print("percentge : %s%%"%round(_per*100,2))
```

    percentge : 0.53%

```python
# util
def group_cals_func(df, group_col, target_col, agg):

    grouped = df.groupby(group_col)

    if agg == 'mean':
        return grouped[target_col].mean().sort_values(ascending=False)
    elif agg == 'sum':
        return grouped[target_col].sum().sort_values(ascending=False)
```

```python
# 감염 케이스 별 확진자 수
group_cals_func(case, 'infection_case', 'confirmed', 'mean')
```

    infection_case
    Shincheonji Church                       347.466667
    Second Mi-Ju Hospital                    196.000000
    Hansarang Convalescent Hospital          128.000000
    gym facility in Cheonan                  103.000000
    Daesil Convalescent Hospital             100.000000
    contact with patient                      78.125000
    River of Grace Community Church           72.000000
    Bonghwa Pureun Nursing Home               68.000000
    Gyeongsan Seorin Nursing Home             66.000000
    etc                                       60.588235
    Guro-gu Call Center                       55.333333
    Pilgrimage to Israel                      49.000000
    overseas inflow                           45.705882
    Cheongdo Daenam Hospital                  41.000000
    Yechun-gun                                40.000000
    Fatima Hospital                           37.000000
    Milal Shelter                             36.000000
    Ministry of Oceans and Fisheries          30.000000
    Uijeongbu St. Mary’s Hospital             30.000000
    Bundang Jesaeng Hospital                  22.000000
    Manmin Central Church                     21.500000
    Onchun Church                             20.500000
    Dongan Church                             20.000000
    Gyeongsan Jeil Silver Town                17.000000
    Gyeongsan Cham Joeun Community Center     16.000000
    Eunpyeong St. Mary's Hospital             14.000000
    Seongdong-gu APT                          13.000000
    Goesan-gun Jangyeon-myeon                 11.000000
    Geochang Church                           10.000000
    Wings Tower                               10.000000
    Jongno Community Center                   10.000000
    Suwon Saeng Myeong Saem Church            10.000000
    Geochang-gun Woongyang-myeon               8.000000
    gym facility in Sejong                     8.000000
    Hanmaeum Changwon Hospital                 7.000000
    Jung-gu Fashion Company                    7.000000
    Changnyeong Coin Karaoke                   7.000000
    Haeundae-gu Catholic Church                6.000000
    Seosan-si Laboratory                       5.500000
    Suyeong-gu Kindergarten                    5.000000
    Jin-gu Academy                             4.000000
    Wonju-si Apartments                        3.000000
    Korea Forest Engineer Institute            3.000000
    Name: confirmed, dtype: float64

```python
province_code_mapper = dict(zip(wt.code, wt.province))
province_code_mapper
```

    {10000: 'Seoul',
     11000: 'Busan',
     12000: 'Daegu',
     13000: 'Gwangju',
     14000: 'Incheon',
     15000: 'Daejeon',
     16000: 'Ulsan',
     20000: 'Gyeonggi-do',
     30000: 'Gangwon-do',
     40000: 'Chunghceongbuk-do',
     41000: 'Chungcheongnam-do',
     50000: 'Jeollabuk-do',
     51000: 'Jeollanam-do',
     60000: 'Gyeongsangbuk-do',
     61000: 'Gyeongsangnam-do',
     70000: 'Jeju-do'}

### 감염 케이스 중 집단 감염일 확률?

```python
_per = len(case[case['group'] == True])/len(case)*100
print("percentage : %s%%"%round(_per,3))
```

    percentage : 55.357%

### contact number 따른 확률변수?

- 데이터를 수집한다.

- 수집한 데이터가 어떤 확률변수의 표본 데이터라고 가정

- 데이터를 사용하여 해당 확률변수의 확률분포함수의 모양을 결정

- 결정된 확률변수로부터 다음에 생성될 데이터나 데이터 특성을 예측

```python
import scipy.stats as stats
import sympy
```

```python
_p_info = p_info[['sex','age','province','infection_case','contact_number','symptom_onset_date','confirmed_date','released_date','deceased_date']]
_p_info
```

      sex
      age
      province
      infection_case
      contact_number
      symptom_onset_date
      confirmed_date
      released_date
      deceased_date

      0
      male
      50s
      Seoul
      overseas inflow
      75.0
      2020-01-22
      2020-01-23
      2020-02-05
      NaN

      1
      male
      30s
      Seoul
      overseas inflow
      31.0
      NaN
      2020-01-30
      2020-03-02
      NaN

      2
      male
      50s
      Seoul
      contact with patient
      17.0
      NaN
      2020-01-30
      2020-02-19
      NaN

      3
      male
      20s
      Seoul
      overseas inflow
      9.0
      2020-01-26
      2020-01-30
      2020-02-15
      NaN

      4
      female
      20s
      Seoul
      contact with patient
      2.0
      NaN
      2020-01-31
      2020-02-24
      NaN

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3383
      female
      20s
      Jeju-do
      overseas inflow
      14.0
      NaN
      2020-03-29
      NaN
      NaN

      3384
      female
      20s
      Jeju-do
      overseas inflow
      18.0
      NaN
      2020-04-03
      NaN
      NaN

      3385
      male
      30s
      Jeju-do
      contact with patient
      5.0
      NaN
      2020-04-03
      NaN
      NaN

      3386
      female
      20s
      Jeju-do
      overseas inflow
      9.0
      NaN
      2020-04-03
      NaN
      NaN

      3387
      female
      10s
      Jeju-do
      overseas inflow
      6.0
      NaN
      2020-04-14
      NaN
      NaN

3388 rows × 9 columns

```python
case.infection_case
```

    0                Guro-gu Call Center
    1                      Dongan Church
    2              Manmin Central Church
    3      Eunpyeong St. Mary's Hospital
    4                   Seongdong-gu APT
                       ...
    107             contact with patient
    108                              etc
    109                  overseas inflow
    110             contact with patient
    111                              etc
    Name: infection_case, Length: 112, dtype: object

```python
p_info
```

      patient_id
      global_num
      sex
      birth_year
      age
      country
      province
      city
      disease
      infection_case
      infection_order
      infected_by
      contact_number
      symptom_onset_date
      confirmed_date
      released_date
      deceased_date
      state

      0
      1000000001
      2.0
      male
      1964.0
      50s
      Korea
      Seoul
      Gangseo-gu
      NaN
      overseas inflow
      1.0
      NaN
      75.0
      2020-01-22
      2020-01-23
      2020-02-05
      NaN
      released

      1
      1000000002
      5.0
      male
      1987.0
      30s
      Korea
      Seoul
      Jungnang-gu
      NaN
      overseas inflow
      1.0
      NaN
      31.0
      NaN
      2020-01-30
      2020-03-02
      NaN
      released

      2
      1000000003
      6.0
      male
      1964.0
      50s
      Korea
      Seoul
      Jongno-gu
      NaN
      contact with patient
      2.0
      2.002000e+09
      17.0
      NaN
      2020-01-30
      2020-02-19
      NaN
      released

      3
      1000000004
      7.0
      male
      1991.0
      20s
      Korea
      Seoul
      Mapo-gu
      NaN
      overseas inflow
      1.0
      NaN
      9.0
      2020-01-26
      2020-01-30
      2020-02-15
      NaN
      released

      4
      1000000005
      9.0
      female
      1992.0
      20s
      Korea
      Seoul
      Seongbuk-gu
      NaN
      contact with patient
      2.0
      1.000000e+09
      2.0
      NaN
      2020-01-31
      2020-02-24
      NaN
      released

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3383
      7000000009
      9651.0
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      14.0
      NaN
      2020-03-29
      NaN
      NaN
      isolated

      3384
      7000000010
      NaN
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      18.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated

      3385
      7000000011
      NaN
      male
      NaN
      30s
      Korea
      Jeju-do
      Jeju-do
      NaN
      contact with patient
      NaN
      7.000000e+09
      5.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated

      3386
      7000000012
      NaN
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      9.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated

      3387
      7000000013
      NaN
      female
      NaN
      10s
      China
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      6.0
      NaN
      2020-04-14
      NaN
      NaN
      isolated

3388 rows × 18 columns

```python
p_info.infection_case.value_counts()
```

    contact with patient                     968
    overseas inflow                          604
    etc                                      572
    Guro-gu Call Center                      112
    Shincheonji Church                       107
    Onchun Church                             33
    Bonghwa Pureun Nursing Home               31
    gym facility in Cheonan                   30
    Ministry of Oceans and Fisheries          28
    Cheongdo Daenam Hospital                  21
    Dongan Church                             17
    Eunpyeong St. Mary's Hospital             16
    Gyeongsan Seorin Nursing Home             15
    Seongdong-gu APT                          13
    Gyeongsan Jeil Silver Town                12
    Milal Shelter                             11
    Gyeongsan Cham Joeun Community Center     10
    Geochang Church                            6
    gym facility in Sejong                     4
    Changnyeong Coin Karaoke                   4
    Suyeong-gu Kindergarten                    3
    Pilgrimage to Israel                       2
    River of Grace Community Church            1
    Name: infection_case, dtype: int64

```python
# 연령대 별 감염 수
age_infect_cnt_dict = dict(zip(p_info.age.value_counts().index, p_info.age.value_counts().values))
age_infect_cnt_dict
```

    {'20s': 802,
     '50s': 597,
     '40s': 455,
     '30s': 438,
     '60s': 401,
     '70s': 204,
     '80s': 156,
     '10s': 150,
     '0s': 55,
     '90s': 45,
     '100s': 1}

```python
# 연령대별 감염률
age_infect_rate_dict = {}
for k,v in age_infect_cnt_dict.items():
    age_infect_rate_dict[k] = v/sum([v for k,v in age_infect_cnt_dict.items()])

age_infect_rate_dict
```

    {'20s': 0.23671782762691854,
     '50s': 0.17621015348288074,
     '40s': 0.13429752066115702,
     '30s': 0.1292798110979929,
     '60s': 0.11835891381345927,
     '70s': 0.0602125147579693,
     '80s': 0.04604486422668241,
     '10s': 0.04427390791027155,
     '0s': 0.016233766233766232,
     '90s': 0.013282172373081463,
     '100s': 0.00029515938606847696}

```python
age_infect_cnt_df = pd.DataFrame(list(age_infect_cnt_dict.items()), columns=['age', 'cnt'])
age_infect_cnt_df.plot.hist(bins=14)
```

```python
age_infect_rate_df = pd.DataFrame(list(age_infect_rate_dict.items()), columns=['age', 'rate'])
age_infect_rate_df.plot.hist(bins=14)
```

```python
sns.distplot(age_infect_rate_df['rate'])
```

```python
sns.distplot(p_info['contact_number'])
```

```python
p_info['age_rate'] = p_info['age'].map(age_infect_rate_dict)
p_info
```

      patient_id
      global_num
      sex
      birth_year
      age
      country
      province
      city
      disease
      infection_case
      infection_order
      infected_by
      contact_number
      symptom_onset_date
      confirmed_date
      released_date
      deceased_date
      state
      age_rate

      0
      1000000001
      2.0
      male
      1964.0
      50s
      Korea
      Seoul
      Gangseo-gu
      NaN
      overseas inflow
      1.0
      NaN
      75.0
      2020-01-22
      2020-01-23
      2020-02-05
      NaN
      released
      0.176210

      1
      1000000002
      5.0
      male
      1987.0
      30s
      Korea
      Seoul
      Jungnang-gu
      NaN
      overseas inflow
      1.0
      NaN
      31.0
      NaN
      2020-01-30
      2020-03-02
      NaN
      released
      0.129280

      2
      1000000003
      6.0
      male
      1964.0
      50s
      Korea
      Seoul
      Jongno-gu
      NaN
      contact with patient
      2.0
      2.002000e+09
      17.0
      NaN
      2020-01-30
      2020-02-19
      NaN
      released
      0.176210

      3
      1000000004
      7.0
      male
      1991.0
      20s
      Korea
      Seoul
      Mapo-gu
      NaN
      overseas inflow
      1.0
      NaN
      9.0
      2020-01-26
      2020-01-30
      2020-02-15
      NaN
      released
      0.236718

      4
      1000000005
      9.0
      female
      1992.0
      20s
      Korea
      Seoul
      Seongbuk-gu
      NaN
      contact with patient
      2.0
      1.000000e+09
      2.0
      NaN
      2020-01-31
      2020-02-24
      NaN
      released
      0.236718

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3383
      7000000009
      9651.0
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      14.0
      NaN
      2020-03-29
      NaN
      NaN
      isolated
      0.236718

      3384
      7000000010
      NaN
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      18.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated
      0.236718

      3385
      7000000011
      NaN
      male
      NaN
      30s
      Korea
      Jeju-do
      Jeju-do
      NaN
      contact with patient
      NaN
      7.000000e+09
      5.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated
      0.129280

      3386
      7000000012
      NaN
      female
      NaN
      20s
      Korea
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      9.0
      NaN
      2020-04-03
      NaN
      NaN
      isolated
      0.236718

      3387
      7000000013
      NaN
      female
      NaN
      10s
      China
      Jeju-do
      Jeju-do
      NaN
      overseas inflow
      NaN
      NaN
      6.0
      NaN
      2020-04-14
      NaN
      NaN
      isolated
      0.044274

3388 rows × 19 columns

```python
# 연령대에 따른 확률분포 도식화?
sns.distplot(p_info['age_rate'])
```

```python
# 유동 인구수
sns.distplot(sf['fp_num'])
```

```python
import seaborn as sns
print(sp.stats.describe(x))
```

    DescribeResult(nobs=432000, minmax=(4480, 127640), mean=27860.034884259258, variance=172208888.73579407, skewness=1.9056691047057897, kurtosis=7.6358041476710845)

```python
print(x.describe())
```

    count    432000.000000
    mean      27860.034884
    std       13122.838441
    min        4480.000000
    25%       18940.000000
    50%       25690.000000
    75%       34000.000000
    max      127640.000000
    Name: fp_num, dtype: float64

## 기댓값

- 확률밀도함수나 확률분포함수는 확률변수의 전체적인 성격을 설명하는데, 때로 우리는 몇 개의 수치로 확률분포의 성질을 요약하고자 하기도 함
- 이러한 성질을 요약하는 수치들 중 하나로 변수의 expectation을 생각해보자

```python
age_infect_rate_dict
```

    {'20s': 0.23671782762691854,
     '50s': 0.17621015348288074,
     '40s': 0.13429752066115702,
     '30s': 0.1292798110979929,
     '60s': 0.11835891381345927,
     '70s': 0.0602125147579693,
     '80s': 0.04604486422668241,
     '10s': 0.04427390791027155,
     '0s': 0.016233766233766232,
     '90s': 0.013282172373081463,
     '100s': 0.00029515938606847696}

```python
len(age_infect_rate_dict)
```

    11

```python
# 미성년 (0s or 10s)
# 청장년 (20s ~ 50s)
# 노년 (60s ~ 100s)
```

```python
# 10명을 임의로 선정했을 때 미성년일 확률?

a = list(age_infect_rate_dict.keys())
```

```python
# 임의로 두 명을 뽑았을 때 나올 수 있는 모든 가정의 수
# p = itertools.permutations(a, 2)  # 순열

b = itertools.combinations(a,2) # 조합
len(list(itertools.combinations(a,2)))
```

    55

```python
age_infect_rate_dict
```

    {'20s': 0.23671782762691854,
     '50s': 0.17621015348288074,
     '40s': 0.13429752066115702,
     '30s': 0.1292798110979929,
     '60s': 0.11835891381345927,
     '70s': 0.0602125147579693,
     '80s': 0.04604486422668241,
     '10s': 0.04427390791027155,
     '0s': 0.016233766233766232,
     '90s': 0.013282172373081463,
     '100s': 0.00029515938606847696}

```python
# 두 명을 뽑았을때 미성년이 나오는 모든 경우의 수
len(d) + len(a)
```

    121

```python
# debug 용 function
def permutations_with_replacement(data, n=2):
    _list = [data]*n
    return list(itertools.product(*_list))
```

```python
num_of_cases = permutations_with_replacement(a,2)
```

```python
num_of_cases
```

    [('20s', '20s'),
     ('20s', '50s'),
     ('20s', '40s'),
     ('20s', '30s'),
     ('20s', '60s'),
     ('20s', '70s'),
     ('20s', '80s'),
     ('20s', '10s'),
     ('20s', '0s'),
     ('20s', '90s'),
     ('20s', '100s'),
     ('50s', '20s'),
     ('50s', '50s'),
     ('50s', '40s'),
     ('50s', '30s'),
     ('50s', '60s'),
     ('50s', '70s'),
     ('50s', '80s'),
     ('50s', '10s'),
     ('50s', '0s'),
     ('50s', '90s'),
     ('50s', '100s'),
     ('40s', '20s'),
     ('40s', '50s'),
     ('40s', '40s'),
     ('40s', '30s'),
     ('40s', '60s'),
     ('40s', '70s'),
     ('40s', '80s'),
     ('40s', '10s'),
     ('40s', '0s'),
     ('40s', '90s'),
     ('40s', '100s'),
     ('30s', '20s'),
     ('30s', '50s'),
     ('30s', '40s'),
     ('30s', '30s'),
     ('30s', '60s'),
     ('30s', '70s'),
     ('30s', '80s'),
     ('30s', '10s'),
     ('30s', '0s'),
     ('30s', '90s'),
     ('30s', '100s'),
     ('60s', '20s'),
     ('60s', '50s'),
     ('60s', '40s'),
     ('60s', '30s'),
     ('60s', '60s'),
     ('60s', '70s'),
     ('60s', '80s'),
     ('60s', '10s'),
     ('60s', '0s'),
     ('60s', '90s'),
     ('60s', '100s'),
     ('70s', '20s'),
     ('70s', '50s'),
     ('70s', '40s'),
     ('70s', '30s'),
     ('70s', '60s'),
     ('70s', '70s'),
     ('70s', '80s'),
     ('70s', '10s'),
     ('70s', '0s'),
     ('70s', '90s'),
     ('70s', '100s'),
     ('80s', '20s'),
     ('80s', '50s'),
     ('80s', '40s'),
     ('80s', '30s'),
     ('80s', '60s'),
     ('80s', '70s'),
     ('80s', '80s'),
     ('80s', '10s'),
     ('80s', '0s'),
     ('80s', '90s'),
     ('80s', '100s'),
     ('10s', '20s'),
     ('10s', '50s'),
     ('10s', '40s'),
     ('10s', '30s'),
     ('10s', '60s'),
     ('10s', '70s'),
     ('10s', '80s'),
     ('10s', '10s'),
     ('10s', '0s'),
     ('10s', '90s'),
     ('10s', '100s'),
     ('0s', '20s'),
     ('0s', '50s'),
     ('0s', '40s'),
     ('0s', '30s'),
     ('0s', '60s'),
     ('0s', '70s'),
     ('0s', '80s'),
     ('0s', '10s'),
     ('0s', '0s'),
     ('0s', '90s'),
     ('0s', '100s'),
     ('90s', '20s'),
     ('90s', '50s'),
     ('90s', '40s'),
     ('90s', '30s'),
     ('90s', '60s'),
     ('90s', '70s'),
     ('90s', '80s'),
     ('90s', '10s'),
     ('90s', '0s'),
     ('90s', '90s'),
     ('90s', '100s'),
     ('100s', '20s'),
     ('100s', '50s'),
     ('100s', '40s'),
     ('100s', '30s'),
     ('100s', '60s'),
     ('100s', '70s'),
     ('100s', '80s'),
     ('100s', '10s'),
     ('100s', '0s'),
     ('100s', '90s'),
     ('100s', '100s')]

```python
len(num_of_cases)
```

    121

```python
# 미성년일 확률
prob_of_minor = age_infect_rate_dict['0s']+age_infect_rate_dict['10s']
print('Probability of minor : ', prob_of_minor)

# 성년일 확률
prob_of_adult = 1-(prob_of_minor)
print('Probability of adult : ', prob_of_adult)
```

    Probability of minor :  0.062046004842615014
    Probability of adult :  0.937953995157385

#### 관심있는 변수 -> 두 명을 추출했을 때, 추출한 인원 중 한 명 이상이 미성년일 확률

- P(X=0) = 0.003849706716929806
    + 임의의 두 명이 모두 미성년인 경우

- P(X=1) = 0.00002964048361274893
    + 임의의 두 명 중 한 명만 미성년인 경우

- P(X=2) = 0.9923154068079466
    + 임의의 두 명이 모두 성인인 경우

```python
# P(X=2)
x_0 = prob_of_minor**2
print(x_0)

# P(X=1)
x_1 = 2*(x_0**2)
print(x_1)

# P(X=1)
x_2 = (1-x_0)**2
print(x_2)
```

    0.003849706716929806
    2.964048361274893e-05
    0.9923154068079466

```python
# x(추출한 인원 중 한 명 이상이 미성년)의 기대값
# E(X) = x_2*0 + x_1*1 + x_1*1 + x_0*2
e = x_2*0 + x_1*1 + x_1*1 + x_0*2
e

# 이거 맞나??
```

    0.00775869440108511

```python
# x(추출한 인원 중 한 명 이상이 성년)의 기대값
# E(X) = x_2*0 + x_1*1 + x_1*1 + x_0*2
e2 = x_2*2 + x_1*1 + x_1*1 + x_0*0
e2
```

    1.9846900945831185

```python
# probabilily mass function
rand_var_dict = {'x_0': x_0, 'x_1':x_1,'x_2':x_2}
```

```python
rand_var_dict
```

    {'x_0': 0.003849706716929806,
     'x_1': 2.964048361274893e-05,
     'x_2': 0.9923154068079466}

```python
_k = [k for k,v in rand_var_dict.items()]
_v = [v for k,v in rand_var_dict.items()]
```

```python
plt.bar(_k, _v)
plt.ylabel("Probability")
plt.xlabel("Random Variable")
plt.show()
plt.close()
```

```python

```


## Calibration Workshop

In this Notebook we will:
- Load data and train a model
- Assess the Calibration of the model
- Explore various methods to calibrate the results of the model

Notebook requires `numpy`, `pandas`, `scikit-learn` as well as the `ml-insights` and `betacal` packages

```python
# !pip install ml_insights
# !pip install betacal

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import ml_insights as mli
%matplotlib inline

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss, brier_score_loss, roc_auc_score
from sklearn.isotonic import IsotonicRegression
from sklearn.linear_model import LogisticRegression
from sklearn.calibration import calibration_curve
from betacal import BetaCalibration
mli.__version__
```

    '0.1.6'

### MIMIC ICU Data*

We illustrate calibration using a mortality model on the MIMIC ICU data.  Each row represents a hospital stay of an individual patient.  We have many lab values and vital sign measurements, as well as an indicator of whether or not the patient died in the hospital.

*MIMIC-III, a freely accessible critical care database. Johnson AEW, Pollard TJ, Shen L, Lehman L, Feng M, Ghassemi M, Moody B, Szolovits P, Celi LA, and Mark RG. Scientific Data (2016).
https://mimic.physionet.org

```python
# Load dataset derived from the MMIC database
lab_aug_df = pd.read_csv("data/lab_vital_icu_table.csv")
```

```python
# Impute the median for in each column to replace NA's
for i in range(len(lab_aug_df.columns)):
    if lab_aug_df.iloc[:,i].dtype!='O':
        lab_aug_df.iloc[:,i].fillna(lab_aug_df.iloc[:,i].median(),inplace=True)
```

## Lesson 1: Assessing Calibration

We will be building a model to predict mortality in the ICU based on vital signs and lab values.  To start, we will just pick a few different ones on which to build our model.

```python
# Choose a subset of variables
feature_set_1 = ['bun_min',
       'bun_max', 'wbc_min', 'wbc_max','sysbp_max', 'sysbp_min']

X_1 = lab_aug_df.loc[:,feature_set_1]
y = lab_aug_df['hospital_expire_flag']
```

We now divide the data into training, calibration, and test sets.  The training set will be used to fit the model, the calibration set will be used to calibrate the probabilities, and the test set will be used to evaluate the performance. Later, we will learn about cross-validation approaches that avoid the need for a separate calibration set.

Below are the variables used to control the size of the train, calibration, and test sets, as well as the random state used for split generation *and* random forest model generation.

Note that there will be a lot of variance in the performance of the methods as we change these parameters.  Therefore it is important not to draw overly broad conclusions from individual runs.  At the end of this notebook will be an exercise to change these parameters around and observe the variation.

```python
train_perc = .6
calib_perc = .05
test_perc = 1-train_perc-calib_perc
rs = 42
```

```python
X_train_calib_1, X_test_1, y_train_calib_1, y_test_1 = train_test_split(X_1, y, test_size=test_perc, random_state=rs)
```

```python
X_train_1, X_calib_1, y_train_1, y_calib_1 = train_test_split(X_train_calib_1, y_train_calib_1,
                                                              test_size=calib_perc/(1-test_perc),
                                                              random_state=rs)
```

```python
X_train_1.shape, X_calib_1.shape, X_test_1.shape
```

    ((35834, 6), (2987, 6), (20905, 6))

```python
# To understand the problem better, let's see what percentage of patients overall died in the ICU
np.mean(y_train_1)
```

    0.1092537813249986

Next, we will fit a Random Forest model to our training data.  Then we'll use that model to predict "probabilities" on our validation and test sets.

I use quotes on "probabilities" because these numbers, which are the percentage of trees that voted "yes" are better understood as mere scores.  A higher value should generally indicate a higher probability of mortality.  However, in general, one should not expect these to be well-calibrated probabilities.  The fact that, say, 60% of the trees voted "yes" on a particular case does not necessarily mean that that case has a 60% probability of mortality.

```python
rfmodel1 = RandomForestClassifier(n_estimators = 500, class_weight='balanced_subsample',
                                  random_state=rs, n_jobs=-1 )
rfmodel1.fit(X_train_1,y_train_1)
```

    RandomForestClassifier(bootstrap=True, class_weight='balanced_subsample',
                           criterion='gini', max_depth=None, max_features='auto',
                           max_leaf_nodes=None, min_impurity_decrease=0.0,
                           min_impurity_split=None, min_samples_leaf=1,
                           min_samples_split=2, min_weight_fraction_leaf=0.0,
                           n_estimators=500, n_jobs=-1, oob_score=False,
                           random_state=42, verbose=0, warm_start=False)

```python
rf1_preds_test_uncalib = rfmodel1.predict_proba(X_test_1)[:,1]
```

```python

```

## Assessing Calibration: Log-loss (aka Cross-Entropy aka Negative Mean Log-Likelihood)
- The `log_loss` is a common metric to measure the "quality" of predicted probabilities
- AUROC measures the quality of the **ranking** but does not assess **calibration**
- `log_loss` assesses the combination of discrimination and calibration.
- `log_loss` is difficult to interpret on its own, generally used comparatively.

$\begin{equation}
\mbox{log_loss} = \frac{1}{n} \left(\sum_{\mbox{pos cases}} -log(p_i) + \sum_{\mbox{neg cases}} -log(1- p_i)\right)
\end{equation}$

- If you predicted a probability of .25 for a case, and it happened, your loss for that case would be $-\log(.25) = \log(1/.25) = \log(4)$
- If you predicted a probability of .8 for a case, and it *didn't* happen, your loss for that case would be $-\log(1-.8) = \log(1/.2) = \log(5)$
- Loss is 0 when you are certain about the outcome and you are right
- Loss is $\infty$ when you are certain about the outcome and you are wrong

```python
roc_auc_score(y_test_1, rf1_preds_test_uncalib), log_loss(y_test_1, rf1_preds_test_uncalib)
```

    (0.7645201843381769, 0.31209048301332465)

```python
# If I divide all probabilities by 4, AUROC is the same, but log_loss gets worse
roc_auc_score(y_test_1, rf1_preds_test_uncalib/4), log_loss(y_test_1, rf1_preds_test_uncalib/4)
```

    (0.7645201843381769, 0.3875879899067894)

```python

```

## Assessing Calibration: Brier Score
$\begin{equation}
\mbox{Brier_score} = \frac{1}{n} \left(\sum_{\mbox{all cases}} (y_i-p_i)^2 \right)
\end{equation}$

Brier score is a fancy name for the mean squared error between the predicted probabilities and the true (0/1) answer.  If I predict .7 on a case that was true (1), my error for that case is $(1-.7)^2 = .09$

Average the scores on all cases, and that gives the Brier score.

Note that the "worst case" for Brier score is 1, whereas for log-loss it is $\infty$.

```python
brier_score_loss(y_test_1, rf1_preds_test_uncalib)
```

    0.0849214801158517

### Log-loss vs Brier Score

The main difference between Brier score and log-loss is how they deal with small probabilities.  Suppose the "true" probability is .01 and you predict a probability of .0001, is that a "big" error?

According to log-loss, it is.  You have understated the true probability by a factor of 100.  If you were working in insurance, and predicting the probability of a car accident, the insurance company would be paying out 100x as much as they thought they would.

According to Brier score, it is not that a big deal.  For some applications, that may be appropriate.  For example, if you are modeling the probability someone will vote for Candidate A vs Candidate B, and planning to use that model to estimate election results on some population, it doesn't matter to distinguish between very low probabilities.

This is actually a very "deep" topic.  We will focus primarily on the log-loss, but will also show the results on Brier score.

## Assessing Calibration: Reliability Diagram
A visual way to check the calibration of a model is to create a "Reliability Diagram".  The idea behind the reliability diagram is the following:
- Bin the interval [0,1] into smaller subsets (e.g. [0, 0.05], [0.05, .1], ... [.95,1])
- Find the empirical probabilities when the probabilities fell into each bin  (if there were 20 times, and 9 of them were "yes", the empirical probability is .45)
- Plot the predicted probability (average of predicted probabilities in each bin) (x-axis) vs the empirical probabilities(y-axis)
- When the dots are (significantly) above the line y=x, the model is under-predicting the true probability, if below the line, model is over-predicting the true probability.

We will use the ml-insights `plot_reliability_diagram` function.  It has a fair bit of flexibility that we will explore.
Some features include:
- custom bins
- accompanying histogram
- error bars
- logit scaling to explore calibration of very small and very large probabilities

```python
mli.plot_reliability_diagram?
```

    Signature:
    mli.plot_reliability_diagram(
        y,
        x,
        bins=array([0.  , 0.05, 0.1 , 0.15, 0.2 , 0.25, 0.3 , 0.35, 0.4 , 0.45, 0.5 ,
           0.55, 0.6 , 0.65, 0.7 , 0.75, 0.8 , 0.85, 0.9 , 0.95, 1.  ]),
        show_baseline=True,
        error_bars=True,
        error_bar_alpha=0.05,
        show_histogram=False,
        scaling='none',
        scaling_eps=0.0001,
        scaling_base=10,
        c='red',
        **kwargs,
    )
    Docstring:
    Plots a reliability diagram of predicted vs empirical probabilities.

    Parameters
    ----------
    y: array-like, length (n_samples). The true outcome values as integers (0 or 1)

    x: The predicted probabilities, between 0 and 1 inclusive.

    bins: array-like, the endpoints of the bins used to aggregate and estimate the
        empirical probabilities.  Default is 20 equally sized bins
        from 0 to 1, i.e. [0,0.05,0.1,...,.95, .1].

    show_baseline: whether or not to print a dotted black line representing
        y=x (perfect calibration).  Default is True

    error_bars: whether to show error bars reflecting the confidence
        interval under the assumption that the input probabilities are
        perfectly calibrated. Default is True.

    error_bar_alpha: The alpha value to use for the error_bars.  Default
        is .05 (a 95% CI).  Confidence intervals are based on the exact
        binomial distribution, not the normal approximation.

    show_histogram: Whether or not to show a separate histogram of the
        number of values in each bin.  Default is False

    scaling: Default is 'none'. Alternative is 'logit' which is useful for
        better examination of calibration near 0 and 1.  Values shown are
        on the scale provided and then tick marks are relabeled.

    scaling_eps: default is .0001.  Ignored unless scaling='logit'. This
        indicates the smallest meaningful positive probability you
        want to consider.

    scaling_base: default is 10. Ignored unless scaling='logit'. This
        indicates the base used when scaling back and forth.  Matters
        only in how it affects the automatic tick marks.

    c: color of the plotted points.  Default is 'red'.

    **kwargs: additional args to be passed to the plt.scatter matplotlib call.

    Returns
    -------
    A dictionary containing the x and y points plotted (unscaled) and the
        count in each bin.
    File:      ~/Work/temp_mli_development/introspective/ml_insights/calibration_utils.py
    Type:      function

```python
# This is the default plot
rd = mli.plot_reliability_diagram(y_test_1, rf1_preds_test_uncalib);
```

Above we see the default reliability diagram.  While most of the points seem to be within the error bars there are a couple of flaws to point out:
1. Between about .2 and .45 we are consistently over-predicting the mortality.  Though they are right on the edge of the error bar, having many in a row with errors in the same direction indicates this is not random noise.
1. It is hard to see the smaller probabilities well (in the first two bins), but they look like they may be under-predicting.  We will demonstrate how to explore this better

First we will explore some options in the `plot_reliability_diagram` function

```python
# You can opt to have a histogram showing the counts in each bin
plt.figure(figsize=(10,5))
mli.plot_reliability_diagram(y_test_1, rf1_preds_test_uncalib, show_histogram=True);
```

We see that we have lots of observations with small predicted probabilities and fewer with large probabilities.  Suppose we want to subdivide further the bins where we have lots of data, and aggregate the bins that have less data.

```python

```

```python
plt.figure(figsize=(10,5))
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
mli.plot_reliability_diagram(y_test_1, rf1_preds_test_uncalib, bins=custom_bins_a, show_histogram=True);
```

Again, we may be underpredicting close to 0, but it is hard to tell.  To look closer, we can use the "logit" scaling.  This scaling uses more area at probabilities close to 0 and 1, and less area close to .5

```python
plt.figure(figsize=(10,10))
rd = mli.plot_reliability_diagram(y_test_1, rf1_preds_test_uncalib, scaling='logit',
                                 bins=custom_bins_a, marker='.')
```

Now we are able to see that we are underpredicting at low probabilities and then overpredicting around .3.  Let's extract a few numbers by looking at the dictionary `rd` returned by the function

```python
rd.keys()
```

    dict_keys(['pred_probs', 'emp_probs', 'bin_counts'])

```python
rd['pred_probs'], rd['emp_probs'], rd['bin_counts']
```

    (array([0.00172195, 0.01404711, 0.02396347, 0.03869902, 0.07250134,
            0.16990728, 0.37734362, 0.60201047, 0.81447059]),
     array([0.01230172, 0.04026354, 0.03924419, 0.05680425, 0.07761506,
            0.14967427, 0.33951613, 0.55235602, 0.81176471]),
     array([3089., 1366., 1376., 2447., 4780., 6140., 1240.,  382.,   85.]))

Here we can see clearly that for the smallest bin, we have an average predicted probability of about .0017 but empirically (on 3089 trials) we have a probability of .012 -- off by a factor of 7!

```python
.012/.0017
```

    7.058823529411765

#### sklearn `calibration_curve`
- Scikit-learn has a function `calibration_curve` that will give the x and y coordinates for a number of bins.
- The rest of the plotting is up to you
- It does not support custom bin widths (October 2020)

```python
prob_true, prob_pred = calibration_curve(y_test_1, rf1_preds_test_uncalib, n_bins=20)
plt.scatter(prob_pred, prob_true)
plt.plot(np.linspace(0,1,11),np.linspace(0,1,11), color='k')
```

## Exercise
### It's your turn!
Repeat this process for a bigger set of features below

```python
# Choose a subset of variables

# feature_set_2 = feature_set_1 + ['lactate_min', 'lactate_max', 'platelet_min', 'platelet_max',
#        'potassium_min', 'potassium_max', 'ptt_min', 'ptt_max', 'inr_min',
#        'inr_max']
```

```python
# Repeat train, calib, test split

# X_2 = lab_aug_df.loc[:,feature_set_2]
# X_train_calib_2, X_test_2, y_train_calib_2, y_test_2 = train_test_split(X_2, y, test_size=test_perc, random_state=rs)
# X_train_2, X_calib_2, y_train_2, y_calib_2 = train_test_split(X_train_calib_2, y_train_calib_2,
#                                                               test_size=calib_perc/(1-test_perc),
#                                                               random_state=rs)
```

```python
# Fit a Random Forest Model

# rfmodel2 = RandomForestClassifier(n_estimators=500, class_weight='balanced_subsample',
#                                   random_state=rs, n_jobs=-1 )
# rfmodel2.fit(X_train_2,y_train_2);
```

```python
# rf2_preds_test_uncalib = rfmodel2.predict_proba(X_test_2)[:,1]
```

```python
# roc_auc_score(y_test_2, rf2_preds_test_uncalib), log_loss(y_test_2, rf2_preds_test_uncalib)
```

Use the cells below to explore the calibration of this model.

- Is it well calibrated?
- How does it compare to the previous model?

```python
# Plot the default reliability diagram
# mli.plot_reliability_diagram();
```

```python
# Display the histogram on the side

# plt.figure(figsize=(8,4))
# mli.plot_reliability_diagram(...);
```

```python
# Create a custom set of bins

# bins_custom_b = np.array([])
# mli.plot_reliability_diagram(...);

```

```python
# Use the logit scaling

# plt.figure(figsize=(10,4))
# mli.plot_reliability_diagram(...);
```

### Extra Credit:
Fit another kind of model (Boosting, Logistic Regression, etc.) on the same data set and assess the calibration of that model

```python

```

## Lesson 2: Calibrating a Model
Since our models are not well-calibrated, we would like to fix this.

### Getting a **Calibration** Data Set

We will discuss two ways to get a data set on which to perform calibration:

- Use an independent calibration set
- Using Cross-validation to generate scores from the training set.

The first method is simpler, but requires a separate data set, meaning that you will have less data to train your model with.  It is good to use if you have plenty of data.

The second approach takes more time, but is generally more data-efficient.  We generate a set of cross-validated predictions on the training data.  These predictions come from models that are close to, but not exactly identical to, your original model.  However, this discrepancy is usually minor and offset by having more data on which to calibrate.

### Method of Calibration

The data set for calibration is a set of scores and the corresponding binary outcomes.  The goal is then to find a function that "fits" the relationship between the scores and the "actual" probabilities (as determined empirically in the calibration set.  We will review 4 methods of calibration:

- Platt Scaling
- Beta Calibration
- Isotonic Regression
- SplineCalib

## Approach A: Independent calibration set
Overall process:
- Need separate training and calibration sets (plus a test set to evaluate)
- Fit model on training set data
- Make predictions on calibration set.
- Use those predictions + true answers to fit a calibration object.
- Use model to make predictions on test set
- Use calibrator to calibrate those predictions
- Evaluate log_loss, reliability diagram on calibrated predictions

```python
calibset_preds_uncalib_1 = rfmodel1.predict_proba(X_calib_1)[:,1]
testset_preds_uncalib_1 = rfmodel1.predict_proba(X_test_1)[:,1]
```

## Method 1: Platt Scaling
Assumes that there is a logistic relationship between the scores $z$ and the true probability $p$.

$\log\left(\frac{p}{1-p}\right) = \alpha + \beta z$

$p = \frac{1}{1+\exp(-(\alpha + \beta z))}$

So it fits the two parameters $\alpha$ and $\beta$ just like in logistic regression!

- Very restrictive set of possible functions
- Needs very little data
- Historically, came from the observation (and subsequent theoretical arguments) that a logistic regression was the "right" calibration for Support Vector Machines

Reference: Platt, J. (1999). Probabilistic outputs for support vector machines and comparison to regularized likelihood methods. Advances in Large Margin Classifiers (pp.61–74).

```python
# Fit Platt scaling (logistic calibration)
lr = LogisticRegression(C=99999999999, solver='lbfgs')
lr.fit(calibset_preds_uncalib_1.reshape(-1,1), y_calib_1)

```

    LogisticRegression(C=99999999999, class_weight=None, dual=False,
                       fit_intercept=True, intercept_scaling=1, l1_ratio=None,
                       max_iter=100, multi_class='warn', n_jobs=None, penalty='l2',
                       random_state=None, solver='lbfgs', tol=0.0001, verbose=0,
                       warm_start=False)

```python
calibset_platt_probs = lr.predict_proba(calibset_preds_uncalib_1.reshape(-1,1))[:,1]
testset_platt_probs = lr.predict_proba(testset_preds_uncalib_1.reshape(-1,1))[:,1]
```

```python

```

```python
mli.plot_reliability_diagram(y_calib_1, calibset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, lr.predict_proba(tvec.reshape(-1,1))[:,1]);
plt.title('Platt Calibration Curve on Calibration Data');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, lr.predict_proba(tvec.reshape(-1,1))[:,1])
plt.title('Platt Calibration Curve on Test Data');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_platt_probs);
plt.title('Reliability Diagram on Test Data\n after Platt Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_platt_probs, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Platt Calibrated Model');
```

Using the logit scaling, we can see that the calibration does poorly at the small values.

```python

```

```python

```

```python
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Platt calibrated log_loss = 0.29833851247471244
    Uncalibrated log_loss = 0.31209048301332465

Despite this, we see an improvement in log_loss.  Generally, making mistakes by predicting "closer to .5" is better for log loss (i.e. better to overpredict rare events and underpredict near certain events)

## Method 2: Isotonic Regression
- Fits a piecewise constant, monotonically increasing, function to map the scores to probabilities.
- Uses the PAV (Pool Adjacent Violators, also called PAVA) algorithm.
- Does not assume a particular parametric form.
- Tends to be better than Platt scaling with enough data
- Tends to overfit: ("choppy" with unrealistic jumps)

Reference: Zadrozny,  B., & Elkan, C.(2001). Obtaining calibrated probability estimates from decision trees and naive bayesian classifiers. ICML (pp.609–616).

Zadrozny,  B., & Elkan, C. (2002). Transforming classifier scores into accurate multiclass probability estimates. KDD (pp.694–699).

```python
iso = IsotonicRegression(out_of_bounds = 'clip')
iso.fit(calibset_preds_uncalib_1, y_calib_1)
```

    IsotonicRegression(increasing=True, out_of_bounds='clip', y_max=None,
                       y_min=None)

```python
calibset_iso_probs = iso.predict(calibset_preds_uncalib_1)
testset_iso_probs = iso.predict(testset_preds_uncalib_1)
```

```python
mli.plot_reliability_diagram(y_calib_1, calibset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, iso.predict(tvec), label='Isotonic');
plt.title('Isotonic Calibration Curve on Calibration Data');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, iso.predict(tvec), label='Isotonic');
plt.title('Isotonic Calibration Curve on Test Data');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_iso_probs);
plt.title('Reliability Diagram on Test Data\n after Isotonic Calibration');
```

Here we see some of the artifacts of isotonic calibration.  Some bins have few to no points after calibration due to the vertical "steps" in the function.

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_iso_probs, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Isotonic Calibrated Model');
```

```python

```

```python
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Isotonic calibrated log_loss = 0.31568907061162055
    Platt calibrated log_loss = 0.29833851247471244
    Uncalibrated log_loss = 0.31209048301332465

```python

```

## Method 3: Beta Calibration

"A well-founded and easily implemented improvement on logistic calibration for binary classifiers."

$p = \left(1+ 1 / \left( \exp(c) \frac{z^a}{(1-z)^b} \right) \right)^{-1}$

- Similar to Platt scaling with a couple of important improvements
- Is a 3-parameter family of curves rather than 2-parameter
- Family of curves *includes* the line $y=x$ (so it won't mess it up if it's already calibrated)

Reference: Kull, M., Filho, T.S. & Flach, P.. (2017). Beta calibration: a well-founded and easily implemented improvement on logistic calibration for binary classifiers. Proceedings of the 20th International Conference on Artificial Intelligence and Statistics, in PMLR 54:623-631

```python
# Fit three-parameter beta calibration
bc = BetaCalibration()
bc.fit(calibset_preds_uncalib_1, y_calib_1)

```

    BetaCalibration(parameters=None)

```python
calibset_bc_probs = bc.predict(calibset_preds_uncalib_1)
testset_bc_probs = bc.predict(testset_preds_uncalib_1)

```

```python
mli.plot_reliability_diagram(y_calib_1, calibset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, bc.predict(tvec))
plt.title('Beta Calibration Curve on Calibration Set');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, bc.predict(tvec))
plt.title('Beta Calibration Curve on Test Set');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_bc_probs);
plt.title('Reliability Diagram on Test Data\n after Beta Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_bc_probs, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Beta Calibrated Model');
```

```python
print('Beta calibrated log_loss = {}'.format(log_loss(y_test_1, testset_bc_probs)))
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Beta calibrated log_loss = 0.30257500027883283
    Isotonic calibrated log_loss = 0.31568907061162055
    Platt calibrated log_loss = 0.29833851247471244
    Uncalibrated log_loss = 0.31209048301332465

## Method 4: SplineCalib
- SplineCalib fits a cubic smoothing spline to the relationship between the uncalibrated scores and the calibrated probabilities
- Smoothing splines strike a balance between fitting the points well and having a smooth function
- SplineCalib uses a smoothed logistic function - so the fit to data is measured by likelihood (i.e. log-loss) and the smoothness refers to the integrated second derivative **before** the logistic transformation.
- There is a nuisance parameter that trades off smoothness for fit.  At one extreme it will revert to standard logistic regression (i.e. Platt scaling) and at the other extreme it will be a very wiggly function that fits the data but does not generalize well.

- SplineCalib automatically fits the nuisance parameter (though this can be adjusted by the user)
- The resulting calibration function is not necessarily monotonic.  (In some cases this may be beneficial).

References: Lucena, B. Spline-based Probability Calibration. https://arxiv.org/abs/1809.07751

```python
# Define SplineCalib object
splinecalib = mli.SplineCalib()
splinecalib.fit(calibset_preds_uncalib_1, y_calib_1)
```

```python
calibset_splinecalib_probs = splinecalib.predict(calibset_preds_uncalib_1)
testset_splinecalib_probs = splinecalib.predict(testset_preds_uncalib_1)

```

```python

```

```python
mli.plot_reliability_diagram(y_calib_1, calibset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, splinecalib.predict(tvec))
plt.title('SplineCalib Calibration Curve on Calibration Set');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, splinecalib.predict(tvec))
plt.title('SplineCalib Calibration Curve on Test Set');
```

```python
mli.plot_reliability_diagram(y_test_1, testset_splinecalib_probs);
plt.title('Reliability Diagram on Test Data\n after SplineCalib Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_splinecalib_probs, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for SplineCalib Calibrated Model');
```

```python

```

```python

```

```python

```

```python
print('Spline calibrated log_loss = {}'.format(log_loss(y_test_1, testset_splinecalib_probs)))
print('Beta calibrated log_loss = {}'.format(log_loss(y_test_1, testset_bc_probs)))
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Spline calibrated log_loss = 0.2933565201257039
    Beta calibrated log_loss = 0.30257500027883283
    Isotonic calibrated log_loss = 0.31568907061162055
    Platt calibrated log_loss = 0.29833851247471244
    Uncalibrated log_loss = 0.31209048301332465

On this example, SplineCalib does best, though Beta calibration and Platt scaling both do reasonably well (and are quicker to fit).  Isotonic does relatively poorly.  Note that we used only about 3K data points in our calibration set - Isotonic regression is relatively "data-hungry"

Below, we can see the results measured by Brier Score loss.  Note that there may be a very different pattern here in relative performance of the methods.

```python
print('Spline calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_splinecalib_probs)))
print('Beta calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_bc_probs)))
print('Isotonic calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_iso_probs)))
print('Platt calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_platt_probs)))
print('Uncalibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_preds_uncalib_1)))
```

    Spline calibrated Brier Score = 0.08456186275218386
    Beta calibrated Brier Score = 0.08483115067208609
    Isotonic calibrated Brier Score = 0.08497485130891722
    Platt calibrated Brier Score = 0.08516411121423831
    Uncalibrated Brier Score = 0.0849214801158517

## Your Turn
Calibrate the `rfmodel2` using a couple (or all) of the methods given above.  Compare their performances.  Feel free to cut and paste from above, but try to think about the steps you are doing so it makes sense.

Note that you may get very different results from what happened for `rfmodel1`.

```python

```

```python

```

```python

```

```python

```

## Approach 2: Cross-validation on the training data
The reason to use an independent calibration set (rather than just the training data) is that how the model performs on the training data (that it has already seen) is not indicative of how it will behave on data it has not seen before.  We want the calibration to correct how the model will behave on "new" data, not the training data.

Another approach is to take a cross-validation approach to generating calibration data.  We divide the training data into k "folds", leave one fold out, train our model (i.e. the choice of model and hyperparameter settings) on the remaining k-1 folds, and then make predictions on the left-out fold.  After doing this process k times, each time leaving out a different fold, we will have a set of predictions, each of which was generated by 1 of k slightly different models, but was always generated by a model that did not see that training point.  Done properly (assuming no "leakage" across the folds), this set of predictions and answers will serve as an appropriate calibration set.

- Advantages: more data for both training *and* calibration.
- Disadvantages: Must train k+1 models.  Also, the calibration data does not come from the exact same model you will be using it on.

ML-Insights (the package containing SplineCalib, as well as other functionality) has a simple function to generate these cross-validated predictions.  We demonstrate it below.

```python
# Get the cross validated predictions given a model and training data.
cv_preds_train = mli.cv_predictions(rfmodel1, X_train_1, y_train_1, clone_model=True)
cv_preds_train1 = cv_preds_train[:,1]
```

```python

```

### Platt scaling with CV Data

```python
# Fit Platt scaling (logistic calibration)
lr_cv = LogisticRegression(C=99999999999, solver='lbfgs')
lr_cv.fit(cv_preds_train1.reshape(-1,1), y_train_1)

```

    LogisticRegression(C=99999999999, class_weight=None, dual=False,
                       fit_intercept=True, intercept_scaling=1, l1_ratio=None,
                       max_iter=100, multi_class='warn', n_jobs=None, penalty='l2',
                       random_state=None, solver='lbfgs', tol=0.0001, verbose=0,
                       warm_start=False)

```python
testset_platt_probs_cv = lr_cv.predict_proba(testset_preds_uncalib_1.reshape(-1,1))[:,1]
```

```python
mli.plot_reliability_diagram(y_train_1, cv_preds_train1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, lr.predict_proba(tvec.reshape(-1,1))[:,1], label='Platt (small calib set)')
plt.plot(tvec, lr_cv.predict_proba(tvec.reshape(-1,1))[:,1], label='Platt (cv calib set)')
plt.title('Platt Calibration Curve on Calibration Data');
plt.legend();
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, lr.predict_proba(tvec.reshape(-1,1))[:,1], label='Platt (small calib set)')
plt.plot(tvec, lr_cv.predict_proba(tvec.reshape(-1,1))[:,1], label='Platt (cv calib set)')
plt.title('Platt Calibration Curve on Test Data');
plt.legend();
```

We see that the two curves are not very different.  Since Platt scaling fits just two parameters, the function does not change much with more data.  The upside is that you don't need a lot of calibration data to use Platt scaling!

```python
mli.plot_reliability_diagram(y_test_1, testset_platt_probs_cv);
plt.title('Reliability Diagram on Test Data\n after Platt Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_platt_probs_cv, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Platt Calibrated Model');
```

```python

```

```python
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs_cv)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Platt calibrated log_loss = 0.2980753769991345
    Uncalibrated log_loss = 0.31209048301332465

Having extra data did not really change our results

### Isotonic with CV data

```python
iso_cv = IsotonicRegression(out_of_bounds = 'clip')
iso_cv.fit(cv_preds_train1, y_train_1)
```

    IsotonicRegression(increasing=True, out_of_bounds='clip', y_max=None,
                       y_min=None)

```python
testset_iso_probs_cv = iso_cv.predict(testset_preds_uncalib_1)
```

```python
mli.plot_reliability_diagram(y_train_1, cv_preds_train1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, iso.predict(tvec), label='Isotonic (small calib set)');
plt.plot(tvec, iso_cv.predict(tvec), label='Isotonic (cv calib set)');
plt.title('Isotonic Calibration Curve on Calibration Data');
plt.legend();
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, iso.predict(tvec), label='Isotonic (small calib set)');
plt.plot(tvec, iso_cv.predict(tvec), label='Isotonic (cv calib set)');
plt.title('Isotonic Calibration Curve on Test Data');
plt.legend();
```

We see above, that with more data, the Isotonic regression fits the test data better, particularly in the middle of the range.  Qualitatively, the vertical jumps are less extreme.

```python
mli.plot_reliability_diagram(y_test_1, testset_iso_probs_cv);
plt.title('Reliability Diagram on Test Data\n after Isotonic Calibration');
```

The isotonic regression appears to have benefitted from the extra calibration data.

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_iso_probs_cv, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Isotonic Calibrated Model');
```

```python

```

```python
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs_cv)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs_cv)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Isotonic calibrated log_loss = 0.3031159383649809
    Platt calibrated log_loss = 0.2980753769991345
    Uncalibrated log_loss = 0.31209048301332465

With the larger data set we see much better performance from isotonic regression.

### Beta with CV data

```python
# Fit three-parameter beta calibration
bc_cv = BetaCalibration()
bc_cv.fit(cv_preds_train1, y_train_1)

```

    BetaCalibration(parameters=None)

```python
testset_bc_probs_cv = bc_cv.predict(testset_preds_uncalib_1)

```

```python
mli.plot_reliability_diagram(y_train_1, cv_preds_train1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, bc.predict(tvec), label='Beta (small calib set)')
plt.plot(tvec, bc_cv.predict(tvec), label='Beta (cv calib set)')
plt.title('Beta Calibration Curve on Calibration Data');
plt.legend();
```

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, bc.predict(tvec), label='Beta (small calib set)')
plt.plot(tvec, bc_cv.predict(tvec), label='Beta (cv calib set)')
plt.title('Beta Calibration Curve on Test Set');
plt.legend();
```

We see a small difference in the curves, but not a huge change from having 3K points vs 45K points to learn from

```python
mli.plot_reliability_diagram(y_test_1, testset_bc_probs_cv);
plt.title('Reliability Diagram on Test Data\n after Beta Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_bc_probs_cv, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for Beta Calibrated Model');
```

Not much of a change from the smaller calib set

```python
print('Beta calibrated log_loss = {}'.format(log_loss(y_test_1, testset_bc_probs_cv)))
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs_cv)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs_cv)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Beta calibrated log_loss = 0.2972470914743036
    Isotonic calibrated log_loss = 0.3031159383649809
    Platt calibrated log_loss = 0.2980753769991345
    Uncalibrated log_loss = 0.31209048301332465

Now, Beta calibration does slightly better than Platt scaling.  Isotonic has improved.

### SplineCalib with CV data

```python
splinecalib_cv = mli.SplineCalib()
splinecalib_cv.fit(cv_preds_train, y_train_1)
```

```python
testset_splinecalib_probs_cv = splinecalib_cv.predict(testset_preds_uncalib_1)
```

```python
mli.plot_reliability_diagram(y_train_1, cv_preds_train1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, splinecalib.predict(tvec), label='SplineCalib (small calib set)')
plt.plot(tvec, splinecalib_cv.predict(tvec), label='SplineCalib (cv calib set)')
plt.title('SplineCalib Calibration Curve on Calibration Data');
plt.legend();
```

With the larger data set, the calibration curve is now close to the line y=x

```python
mli.plot_reliability_diagram(y_test_1, testset_preds_uncalib_1, error_bars=False);
tvec = np.linspace(.01, .99, 99)
plt.plot(tvec, splinecalib.predict(tvec), label='SplineCalib (small calib set)')
plt.plot(tvec, splinecalib_cv.predict(tvec), label='SplineCalib (cv calib set)')
plt.title('SplineCalib Calibration Curve on Test Set');
plt.legend();
```

```python
mli.plot_reliability_diagram(y_test_1, testset_splinecalib_probs_cv);
plt.title('Reliability Diagram on Test Data\n after Spline Calibration');
```

```python
custom_bins_a = np.array([0,.01,.02,.03,.05, .1, .3, .5, .75, 1])
rd = mli.plot_reliability_diagram(y_test_1, testset_splinecalib_probs_cv, scaling='logit', bins=custom_bins_a);
plt.title('Reliability Diagram on Test Data\n for SplineCalib Calibrated Model');
```

The points are very close to the line y=x

```python

```

```python
print('Spline calibrated log_loss = {}'.format(log_loss(y_test_1, testset_splinecalib_probs_cv)))
print('Beta calibrated log_loss = {}'.format(log_loss(y_test_1, testset_bc_probs_cv)))
print('Isotonic calibrated log_loss = {}'.format(log_loss(y_test_1, testset_iso_probs_cv)))
print('Platt calibrated log_loss = {}'.format(log_loss(y_test_1, testset_platt_probs_cv)))
print('Uncalibrated log_loss = {}'.format(log_loss(y_test_1, testset_preds_uncalib_1)))
```

    Spline calibrated log_loss = 0.2938099296324512
    Beta calibrated log_loss = 0.2972470914743036
    Isotonic calibrated log_loss = 0.3031159383649809
    Platt calibrated log_loss = 0.2980753769991345
    Uncalibrated log_loss = 0.31209048301332465

```python
print('Spline calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_splinecalib_probs_cv)))
print('Beta calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_bc_probs_cv)))
print('Isotonic calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_iso_probs_cv)))
print('Platt calibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_platt_probs_cv)))
print('Uncalibrated Brier Score = {}'.format(brier_score_loss(y_test_1, testset_preds_uncalib_1)))
```

    Spline calibrated Brier Score = 0.08464770699590742
    Beta calibrated Brier Score = 0.085288767598558
    Isotonic calibrated Brier Score = 0.08480539303012971
    Platt calibrated Brier Score = 0.08511128351457842
    Uncalibrated Brier Score = 0.0849214801158517

```python

```

## Your Turn

- Use the `mli.cv_predictions` functions on `rfmodel2` to create a cross-validated set on which to fit your calibration.

- Then calibrate `rfmodel2` using this set, rather than the separate calibration set.  Compare the results.

- How do the results compare to `rfmodel1`?

## Extra Credit
- Make a copy of the original notebook.  Change the random state `rs` in cell [5] and then do "Kernel->Restart and Run All".  Observe how the relative performance changes for the different methods both using the calibration set and the cross-validated approach

- Adjust the training and calibration sizes and see how it affects the performances.

### Some things you might notice
- Isotonic Regression can be quite variable - especially with smaller calibration sets.
- Isotonic Regression tends to improve quite a bit as it gets more data
- Beta Calibration tends to beat Platt Scaling (though not always)
- The rankings for log-loss are often not the same as for Brier Score

```python

```

```python

```