# 1. Loading data

```python
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Load subset of the training data
X_train = pd.read_csv('data/train.csv', nrows=1000000, parse_dates=['click_time'])

# Show the head of the table
X_train.head()
```

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0

# 2. Creating Features
## 2.1 Extracting time information
First extract day, minute, hour, second from the click_time.

```python
X_train['day'] = X_train['click_time'].dt.day.astype('uint8')
X_train['hour'] = X_train['click_time'].dt.hour.astype('uint8')
X_train['minute'] = X_train['click_time'].dt.minute.astype('uint8')
X_train['second'] = X_train['click_time'].dt.second.astype('uint8')
X_train.head()
```

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed
      day
      hour
      minute
      second

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0
      6
      14
      32
      21

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0
      6
      14
      33
      34

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0
      6
      14
      34
      12

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0
      6
      14
      34
      52

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0
      6
      14
      35
      8

```python
day = X_train['day'].value_counts()
day.plot(kind='barh')
```

```python
hour = X_train['hour'].value_counts()
hour.plot(kind='barh')
```

```python
minute = X_train['minute'].value_counts()
minute.plot(kind='barh')
```

```python
second = X_train['second'].value_counts()
second.plot(kind='barh')
```

## 2.2. Confidence Rates for is_attributed
My thought is that some ips, apps, devices, etc. might have higher frequencies of is_attributed, and I wish to add that information, i.e. I'm calculating the following "attributed rates":

\begin{equation}
\text{P}\,\text{(is_attributed}\,\,|\,\,\text{category)}
\end{equation}

or in some cases two- or multiple-paired combinations:

\begin{equation}
\text{P}\,\text{(is_attributed}\,\,|\,\,\text{category_1, category_2)}
\end{equation}

The danger of this is that if a given category-combination has very few clicks, then the statistical significance of above equations cannot be trusted. Therefore I'll be weighing the rates by the following confidence rates:

\begin{equation}
    \text{conf}_{\text{is_attributed}} = \frac{\log(\text{views}_{\text{category_1}})}{\log(100000)}
\end{equation}

where the value 100000 has been chosen arbitrarily to such that if a given category has 1000 views, then it gets a confidence weight of 60%, if it has 100 views then onfly a confidence weight of 40% etc.

```python
ATTRIBUTION_CATEGORIES = [
    # V1 Features #
    ###############
    ['ip'], ['app'], ['device'], ['os'], ['channel'],

    # V2 Features #
    ###############
    ['app', 'channel'],
    ['app', 'os'],
    ['app', 'device'],

    # V3 Features #
    ###############
    ['channel', 'os'],
    ['channel', 'device'],
    ['os', 'device']
]

covs = []
corrs = []
# Find frequency of is_attributed for each unique value in column
freqs = {}
for cols in ATTRIBUTION_CATEGORIES:

    # New feature name
    new_feature = '_'.join(cols)+'_confRate'
    covs.append(new_feature)
    corrs.append(new_feature)

    # Perform the groupby
    group_object = X_train.groupby(cols)

    # Group sizes
    group_sizes = group_object.size()
    log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence
    print(">> Calculating confidence-weighted rate for: {}.\n   Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}".format(
        cols, new_feature,
        group_sizes.max(),
        np.round(group_sizes.mean(), 2),
        np.round(group_sizes.median(), 2),
        group_sizes.min()
    ))

    # Aggregation function
    def rate_calculation(x):
        """Calculate the attributed rate. Scale by confidence"""
        rate = x.sum() / float(x.count())
        conf = np.min([1, np.log(x.count()) / log_group])
        return rate * conf

    # Perform the merge
    X_train = X_train.merge(
        group_object['is_attributed']. \
            apply(rate_calculation). \
            reset_index(). \
            rename(
                index=str,
                columns={'is_attributed': new_feature}
            )[cols + [new_feature]],
        on=cols, how='left'
    )

X_train.head()
X_train
```

    >> Calculating confidence-weighted rate for: ['ip'].
       Saving to: ip_confRate. Group Max /Mean / Median / Min: 6437 / 25.25 / 10.0 / 1
    >> Calculating confidence-weighted rate for: ['app'].
       Saving to: app_confRate. Group Max /Mean / Median / Min: 141851 / 4672.9 / 12.0 / 1
    >> Calculating confidence-weighted rate for: ['device'].
       Saving to: device_confRate. Group Max /Mean / Median / Min: 938053 / 3355.7 / 1.0 / 1
    >> Calculating confidence-weighted rate for: ['os'].
       Saving to: os_confRate. Group Max /Mean / Median / Min: 239642 / 6250.0 / 38.0 / 1
    >> Calculating confidence-weighted rate for: ['channel'].
       Saving to: channel_confRate. Group Max /Mean / Median / Min: 92534 / 6493.51 / 1200.0 / 1
    >> Calculating confidence-weighted rate for: ['app', 'channel'].
       Saving to: app_channel_confRate. Group Max /Mean / Median / Min: 61413 / 1915.71 / 18.0 / 1
    >> Calculating confidence-weighted rate for: ['app', 'os'].
       Saving to: app_os_confRate. Group Max /Mean / Median / Min: 34853 / 352.24 / 8.0 / 1
    >> Calculating confidence-weighted rate for: ['app', 'device'].
       Saving to: app_device_confRate. Group Max /Mean / Median / Min: 134290 / 1342.28 / 2.0 / 1
    >> Calculating confidence-weighted rate for: ['channel', 'os'].
       Saving to: channel_os_confRate. Group Max /Mean / Median / Min: 22462 / 174.76 / 15.0 / 1
    >> Calculating confidence-weighted rate for: ['channel', 'device'].
       Saving to: channel_device_confRate. Group Max /Mean / Median / Min: 87519 / 1300.39 / 3.0 / 1
    >> Calculating confidence-weighted rate for: ['os', 'device'].
       Saving to: os_device_confRate. Group Max /Mean / Median / Min: 233133 / 1721.17 / 3.0 / 1

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed
      day
      hour
      ...
      app_confRate
      device_confRate
      os_confRate
      channel_confRate
      app_channel_confRate
      app_os_confRate
      app_device_confRate
      channel_os_confRate
      channel_device_confRate
      os_device_confRate

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0
      6
      14
      ...
      0.000431
      0.001193
      0.001146
      0.004071
      0.004164
      0.000201
      0.000445
      0.001835
      0.004109
      0.001182

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      5
      18787
      3
      1
      16
      379
      2017-11-06 14:36:26
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000851
      0.000497
      0.000360
      0.000559
      0.000444
      0.002329
      0.000499
      0.000882

      6
      103022
      3
      1
      23
      379
      2017-11-06 14:37:44
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000328
      0.000497
      0.000360
      0.000000
      0.000444
      0.000000
      0.000499
      0.000336

      7
      114221
      3
      1
      19
      379
      2017-11-06 14:37:59
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      8
      165970
      3
      1
      13
      379
      2017-11-06 14:38:10
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      9
      74544
      64
      1
      22
      459
      2017-11-06 14:38:23
      NaN
      0
      6
      14
      ...
      0.000256
      0.001193
      0.001859
      0.000161
      0.000256
      0.000000
      0.000260
      0.000000
      0.000165
      0.001944

      10
      172522
      3
      1
      25
      379
      2017-11-06 14:38:27
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000922
      0.000497
      0.000360
      0.000212
      0.000444
      0.000000
      0.000499
      0.000956

      11
      105861
      3
      1
      13
      379
      2017-11-06 14:38:51
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      12
      210962
      3
      1
      19
      379
      2017-11-06 14:39:29
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      13
      124979
      3
      1
      18
      379
      2017-11-06 14:40:16
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000914
      0.000497
      0.000360
      0.000318
      0.000444
      0.000000
      0.000499
      0.000925

      14
      38816
      3
      1
      18
      379
      2017-11-06 14:40:39
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000914
      0.000497
      0.000360
      0.000318
      0.000444
      0.000000
      0.000499
      0.000925

      15
      80447
      3
      1
      19
      379
      2017-11-06 14:40:51
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      16
      134575
      3
      1
      13
      379
      2017-11-06 14:43:10
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      17
      57576
      3
      1
      19
      379
      2017-11-06 14:43:14
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      18
      7755
      3
      1
      13
      379
      2017-11-06 14:43:25
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      19
      91749
      3
      1
      19
      379
      2017-11-06 14:43:51
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      20
      50694
      3
      1
      18
      379
      2017-11-06 14:44:02
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000914
      0.000497
      0.000360
      0.000318
      0.000444
      0.000000
      0.000499
      0.000925

      21
      111385
      3
      1
      18
      379
      2017-11-06 14:44:15
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000914
      0.000497
      0.000360
      0.000318
      0.000444
      0.000000
      0.000499
      0.000925

      22
      125222
      14
      1
      3
      478
      2017-11-06 14:44:51
      NaN
      0
      6
      14
      ...
      0.000431
      0.001193
      0.000766
      0.004071
      0.004164
      0.000000
      0.000445
      0.000000
      0.004109
      0.000789

      23
      191759
      3
      1
      13
      379
      2017-11-06 14:44:51
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      24
      53418
      3
      1
      17
      379
      2017-11-06 14:44:52
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000893
      0.000497
      0.000360
      0.000304
      0.000444
      0.000000
      0.000499
      0.000944

      25
      183168
      3
      1
      19
      379
      2017-11-06 14:47:49
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      26
      47902
      3
      1
      17
      379
      2017-11-06 14:48:07
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000893
      0.000497
      0.000360
      0.000304
      0.000444
      0.000000
      0.000499
      0.000944

      27
      209663
      3
      1
      13
      379
      2017-11-06 14:48:55
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001146
      0.000497
      0.000360
      0.000366
      0.000444
      0.000000
      0.000499
      0.001182

      28
      129614
      3
      1
      20
      379
      2017-11-06 14:49:36
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.000958
      0.000497
      0.000360
      0.000220
      0.000444
      0.002091
      0.000499
      0.001000

      29
      208347
      3
      1
      19
      379
      2017-11-06 14:49:38
      NaN
      0
      6
      14
      ...
      0.000446
      0.001193
      0.001394
      0.000497
      0.000360
      0.000275
      0.000444
      0.000235
      0.000499
      0.001428

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      999970
      32115
      18
      1
      22
      134
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000369
      0.001193
      0.001859
      0.000562
      0.000700
      0.000478
      0.000402
      0.002892
      0.000583
      0.001944

      999971
      28113
      15
      1
      13
      245
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000210
      0.001193
      0.001146
      0.000064
      0.000078
      0.000069
      0.000220
      0.000042
      0.000068
      0.001182

      999972
      175237
      18
      1
      19
      439
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000369
      0.001193
      0.001394
      0.000470
      0.000351
      0.000737
      0.000402
      0.000255
      0.000471
      0.001428

      999973
      114656
      3
      1
      36
      137
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.000000
      0.000393
      0.000347
      0.000000
      0.000444
      0.000000
      0.000434
      0.000000

      999974
      43793
      6
      1
      18
      125
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000085
      0.001193
      0.000914
      0.000063
      0.000069
      0.000523
      0.000087
      0.000000
      0.000065
      0.000925

      999975
      166358
      26
      1
      12
      266
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000564
      0.001193
      0.000753
      0.001060
      0.001060
      0.000000
      0.000566
      0.000000
      0.001063
      0.000778

      999976
      67494
      9
      1
      19
      234
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.001193
      0.001394
      0.000749
      0.000749
      0.000841
      0.000888
      0.000988
      0.000859
      0.001428

      999977
      73516
      3
      1
      13
      280
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.001146
      0.000351
      0.000320
      0.000366
      0.000444
      0.000625
      0.000304
      0.001182

      999978
      109744
      9
      1
      20
      234
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.001193
      0.000958
      0.000749
      0.000749
      0.000973
      0.000888
      0.000000
      0.000859
      0.001000

      999979
      61706
      3
      1
      13
      280
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.001146
      0.000351
      0.000320
      0.000366
      0.000444
      0.000625
      0.000304
      0.001182

      999980
      141925
      28
      1
      19
      135
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000145
      0.001193
      0.001394
      0.000303
      0.000145
      0.000000
      0.000145
      0.000000
      0.000309
      0.001428

      999981
      93155
      12
      1
      19
      178
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000134
      0.001193
      0.001394
      0.000147
      0.000157
      0.000104
      0.000141
      0.000216
      0.000157
      0.001428

      999982
      15530
      9
      1
      18
      466
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.001193
      0.000914
      0.000865
      0.000757
      0.000256
      0.000888
      0.000000
      0.000867
      0.000925

      999983
      2698
      25
      1
      30
      259
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000101
      0.001193
      0.000144
      0.000519
      0.000101
      0.000000
      0.000104
      0.000000
      0.000574
      0.000155

      999984
      29290
      3
      1
      10
      480
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.000962
      0.000174
      0.000162
      0.000000
      0.000444
      0.000000
      0.000175
      0.000995

      999985
      38277
      3
      1
      12
      137
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.000753
      0.000393
      0.000347
      0.000000
      0.000444
      0.000000
      0.000434
      0.000778

      999986
      131635
      24
      1
      1
      105
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000315
      0.001193
      0.001424
      0.000333
      0.000339
      0.000000
      0.000389
      0.000000
      0.000413
      0.001470

      999987
      43698
      18
      1
      19
      121
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000369
      0.001193
      0.001394
      0.000271
      0.000280
      0.000737
      0.000402
      0.000547
      0.000280
      0.001428

      999988
      27838
      12
      1
      19
      265
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000134
      0.001193
      0.001394
      0.000079
      0.000048
      0.000104
      0.000141
      0.000000
      0.000082
      0.001428

      999989
      70271
      2
      1
      6
      435
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000262
      0.001193
      0.001548
      0.000100
      0.000100
      0.000223
      0.000278
      0.000000
      0.000101
      0.001592

      999990
      76788
      3
      1
      12
      205
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.000753
      0.000192
      0.000492
      0.000000
      0.000444
      0.000000
      0.000216
      0.000778

      999991
      191894
      24
      2
      15
      105
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000315
      0.000171
      0.000604
      0.000333
      0.000339
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000

      999992
      105560
      3
      1
      19
      137
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000446
      0.001193
      0.001394
      0.000393
      0.000347
      0.000275
      0.000444
      0.000000
      0.000434
      0.001428

      999993
      73516
      9
      2
      13
      134
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.000171
      0.001146
      0.000562
      0.000655
      0.000809
      0.000214
      0.000418
      0.000000
      0.000253

      999994
      6578
      9
      1
      18
      244
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.001193
      0.000914
      0.001919
      0.001919
      0.000256
      0.000888
      0.000000
      0.002049
      0.000925

      999995
      29748
      9
      1
      12
      134
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000848
      0.001193
      0.000753
      0.000562
      0.000655
      0.000000
      0.000888
      0.000000
      0.000583
      0.000778

      999996
      124520
      12
      1
      15
      178
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000134
      0.001193
      0.000604
      0.000147
      0.000157
      0.000000
      0.000141
      0.000000
      0.000157
      0.000628

      999997
      206446
      18
      1
      42
      107
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000369
      0.001193
      0.000000
      0.000318
      0.000318
      0.000000
      0.000402
      0.000000
      0.000383
      0.000000

      999998
      167577
      12
      1
      13
      265
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000134
      0.001193
      0.001146
      0.000079
      0.000048
      0.000081
      0.000141
      0.000000
      0.000082
      0.001182

      999999
      121848
      24
      1
      19
      105
      2017-11-06 16:21:51
      NaN
      0
      6
      16
      ...
      0.000315
      0.001193
      0.001394
      0.000333
      0.000339
      0.000394
      0.000389
      0.000416
      0.000413
      0.001428

1000000 rows × 67 columns

```python
covsp = {}
for c in covs:
    sub_df = X_train[[c, 'is_attributed']]
    covsp[c] = sub_df[c].cov(sub_df.is_attributed)
covsp
```

    {'app_channel_confRate': 0.00019634360938933638,
     'app_confRate': 0.00020067377284722137,
     'app_device_confRate': 0.0001803796749384625,
     'app_os_confRate': 0.00014759553678794408,
     'channel_confRate': 0.00016506487130292825,
     'channel_device_confRate': 0.0001415220356796749,
     'channel_os_confRate': 0.0001149070616583554,
     'device_confRate': 3.8231808785780286e-05,
     'ip_confRate': 2.0701035635506238e-05,
     'os_confRate': 4.7006330764345326e-05,
     'os_device_confRate': 3.514337207089287e-05}

```python
# histogram of covariance of each new column
plt.bar(covsp.keys(), covsp.values())
plt.xticks(rotation=90)
```

```python
corrsp = {}
for c in corrs:
    sub_df = X_train[[c, 'is_attributed']]
    corrsp[c] = sub_df[c].cov(sub_df.is_attributed)
corrsp
```

    {'app_channel_confRate': 0.00019634360938933638,
     'app_confRate': 0.00020067377284722137,
     'app_device_confRate': 0.0001803796749384625,
     'app_os_confRate': 0.00014759553678794408,
     'channel_confRate': 0.00016506487130292825,
     'channel_device_confRate': 0.0001415220356796749,
     'channel_os_confRate': 0.0001149070616583554,
     'device_confRate': 3.8231808785780286e-05,
     'ip_confRate': 2.0701035635506238e-05,
     'os_confRate': 4.7006330764345326e-05,
     'os_device_confRate': 3.514337207089287e-05}

```python
# histogram of correlation of each column
plt.bar(corrsp.keys(), corrsp.values())
plt.xticks(rotation=90)
```

## 2.3. Group-By-Aggregation

```python
# Define all the groupby transformations
GROUPBY_AGGREGATIONS = [

    # V1 - GroupBy Features #
    #########################
    # Variance in day, for ip-app-channel
    {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},
    # Variance in hour, for ip-app-os
    {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},
    # Variance in hour, for ip-day-channel
    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},
    # Count, for ip-day-hour
    {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},
    # Count, for ip-app
    {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},
    # Count, for ip-app-os
    {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},
    # Count, for ip-app-day-hour
    {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},
    # Mean hour, for ip-app-channel
    {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'},

    # V2 - GroupBy Features #
    #########################
    # Average clicks on app by distinct users; is it an app they return to?
    {'groupby': ['app'],
     'select': 'ip',
     'agg': lambda x: float(len(x)) / len(x.unique()),
     'agg_name': 'AvgViewPerDistinct'
    },
    # How popular is the app or channel?
    {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},
    {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},

    # V3 - GroupBy Features                                              #
    # https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 #
    ######################################################################
    {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'},
    {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'},
    {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'},
    {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'},
    {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'},
    {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'},
    {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'},
    {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'},
    {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'},
    {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'},
    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'}
]
covs = []
corrs = []
# Apply all the groupby transformations
for spec in GROUPBY_AGGREGATIONS:

    # Name of the aggregation we're applying
    agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']

    # Name of new feature
    new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])
    covs.append(new_feature)
    corrs.append(new_feature)

    # Info
    print("Grouping by {}, and aggregating {} with {}".format(
        spec['groupby'], spec['select'], agg_name
    ))

    # Unique list of features to select
    all_features = list(set(spec['groupby'] + [spec['select']]))

    # Perform the groupby
    gp = X_train[all_features]. \
        groupby(spec['groupby'])[spec['select']]. \
        agg(spec['agg']). \
        reset_index(). \
        rename(index=str, columns={spec['select']: new_feature})

    # Merge back to X_total
    if 'cumcount' == spec['agg']:
        X_train[new_feature] = gp[0].values
    else:
        X_train = X_train.merge(gp, on=spec['groupby'], how='left')

     # Clear memory
    del gp
    gc.collect()

X_train.head()
```

    Grouping by ['ip', 'app', 'channel'], and aggregating day with var
    Grouping by ['ip', 'app', 'os'], and aggregating hour with var
    Grouping by ['ip', 'day', 'channel'], and aggregating hour with var
    Grouping by ['ip', 'day', 'hour'], and aggregating channel with count
    Grouping by ['ip', 'app'], and aggregating channel with count
    Grouping by ['ip', 'app', 'os'], and aggregating channel with count
    Grouping by ['ip', 'app', 'day', 'hour'], and aggregating channel with count
    Grouping by ['ip', 'app', 'channel'], and aggregating hour with mean
    Grouping by ['app'], and aggregating ip with AvgViewPerDistinct
    Grouping by ['app'], and aggregating channel with count
    Grouping by ['channel'], and aggregating app with count
    Grouping by ['ip'], and aggregating channel with nunique
    Grouping by ['ip'], and aggregating app with nunique
    Grouping by ['ip', 'day'], and aggregating hour with nunique
    Grouping by ['ip', 'app'], and aggregating os with nunique
    Grouping by ['ip'], and aggregating device with nunique
    Grouping by ['app'], and aggregating channel with nunique
    Grouping by ['ip', 'device', 'os'], and aggregating app with nunique
    Grouping by ['ip', 'device', 'os'], and aggregating app with cumcount
    Grouping by ['ip'], and aggregating app with cumcount
    Grouping by ['ip'], and aggregating os with cumcount
    Grouping by ['ip', 'day', 'channel'], and aggregating hour with var

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed
      day
      hour
      ...
      ip_nunique_app
      ip_day_nunique_hour
      ip_app_nunique_os
      ip_nunique_device
      app_nunique_channel
      ip_device_os_nunique_app
      ip_device_os_cumcount_app
      ip_cumcount_app
      ip_cumcount_os
      ip_day_channel_var_hour_y

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0
      6
      14
      ...
      19
      2
      5
      1
      34
      16
      0
      0
      0
      1.333333

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0
      6
      14
      ...
      20
      3
      8
      1
      34
      11
      0
      0
      0
      1.000000

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0
      6
      14
      ...
      17
      2
      6
      3
      34
      10
      0
      0
      0
      2.000000

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0
      6
      14
      ...
      31
      2
      18
      6
      26
      24
      0
      0
      0
      1.000000

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0
      6
      14
      ...
      1
      1
      1
      1
      34
      1
      0
      0
      0
      NaN

5 rows × 45 columns

```python
covsp = {}
for c in covs:
    sub_df = X_train[[c, 'is_attributed']]
    covsp[c] = sub_df[c].cov(sub_df.is_attributed)
covsp
```

    {'app_channel_confRate': 0.00019634360938933638,
     'app_confRate': 0.00020067377284722137,
     'app_device_confRate': 0.0001803796749384625,
     'app_os_confRate': 0.00014759553678794408,
     'channel_confRate': 0.00016506487130292825,
     'channel_device_confRate': 0.0001415220356796749,
     'channel_os_confRate': 0.0001149070616583554,
     'device_confRate': 3.8231808785780286e-05,
     'ip_confRate': 2.0701035635506238e-05,
     'os_confRate': 4.7006330764345326e-05,
     'os_device_confRate': 3.514337207089287e-05}

```python
# histogram of covariance of each new column
plt.bar(covsp.keys(), covsp.values())
plt.xticks(rotation=90)
```

```python
corrsp = {}
for c in corrs:
    sub_df = X_train[[c, 'is_attributed']]
    corrsp[c] = sub_df[c].cov(sub_df.is_attributed)
corrsp
```

    {'app_channel_confRate': 0.00019634360938933638,
     'app_confRate': 0.00020067377284722137,
     'app_device_confRate': 0.0001803796749384625,
     'app_os_confRate': 0.00014759553678794408,
     'channel_confRate': 0.00016506487130292825,
     'channel_device_confRate': 0.0001415220356796749,
     'channel_os_confRate': 0.0001149070616583554,
     'device_confRate': 3.8231808785780286e-05,
     'ip_confRate': 2.0701035635506238e-05,
     'os_confRate': 4.7006330764345326e-05,
     'os_device_confRate': 3.514337207089287e-05}

```python
# histogram of correlation of each column
plt.bar(corrsp.keys(), corrsp.values())
plt.xticks(rotation=90)
```

# 2.4. Time till next click
It might be interesting to know e.g. how long it takes for a given ip-app-channel before they perform the next click. So I'll create some features for these as well. ****

```python
GROUP_BY_NEXT_CLICKS = [

    # V1
    {'groupby': ['ip']},
    {'groupby': ['ip', 'app']},
    {'groupby': ['ip', 'channel']},
    {'groupby': ['ip', 'os']},

    # V3
    {'groupby': ['ip', 'app', 'device', 'os', 'channel']},
    {'groupby': ['ip', 'os', 'device']},
    {'groupby': ['ip', 'os', 'device', 'app']}
]
covs = []
corrs = []
# Calculate the time to next click for each group
for spec in GROUP_BY_NEXT_CLICKS:
    # Name of new feature
    new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))

    covs.append(new_feature)
    corrs.append(new_feature)

    # Unique list of features to select
    all_features = spec['groupby'] + ['click_time']

    # Run calculation
    print(f">> Grouping by {spec['groupby']}, and saving time to next click in: {new_feature}")
    X_train[new_feature] = X_train[all_features].groupby(spec['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds

X_train.head()
```

    >> Grouping by ['ip'], and saving time to next click in: ip_nextClick
    >> Grouping by ['ip', 'app'], and saving time to next click in: ip_app_nextClick
    >> Grouping by ['ip', 'channel'], and saving time to next click in: ip_channel_nextClick
    >> Grouping by ['ip', 'os'], and saving time to next click in: ip_os_nextClick
    >> Grouping by ['ip', 'app', 'device', 'os', 'channel'], and saving time to next click in: ip_app_device_os_channel_nextClick
    >> Grouping by ['ip', 'os', 'device'], and saving time to next click in: ip_os_device_nextClick
    >> Grouping by ['ip', 'os', 'device', 'app'], and saving time to next click in: ip_os_device_app_nextClick

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed
      day
      hour
      ...
      channel_os_confRate
      channel_device_confRate
      os_device_confRate
      ip_nextClick
      ip_app_nextClick
      ip_channel_nextClick
      ip_os_nextClick
      ip_app_device_os_channel_nextClick
      ip_os_device_nextClick
      ip_os_device_app_nextClick

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0
      6
      14
      ...
      0.000000
      0.000499
      0.001182
      5290.0
      5340.0
      5444.0
      5307.0
      NaN
      5307.0
      5340.0

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0
      6
      14
      ...
      0.000235
      0.000499
      0.001428
      5177.0
      5177.0
      5177.0
      5239.0
      NaN
      5239.0
      5547.0

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0
      6
      14
      ...
      0.000000
      0.000499
      0.001182
      5175.0
      5175.0
      6005.0
      5205.0
      NaN
      5205.0
      5925.0

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0
      6
      14
      ...
      0.001835
      0.004109
      0.001182
      5108.0
      5110.0
      5137.0
      5108.0
      NaN
      5108.0
      5110.0

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0
      6
      14
      ...
      0.000000
      0.000499
      0.001182
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

5 rows × 74 columns

```python
covsp = {}
for c in covs:
    sub_df = X_train[[c, 'is_attributed']]
    covsp[c] = sub_df[c].cov(sub_df.is_attributed)
covsp
```

    {'ip_app_device_os_channel_nextClick': 0.013632224912572783,
     'ip_app_nextClick': 0.02012430954452766,
     'ip_channel_nextClick': 0.021197197943624572,
     'ip_nextClick': 0.07429679676171368,
     'ip_os_device_app_nextClick': 0.014365914228217959,
     'ip_os_device_nextClick': 0.03967463776528343,
     'ip_os_nextClick': 0.041616729090613515}

```python
# histogram of covariance of each new column
plt.bar(covsp.keys(), covsp.values())
plt.xticks(rotation=90)
```

```python
corrsp = {}
for c in corrs:
    sub_df = X_train[[c, 'is_attributed']]
    corrsp[c] = sub_df[c].cov(sub_df.is_attributed)
corrsp
```

    {'ip_app_device_os_channel_nextClick': 0.013632224912572783,
     'ip_app_nextClick': 0.02012430954452766,
     'ip_channel_nextClick': 0.021197197943624572,
     'ip_nextClick': 0.07429679676171368,
     'ip_os_device_app_nextClick': 0.014365914228217959,
     'ip_os_device_nextClick': 0.03967463776528343,
     'ip_os_nextClick': 0.041616729090613515}

```python
# histogram of correlation of each column
plt.bar(corrsp.keys(), corrsp.values())
plt.xticks(rotation=90)
```

## 2.5. Clicks on app ad before & after
Has the user previously or subsequently clicked the exact same app-device-os-channel? I thought that might be an interesting feature to test out as well.

```python
HISTORY_CLICKS = {
    'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],
    'app_clicks': ['ip', 'app']
}

covs = []
corrs = []
# Go through different group-by combinations
for fname, fset in HISTORY_CLICKS.items():
    covs.append('prev_'+fname)
    corrs.append('prev_'+fname)

    covs.append('future_'+fname)
    corrs.append('future_'+fname)

    # Clicks in the past
    X_train['prev_'+fname] = X_train. \
        groupby(fset). \
        cumcount(). \
        rename('prev_'+fname)

    # Clicks in the future
    X_train['future_'+fname] = X_train.iloc[::-1]. \
        groupby(fset). \
        cumcount(). \
        rename('future_'+fname).iloc[::-1]

# Count cumulative subsequent clicks
X_train.head()
```

      ip
      app
      device
      os
      channel
      click_time
      attributed_time
      is_attributed
      day
      hour
      ...
      ip_app_nextClick
      ip_channel_nextClick
      ip_os_nextClick
      ip_app_device_os_channel_nextClick
      ip_os_device_nextClick
      ip_os_device_app_nextClick
      prev_identical_clicks
      future_identical_clicks
      prev_app_clicks
      future_app_clicks

      0
      83230
      3
      1
      13
      379
      2017-11-06 14:32:21
      NaN
      0
      6
      14
      ...
      5340.0
      5444.0
      5307.0
      NaN
      5307.0
      5340.0
      0
      0
      0
      18

      1
      17357
      3
      1
      19
      379
      2017-11-06 14:33:34
      NaN
      0
      6
      14
      ...
      5177.0
      5177.0
      5239.0
      NaN
      5239.0
      5547.0
      0
      0
      0
      22

      2
      35810
      3
      1
      13
      379
      2017-11-06 14:34:12
      NaN
      0
      6
      14
      ...
      5175.0
      6005.0
      5205.0
      NaN
      5205.0
      5925.0
      0
      0
      0
      9

      3
      45745
      14
      1
      13
      478
      2017-11-06 14:34:52
      NaN
      0
      6
      14
      ...
      5110.0
      5137.0
      5108.0
      NaN
      5108.0
      5110.0
      0
      0
      0
      68

      4
      161007
      3
      1
      13
      379
      2017-11-06 14:35:08
      NaN
      0
      6
      14
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0
      0
      0
      0

5 rows × 78 columns

```python
covsp = {}
for c in covs:
    sub_df = X_train[[c, 'is_attributed']]
    covsp[c] = sub_df[c].cov(sub_df.is_attributed)
covsp
```

    {'future_app_clicks': -0.030995220328220385,
     'future_identical_clicks': -0.004277924923924924,
     'prev_app_clicks': -0.030175219508219452,
     'prev_identical_clicks': -0.0035849242309242235}

```python
# histogram of covariance of each new column
plt.bar(covsp.keys(), covsp.values())
plt.xticks(rotation=90)
```

```python
corrsp = {}
for c in corrs:
    sub_df = X_train[[c, 'is_attributed']]
    corrsp[c] = sub_df[c].cov(sub_df.is_attributed)
corrsp
```

    {'future_app_clicks': -0.030995220328220385,
     'future_identical_clicks': -0.004277924923924924,
     'prev_app_clicks': -0.030175219508219452,
     'prev_identical_clicks': -0.0035849242309242235}

```python
# histogram of correlation of each column
plt.bar(corrsp.keys(), corrsp.values())
plt.xticks(rotation=90)
```

# 3. Evaluating Feature Importance
Having created heaps of features, I'll fit xgBoost to the data, and evaluate the feature importances. First split into X and y

```python
import xgboost as xgb

# Split into X and y
y = X_train['is_attributed']
X = X_train.drop('is_attributed', axis=1).select_dtypes(include=[np.number])

# Create a model
# Params from: https://www.kaggle.com/aharless/swetha-s-xgboost-revised
clf_xgBoost = xgb.XGBClassifier(
    max_depth = 4,
    subsample = 0.8,
    colsample_bytree = 0.7,
    colsample_bylevel = 0.7,
    scale_pos_weight = 9,
    min_child_weight = 0,
    reg_alpha = 4,
    n_jobs = 4,
    objective = 'binary:logistic'
)
# Fit the models
clf_xgBoost.fit(X, y)
```

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,
           colsample_bytree=0.7, gamma=0, learning_rate=0.1, max_delta_step=0,
           max_depth=4, min_child_weight=0, missing=None, n_estimators=100,
           n_jobs=4, nthread=None, objective='binary:logistic', random_state=0,
           reg_alpha=4, reg_lambda=1, scale_pos_weight=9, seed=None,
           silent=True, subsample=0.8)

The feature importances are MinMax scaled, put into a DataFrame, and finally plotted ordered by the mean feature importance.

```python
from sklearn import preprocessing

# Get xgBoost importances
importance_dict = {}
for import_type in ['weight', 'gain', 'cover']:
    importance_dict['xgBoost-'+import_type] = clf_xgBoost.get_booster().get_score(importance_type=import_type)

# MinMax scale all importances
importance_df = pd.DataFrame(importance_dict).fillna(0)
importance_df = pd.DataFrame(
    preprocessing.MinMaxScaler().fit_transform(importance_df),
    columns=importance_df.columns,
    index=importance_df.index
)

# Create mean column
importance_df['mean'] = importance_df.mean(axis=1)

# Plot the feature importances
importance_df.sort_values('mean').plot(kind='bar', figsize=(20, 7))
```

The code in the document is licensed under the MIT License: https://opensource.org/licenses/MIT

All writing in the document is licensed bt The Creative Commons Attribution 3.0 https://creativecommons.org/licenses/by/3.0/us/.


# Reading Files Python

Estimated time needed: **40** minutes

## Objectives

After completing this lab you will be able to:

-   Read text files using Python libraries

Table of Contents

Download Data

Reading Text Files

A Better Way to Open a File

Download Data

```python
import urllib.request
url = 'https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/PY0101EN/labs/example1.txt'
filename = 'Example1.txt'
urllib.request.urlretrieve(url, filename)
```

    ('Example1.txt', <http.client.HTTPMessage at 0x7fc204142c18>)

```python
# Download Example file

!wget -O /resources/data/Example1.txt https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/PY0101EN/labs/example1.txt
```

    --2021-01-29 11:39:14--  https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/PY0101EN/labs/example1.txt
    Resolving s3-api.us-geo.objectstorage.softlayer.net (s3-api.us-geo.objectstorage.softlayer.net)... 67.228.254.196
    Connecting to s3-api.us-geo.objectstorage.softlayer.net (s3-api.us-geo.objectstorage.softlayer.net)|67.228.254.196|:443... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 45 [text/plain]
    Saving to: ‘/resources/data/Example1.txt’

    /resources/data/Exa 100%[===================>]      45  --.-KB/s    in 0s

    2021-01-29 11:39:14 (11.6 MB/s) - ‘/resources/data/Example1.txt’ saved [45/45]

Reading Text Files

One way to read or write a file in Python is to use the built-in open function. The open function provides a File object that contains the methods and attributes you need in order to read, save, and manipulate the file. In this notebook, we will only cover .txt files. The first parameter you need is the file path and the file name. An example is shown as follow:

 The mode argument is optional and the default value is r. In this notebook we only cover two modes:

r Read mode for reading files

w Write mode for writing files

For the next example, we will use the text file Example1.txt. The file is shown as follow:

 We read the file:

```python
# Read the Example1.txt

example1 = "Example1.txt"
file1 = open(example1, "r")
```

 We can view the attributes of the file.

The name of the file:

```python
# Print the path of file

file1.name
```

    'Example1.txt'

 The mode the file object is in:

```python
# Print the mode of file, either 'r' or 'w'

file1.mode
```

    'r'

We can read the file and assign it to a variable :

```python
# Read the file

FileContent = file1.read()
FileContent
```

    'This is line 1 \nThis is line 2\nThis is line 3'

The /n means that there is a new line.

We can print the file:

```python
# Print the file with '\n' as a new line

print(FileContent)
```

    This is line 1
    This is line 2
    This is line 3

The file is of type string:

```python
# Type of file content

type(FileContent)
```

    str

It is very important that the file is closed in the end. This frees up resources and ensures consistency across different python versions.

```python
# Close file after finish

file1.close()
```

A Better Way to Open a File

Using the with statement is better practice, it automatically closes the file even if the code encounters an exception. The code will run everything in the indent block then close the file object.

```python
# Open file using with

with open(example1, "r") as file1:
    FileContent = file1.read()
    print(FileContent)
```

    This is line 1
    This is line 2
    This is line 3

The file object is closed, you can verify it by running the following cell:

```python
# Verify if the file is closed

file1.closed
```

    True

 We can see the info in the file:

```python
# See the content of file

print(FileContent)
```

    This is line 1
    This is line 2
    This is line 3

The syntax is a little confusing as the file object is after the as statement. We also don’t explicitly close the file. Therefore we summarize the steps in a figure:

We don’t have to read the entire file, for example, we can read the first 4 characters by entering three as a parameter to the method **.read()**:

```python
# Read first four characters

with open(example1, "r") as file1:
    print(file1.read(4))
```

    This

Once the method .read(4) is called the first 4 characters are called. If we call the method again, the next 4 characters are called. The output for the following cell will demonstrate the process for different inputs to the method read():

```python
# Read certain amount of characters

with open(example1, "r") as file1:
    print(file1.read(4))
    print(file1.read(4))
    print(file1.read(7))
    print(file1.read(15))
```

    This
     is
    line 1

    This is line 2

The process is illustrated in the below figure, and each color represents the part of the file read after the method read() is called:

 Here is an example using the same file, but instead we read 16, 5, and then 9 characters at a time:

```python
# Read certain amount of characters

with open(example1, "r") as file1:
    print(file1.read(16))
    print(file1.read(5))
    print(file1.read(9))
```

    This is line 1

    This
    is line 2

We can also read one line of the file at a time using the method readline():

```python
# Read one line

with open(example1, "r") as file1:
    print("first line: " + file1.readline())
```

    first line: This is line 1

We can also pass an argument to  readline()  to specify the number of charecters we want to read. However, unlike  read(),  readline() can only read one line at most.

```python
with open(example1, "r") as file1:
    print(file1.readline(20)) # does not read past the end of line
    print(file1.read(20)) # Returns the next 20 chars

```

    This is line 1

    This is line 2
    This

 We can use a loop to iterate through each line:

```python
# Iterate through the lines

with open(example1,"r") as file1:
        i = 0;
        for line in file1:
            print("Iteration", str(i), ": ", line)
            i = i + 1
```

    Iteration 0 :  This is line 1

    Iteration 1 :  This is line 2

    Iteration 2 :  This is line 3

We can use the method readlines() to save the text file to a list:

```python
# Read all lines and save as a list

with open(example1, "r") as file1:
    FileasList = file1.readlines()
```

 Each element of the list corresponds to a line of text:

```python
# Print the first line

FileasList[0]
```

    'This is line 1 \n'

# Print the second line

FileasList[1]

```python
# Print the third line

FileasList[2]
```

    'This is line 3'

 Exercise

Weather Data

Your friend, a rising star in the field of meterology, has called on you to write a script to perform some analysis on weather station data. Given below is a file "resources/ex4.csv", which contains some precipiation data for the month of June.
Each line in the file has the format - Date,Precipation (upto two decimal places). Note how the data is seperated using ','. The first row of the file contains headers and should be ignored.

Your task is to complete the getNAvg function that computes a simple moving average for N days for the precipiation data, where N is a parameter. Your function should return a list of moving averages for the given data.

The formula for a k day moving average over a series - $n_{0},n_{1},n_{2},n_{3}....n_{m}$is:
\begin{align}
M_{i} = M_{i-1} + \frac{n_{i} - n_{i-k}}{k}, \text{for i = k to m }
\\ \text{where $M_{i}$ is the moving average}
\end{align}
The skeleton code has been provided below. Edit only the required function.

Click here for the solution

```python
- Each line of the file has a '\n' char which should be removed
- The lines in the file are read as strings and need to be typecasted to floats
- For a k day moving average, The data points for the last k days must be known

```

```python
##Download the file

!wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%204/ex4.csv
```

    --2021-01-29 12:15:18--  https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%204/ex4.csv
    Resolving cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud (cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud)... 169.63.118.104
    Connecting to cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud (cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud)|169.63.118.104|:443... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 718 [text/csv]
    Saving to: ‘ex4.csv’

    ex4.csv             100%[===================>]     718  --.-KB/s    in 0.001s

    2021-01-29 12:15:18 (999 KB/s) - ‘ex4.csv’ saved [718/718]

```python
import matplotlib.pyplot as plt

statData ="ex4.csv"

def getNAvg(file,N):

    row = 0 # keep track of rows
    lastN = [] # keep track of last N points
    mean = [0] # running avg

    with open(file,"r") as rawData:
        for line in rawData:
            if (row == 0): # Ignore the headers
                row = row + 1
                continue

            line = line.strip('\n')
            lineData = float(line.split(',')[1])

            if (row<=N):
                lastN.append(lineData)
                mean[0] = (lineData + mean[0]*(row-1))/row
            else:
                mean.append( mean[row - N -1]+ (lineData - lastN[0])/N)
                lastN = lastN[1:]
                lastN.append(lineData)

            row = row +1
        return mean

    pass

def plotData(mean,N):

    mean = [round(x,3) for x in mean]
    plt.plot(mean,label=str(N) + ' day average')
    plt.xlabel('Day')
    plt.ylabel('Precipiation')
    plt.legend()

```

#### Once you have finished, you can you use the block below to plot your data

```python
plotData(getNAvg(statData,1),1)
plotData ([0 for x in range(1,5)]+ getNAvg(statData,5),5 )
plotData([0 for x in range(1,7)] + getNAvg(statData,7),7)
```

You can use the code below to verify your progress -

```python
avg5 =[4.18,4.78,4.34,4.72,5.48,5.84,6.84,6.76,6.74,5.46,4.18,2.74,2.52,2.02,2.16,2.82,2.92,4.36,4.74,5.12,5.34,6.4,6.56,6.1,5.74,5.62,4.26]
avg7 =[4.043,4.757,5.071,5.629,6.343,5.886,6.157,5.871,5.243,4.386,3.514,2.714,2.586,2.443,2.571,3.643,4.143,4.443,4.814,5.6,6.314,6.414,5.429,5.443,4.986]

def testMsg(passed):
    if passed:
       return 'Test Passed'
    else :
       return ' Test Failed'

print("getNAvg : ")
try:
    sol5 = getNAvg(statData,5)
    sol7 = getNAvg(statData,7)

    if(len(sol5)==len( avg5) and (len(sol7)==len(avg7))):
        err5 = sum([abs(avg5[index] - sol5[index])for index in range(len(avg5))])
        err7 = sum([abs(avg7[index] - sol7[index])for index in range(len(avg7))])
        print(testMsg((err5 < 1) and (err7 <1)))

    else:
        print(testMsg(False))
except NameError as e:
    print('Error! Code: {c}, Message: {m}'.format(c = type(e).__name__, m = str(e)))
except:
    print("An error occured. Recheck your function")

```

Click here for the solution

```python
import matplotlib.pyplot as plt

statData ="ex4.csv"

def getNAvg(file,N):
    """
    file - File containting all the raw weather station data
    N - The number of days to compute the moving average over

    Return a list of containg the moving average of all data points
    """
    row = 0 # keep track of rows
    lastN = [] # keep track of last N points
    mean = [0] # running avg

    with open(file,"r") as rawData:
        for line in rawData:
            if (row == 0): # Ignore the headers
                row = row + 1
                continue

            line = line.strip('\n')
            lineData = float(line.split(',')[1])

            if (row<=N):
                lastN.append(lineData)
                mean[0] = (lineData + mean[0]*(row-1))/row
            else:
                mean.append( mean[row - N -1]+ (lineData - lastN[0])/N)
                lastN = lastN[1:]
                lastN.append(lineData)

            row = row +1
        return mean

def plotData(mean,N):
        """ Plots running averages """
        mean = [round(x,3) for x in mean]
        plt.plot(mean,label=str(N) + ' day average')
        plt.xlabel('Day')
        plt.ylabel('Precipiation')
        plt.legend()

plotData(getNAvg(statData,1),1)
plotData ([0 for x in range(1,5)]+ getNAvg(statData,5),5 )
plotData([0 for x in range(1,7)] + getNAvg(statData,7),7)

```

    Note: Files with sets of data seperated using ',' or other charecters are called '.csv' files.
    They are a very common way to store data. Usually when dealing with them, An external library is used that does the nitpicky tasks for you. In fact, There are numerous libraries for statistical functions to. You will learn about such libraries later in the course.

The last exercise!

Congratulations, you have completed your first lesson and hands-on lab in Python. However, there is one more thing you need to do. The Data Science community encourages sharing work. The best way to share and showcase your work is to share it on GitHub. By sharing your notebook on GitHub you are not only building your reputation with fellow data scientists, but you can also show it off when applying for a job. Even though this was your first piece of work, it is never too early to start building good habits. So, please read and follow this article to learn how to share your work.

## Author

Joseph Santarcangelo

## Other contributors

Mavis Zhou

## Change Log

| Date (YYYY-MM-DD) | Version | Changed By    | Change Description                                        |
| ----------------- | ------- | ------------- | --------------------------------------------------------- |
| 2020-09-30        | 1.2     | Malika Singla | Weather Data dataset link added                           |
| 2020-09-30        | 1.1     | Arjun Swani   | Added exericse "Weather Data"                             |
| 2020-09-30        | 1.0     | Arjun Swani   | Added blurbs about closing files and read() vs readline() |
| 2020-08-26        | 0.2     | Lavanya       | Moved lab to course repo in GitLab                        |
|                   |         |               |                                                           |
|                   |         |               |                                                           |

##
 © IBM Corporation 2020. All rights reserved.

```python

```


##### Copyright 2018 The TF-Agents Authors.

```
#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
```

# Introduction to RL and Deep Q Networks

    View on TensorFlow.org

    Run in Google Colab

    View source on GitHub

    Download notebook

## Introduction

Reinforcement learning (RL) is a general framework where agents learn to perform actions in an environment so as to maximize a reward. The two main components are the environment, which represents the problem to be solved, and the agent, which represents the learning algorithm.

The agent and environment continuously interact with each other. At each time step, the agent takes an action on the environment based on its *policy* $\pi(a_t|s_t)$, where $s_t$ is the current observation from the environment, and receives a reward $r_{t+1}$ and the next observation $s_{t+1}$ from the environment. The goal is to improve the policy so as to maximize the sum of rewards (return).

Note: It is important to distinguish between the `state` of the environment and the `observation`, which is the part of the environment `state` that the agent can see, e.g. in a poker game, the environment state consists of the cards belonging to all the players and the community cards, but the agent can observe only its own cards and a few community cards. In most literature, these terms are used interchangeably and observation is also denoted as $s$.

This is a very general framework and can model a variety of sequential decision making problems such as games, robotics etc.

## The Cartpole Environment

The Cartpole environment is one of the most well known classic reinforcement learning problems ( the *"Hello, World!"* of RL). A pole is attached to a cart, which can move along a frictionless track. The pole starts upright and the goal is to prevent it from falling over by controlling the cart.

*   The observation from the environment $s_t$ is a 4D vector representing the position and velocity of the cart, and the angle and angular velocity of the pole.
*   The agent can control the system by taking one of 2 actions $a_t$: push the cart right (+1) or left (-1).
*   A reward $r_{t+1} = 1$ is provided for every timestep that the pole remains upright. The episode ends when one of the following is true:
  * the pole tips over some angle limit
  * the cart moves outside of the world edges
  * 200 time steps pass.

The goal of the agent is to learn a policy $\pi(a_t|s_t)$ so as to maximize the sum of rewards in an episode $\sum_{t=0}^{T} \gamma^t r_t$. Here $\gamma$ is a discount factor in $[0, 1]$ that discounts future rewards relative to immediate rewards. This parameter helps us focus the policy, making it care more about obtaining rewards quickly.

## The DQN Agent

The  [DQN (Deep Q-Network) algorithm](https://storage.googleapis.com/deepmind-media/dqn/DQNNaturePaper.pdf) was developed by DeepMind in 2015. It was able to solve a wide range of Atari games (some to superhuman level) by combining reinforcement learning and deep neural networks at scale. The algorithm was developed by enhancing a classic RL algorithm called Q-Learning with deep neural networks and a technique called *experience replay*.

### Q-Learning

Q-Learning is based on the notion of a Q-function. The Q-function (a.k.a the state-action value function) of a policy $\pi$, $Q^{\pi}(s, a)$, measures the expected return or discounted sum of rewards obtained from state $s$ by taking action $a$ first and following policy $\pi$ thereafter. We define the optimal Q-function $Q^*(s, a)$ as the maximum return that can be obtained starting from observation $s$, taking action $a$ and following the optimal policy thereafter. The optimal Q-function obeys the following *Bellman* optimality equation:

$\begin{equation}Q^\ast(s, a) = \mathbb{E}[ r + \gamma \max_{a'} Q^\ast(s', a') ]\end{equation}$

This means that the maximum return from state $s$ and action $a$ is the sum of the immediate reward $r$ and the return (discounted by $\gamma$) obtained by following the optimal policy therafter until the end of the episode (i.e., the maximum reward from the next state $s'$). The expectation is computed both over the distribution of immediate rewards $r$ and possible next states $s'$.

The basic idea behind Q-Learning is to use the Bellman optimality equation as an iterative update $Q_{i+1}(s, a) \leftarrow \mathbb{E}\left[ r + \gamma \max_{a'} Q_{i}(s', a')\right]$, and it can be shown that this converges to the optimal $Q$-function, i.e. $Q_i \rightarrow Q^*$ as $i \rightarrow \infty$ (see the [DQN paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf)).

### Deep Q-Learning

For most problems, it is impractical to represent the $Q$-function as a table containing values for each combination of $s$ and $a$. Instead, we train a function approximator, such as a neural network with parameters $\theta$, to estimate the Q-values, i.e. $Q(s, a; \theta) \approx Q^*(s, a)$. This can done by minimizing the following loss at each step $i$:

$\begin{equation}L_i(\theta_i) = \mathbb{E}_{s, a, r, s'\sim \rho(.)} \left[ (y_i - Q(s, a; \theta_i))^2 \right]\end{equation}$ where $y_i = r +  \gamma \max_{a'} Q(s', a'; \theta_{i-1})$

Here, $y_i$ is called the TD (temporal difference) target, and $y_i - Q$ is called the TD error. $\rho$ represents the behaviour distribution, the distribution over transitions $\{s, a, r, s'\}$ collected from the environment.

Note that the parameters from the previous iteration $\theta_{i-1}$ are fixed and not updated. In practice we use a snapshot of the network parameters from a few iterations ago instead of the last iteration. This copy is called the *target network*.

Q-Learning is an *off-policy* algorithm that learns about the greedy policy $a = \max_{a} Q(s, a; \theta)$ while using a different behaviour policy for acting in the environment/collecting data. This behaviour policy is usually an $\epsilon$-greedy policy that selects the greedy action with probability $1-\epsilon$ and a random action with probability $\epsilon$ to ensure good coverage of the state-action space.

### Experience Replay

To avoid computing the full expectation in the DQN loss, we can minimize it using stochastic gradient descent. If the loss is computed using just the last transition $\{s, a, r, s'\}$, this reduces to standard Q-Learning.

The Atari DQN work introduced a technique called Experience Replay to make the network updates more stable. At each time step of data collection, the transitions are added to a circular buffer called the *replay buffer*. Then during training, instead of using just the latest transition to compute the loss and its gradient, we compute them using a mini-batch of transitions sampled from the replay buffer. This has two advantages: better data efficiency by reusing each transition in many updates, and better stability using uncorrelated transitions in a batch.

## DQN on Cartpole in TF-Agents

TF-Agents provides all the components necessary to train a DQN agent, such as the agent itself, the environment, policies, networks, replay buffers, data collection loops, and metrics. These components are implemented as Python functions or TensorFlow graph ops, and we also have wrappers for converting between them. Additionally, TF-Agents supports TensorFlow 2.0 mode, which enables us to use TF in imperative mode.

Next, take a look at the [tutorial for training a DQN agent on the Cartpole environment using TF-Agents](https://github.com/tensorflow/agents/blob/master/docs/tutorials/1_dqn_tutorial.ipynb).


```
from google.colab import drive
drive.mount('/content/drive')
```

    Mounted at /content/drive

```
current_path = '/content/drive/My Drive/Colab Notebooks/Project_GANder/gankook_cuda/semantic-segmentation-pytorch/pytorch_room_layout/'

import os
os.chdir(current_path)
!ls
```

     build		        info.json	        segment_st.py
     cv2_requirements.txt   lsun.pth.tar	       'sunrgbd (1).zip'
     datasets	        lu_vp_detect.egg-info   sunrgbd.pth.tar
     data_transforms.py     pics		        test_result
     dist		        __pycache__	        Vanish_Point_Detection
     drn_d_105_024_val      README.md	        visualize.m
     drn_d_105_024_val_ms   requirements.txt        XiaohuLuVPDetection
     drn.py		        segment_lsun.py
     features	        segment_rl.py

### Get Vanishing point, inliers

```
# !python XiaohuLuVPDetection/setup.py build
# !python XiaohuLuVPDetection/setup.py install
!pip install -r cv2_requirements.txt
```

    Collecting opencv-python==4.0.0.21
      Downloading https://files.pythonhosted.org/packages/37/49/874d119948a5a084a7ebe98308214098ef3471d76ab74200f9800efeef15/opencv_python-4.0.0.21-cp36-cp36m-manylinux1_x86_64.whl (25.4MB)
         |████████████████████████████████| 25.4MB 1.6MB/s
    Collecting opencv-contrib-python==4.0.0.21
      Downloading https://files.pythonhosted.org/packages/1f/c0/0463d91f297521b2e15e3d682d7077557fe773db72a03a0d2dba899ab8a5/opencv_contrib_python-4.0.0.21-cp36-cp36m-manylinux1_x86_64.whl (31.2MB)
         |████████████████████████████████| 31.2MB 184kB/s
    Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from opencv-python==4.0.0.21->-r cv2_requirements.txt (line 1)) (1.18.5)
    ERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.
    Installing collected packages: opencv-python, opencv-contrib-python
      Found existing installation: opencv-python 4.1.2.30
        Uninstalling opencv-python-4.1.2.30:
          Successfully uninstalled opencv-python-4.1.2.30
      Found existing installation: opencv-contrib-python 4.1.2.30
        Uninstalling opencv-contrib-python-4.1.2.30:
          Successfully uninstalled opencv-contrib-python-4.1.2.30
    Successfully installed opencv-contrib-python-4.0.0.21 opencv-python-4.0.0.21

```
import cv2
cv2.__version__
```

    '4.0.0'

### Functions

```
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
from XiaohuLuVPDetection.lu_vp_detect.vp_detection import VPDetection
import time
# import os
import cv2
# import pylab as pl
from skimage import morphology as mp
import sys
```

```
#             Detect Intersection             #
import math
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.cluster import KMeans
from sympy import Line
from sympy.geometry import Ray

def drawLines(img, lines, color=(255,255,255)):
    """
    Draw lines on an image
    """
    for line in lines:
        for rho,theta in line:
          a = np.cos(theta)
          b = np.sin(theta)
          x0 = a*rho
          y0 = b*rho
          x1 = int(x0 + 1000*(-b))
          y1 = int(y0 + 1000*(a))
          x2 = int(x0 - 1000*(-b))
          y2 = int(y0 - 1000*(a))

          slope = (y1 - y0) / float(x1 - x0)
          angle = math.degrees(math.atan(slope))
          # if abs(angle) < 70:
          cv2.line(img, (x1,y1), (x2,y2), color, 1)

def line(p1, p2):
    A = (p1[1] - p2[1])
    B = (p2[0] - p1[0])
    C = (p1[0]*p2[1] - p2[0]*p1[1])
    return A, B, -C

def intersection2(L1, L2):
    D  = L1[0] * L2[1] - L1[1] * L2[0]
    Dx = L1[2] * L2[1] - L1[1] * L2[2]
    Dy = L1[0] * L2[2] - L1[2] * L2[0]
    if D != 0:
        x = Dx / D
        y = Dy / D
        return x,y
    else:
        return False

def regression(img, x, y, color=(255, 0, 0), axis=1):

  if axis == 1:
    y_at_border = np.array([0, img.shape[0]])
    p = np.polyfit(y, x, deg=1)
    x_at_border = np.poly1d(p)(y_at_border)
  else:
    x_at_border = np.array([0, img.shape[1]])
    p = np.polyfit(x, y, deg=1)
    y_at_border = np.poly1d(p)(x_at_border)

  cv2.line(img, (int(x_at_border[0]), int(y_at_border[0])), (int(x_at_border[1]), int(y_at_border[1])), color, 2)

  return x_at_border, y_at_border

def find_reg_points(img, lines, color=(255,0,0), drawlines=False):

  centroids = list()
  r_xs = list()
  r_ys = list()
  for line_ in lines:
      for rho,theta in line_:
          a = np.cos(theta)
          b = np.sin(theta)
          x0 = a*rho
          y0 = b*rho
          x1 = int(x0 + 1000*(-b))
          y1 = int(y0 + 1000*(a))
          x2 = int(x0 - 1000*(-b))
          y2 = int(y0 - 1000*(a))

          slope = (y1 - y0) / float(x1 - x0)
          angle = math.degrees(math.atan(slope))
          if abs(angle) > 80:
            # print(img.shape[1])
            h_layout = line((0, 0), (img.shape[1], 0))
            h_layout_lower = line((0, img.shape[0]), (img.shape[1], img.shape[0]))
            r = intersection2(h_layout, line((x1, y1), (x2, y2)))
            r_lower = intersection2(h_layout_lower, line((x1, y1), (x2, y2)))
            # cv2.line(img, (int(x1), int(y1)), (int(x2), int(y2)), color, 2)

            center_p = (int((r[0] + r_lower[0]) / 2), int((r[1] + r_lower[1])/ 2))
            centroids.append(center_p)

            r_xs.append((r[0], r_lower[0]))
            r_ys.append((r[1], r_lower[1]))

            if drawlines:
              cv2.line(img, (int(r[0]), int(r[1])), (int(r_lower[0]), int(r_lower[1])), color, 2)
              cv2.circle(img, center_p, 10, (255, 0, 255), -1)

  if drawlines:
    cv2.line(img, (int(0), int(0)), (int(0), int(img.shape[0])), color, 2)
    cv2.line(img, (int(img.shape[1]), int(0)), (int(img.shape[1]), int(img.shape[0])), color, 2)
    cv2.circle(img, (0, int(img.shape[0] / 2)), 10, (255, 0, 255), -1)
    cv2.circle(img, (img.shape[1], int(img.shape[0] / 2)), 10, (255, 0, 255), -1)
  centroids.append((0, int(img.shape[0] / 2)))
  centroids.append((img.shape[1], int(img.shape[0] / 2)))

  return r_xs, r_ys, centroids

from scipy.spatial import distance as sci_dist

def order_points(pts):

  xSorted = pts[np.argsort(pts[:, 0]), :]
  leftMost = xSorted[:2, :]
  rightMost = xSorted[2:, :]
  leftMost = leftMost[np.argsort(leftMost[:, 1]), :]
  (tl, bl) = leftMost
  D = sci_dist.cdist(tl[np.newaxis], rightMost, "euclidean")[0]
  (br, tr) = rightMost[np.argsort(D)[::-1], :]

  return np.array([tl, tr, br, bl], dtype="float32")

def get_vline_points_inborder(vpd, img, show_vline=False):

  vpd.find_vps(img)
  # vps = vpd.vps_2D
  vl_img, vl_list = vpd.create_debug_VP_image(show_vl=True)

  vl_list_for_vline = vl_list
  vl_img_for_vline = vl_img.copy()

  # print('#       Used vl_list for vline work       #')
  # print('vl_list')
  # print()

  #           Extract Vertical line            #
  h_border = Line((0, 0), (img.shape[1], 0))
  h_border_lower = Line((0, img.shape[0]), (img.shape[1], img.shape[0]))

  reg_xs = list()
  reg_ys = list()
  v_lines = list()
  # h_lines_xrange = list()
  for vl in vl_list_for_vline:
      x0, y0, x1, y1 = vl
      slope = (y1 - y0) / float(x1 - x0)
      angle = math.degrees(math.atan(slope))
      # print(angle)
      if abs(angle) > 80:
        # cv2.line(skl_copy, (int(x1), int(y1)), (int(x0), int(y0)), (255, 0, 0), 3,
        #                  cv2.LINE_AA)
        v_line = Line((x0, y0), (x1, y1))
        [top_point] = v_line.intersection(h_border)
        [bot_point] = v_line.intersection(h_border_lower)
        # v_lines.append()
        reg_xs.append((float(top_point[0]), float(bot_point[0])))
        reg_ys.append((float(top_point[1]), float(bot_point[1])))

      # if abs(angle) < 70:
      #   h_lines_xrange.append((min(x0, x1), max(x0, x1)))

  # print('#        Before Sorting      #')
  # print('reg_xs :', reg_xs)

  center_xs = list()
  for reg_x, reg_y in zip(reg_xs, reg_ys):
    center_xs.append(np.mean(reg_x))
    #           Draw Line         #
    regression(vl_img_for_vline, reg_x, reg_y)

  if show_vline:
    plt.imshow(vl_img_for_vline)
    plt.title('Divided Session')
    plt.show()

  sorted_index = list()
  sorted_center_xs = sorted(center_xs)

  for center_x in sorted_center_xs:
    # print('center_x :', center_x)
    sorted_index.append(center_xs.index(center_x))
  print('sorted_index :', sorted_index)   #

  sorted_reg_xs = list()
  sorted_reg_ys = list()
  for s_index in sorted_index:
    sorted_reg_xs.append(reg_xs[s_index])
    sorted_reg_ys.append(reg_ys[s_index])
  reg_xs = sorted_reg_xs
  reg_ys = sorted_reg_ys

  print()
  # print("#        After Sorting       #")
  # print('reg_xs :', reg_xs)
  # print()

  return reg_xs

def remove_outlier_angle(vl_list, copy_vl_list, angle_list, limit_angle=15):

    if len(vl_list) != 0:
      copy_angle_list = angle_list.copy()
      mean_angle_list = np.array(angle_list).mean()
      # print('mean_angle_list :', mean_angle_list)
      # print('angle_list :', angle_list)
      print('max angle_list gap : ', max(angle_list) - mean_angle_list)

      for angle_index, angle in enumerate(copy_angle_list):
        if abs(angle - mean_angle_list) > limit_angle:
          vl_list.remove(copy_vl_list[angle_index])
          # angle_list.remove(copy_angle_list[angle_index])

      return

def get_hline_points_inborder(img, left_border, right_border, vl_list, top_vl, bot_vl, top_down_ratio=0.66):

  ex_top_vl = list()
  ex_bot_vl = list()

  for vl in vl_list:

    x0, y0, x1, y1 = vl
    l1 = Line((x0, y0), (x1, y1))
    #     v_border line     #
    [left_point] = l1.intersection(left_border)
    # print('left_point :', left_point)
    [right_point] = l1.intersection(right_border)

    #      Figure out current vl is top / bottom vl     #
    if vl in top_vl:
      if (left_point[1] < img.shape[0] * top_down_ratio and right_point[1] < img.shape[0] * top_down_ratio):
          ex_top_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

    elif vl in bot_vl:
      if (left_point[1] > img.shape[0] * (1 - top_down_ratio) and right_point[1] > img.shape[0] * (1 - top_down_ratio)):
          ex_bot_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

  return ex_top_vl, ex_bot_vl

def toppest_bottest_vl(img, top_vl, bot_vl, direction):

    max_y = 0
    min_y = img.shape[0]
    toppest_vl = None
    bottest_vl = None
    index = 1 if direction == 'right' else 3

    for vl in top_vl:
      if vl[index] < min_y:
        toppest_vl = vl
        min_y = vl[index]

    for vl in bot_vl:
      if vl[index] > max_y:
        bottest_vl = vl
        max_y = vl[index]

    return toppest_vl, bottest_vl

def line_mirroring(src_vl, src_point, left_border, right_border):

    x0, y0, x1, y1 = src_vl
    slope = (y1 - y0) / float(x1 - x0)
    angle = - math.degrees(math.atan(slope))

    #       white_max_y 를 - toppest_vl angle 로 지나는 직선      #
    print('src_point :', src_point)
    print('angle :', angle)
    endy = 100 * math.sin(math.radians(angle))
    endx = 100 * math.cos(math.radians(angle))
    temp_vl = Line(src_point, (src_point[0] + endx, src_point[1] + endy))
    [left_point] = temp_vl.intersection(left_border)
    [right_point] = temp_vl.intersection(right_border)
    # print('left_point :', left_point)
    mirrored_vl = (float(left_point[0]), float(left_point[1]), float(right_point[0]), float(right_point[1]))

    return mirrored_vl

def choose_4points(toppest_vl, bottest_vl, src_point, left_border, right_border, parallel):

  x1, y1, x0, y0 = toppest_vl
  if parallel:
    top_line = Line((x1, y1), (x0, y0))
    print('top_line.slope :', float(top_line.slope))
    if abs(top_line.slope) < 0.05:
        src_point = (0, 0)
    parallel_tl = top_line.parallel_line(src_point)

    [tl] = parallel_tl.intersection(left_border)
    [tr] = parallel_tl.intersection(right_border)
    br, bl = bottest_vl[2:], bottest_vl[:2]
  else:
    tl, tr, br, bl = toppest_vl[:2], toppest_vl[2:], bottest_vl[2:], bottest_vl[:2]

  return tl, tr, br, bl

def top_bot_mask(mask_img, threshold=1/10):

    top_white_cnt = 0
    bot_white_cnt = 0
    white_min_x = mask_img.shape[1]
    white_max_x = 0
    white_min_y = mask_img.shape[0]
    white_max_y = 0
    temp_min_y = white_min_y
    temp_max_y = white_max_y
    min_x_coord, max_x_coord, min_y_coord, max_y_coord = None, None, None, None
    top_parallel = False
    bot_parallel = False
    for i in range(mask_img.shape[1]):

      #     Check Top Condition     #
      for j in range(int(mask_img.shape[0] * threshold)):
        # if mask_img[j][i] == 255 and j < thr_min_y:
          # thr_min_y = j
          # thr_min_y_coord = (i, j)
        if mask_img[j][i] == 255:
          top_white_cnt += 1
          if i > white_max_x:
            white_max_x = i
            max_x_coord = (i, j)
          if i < white_min_x:
            white_min_x = i
            min_x_coord = (i, j)

          if j < temp_min_y:
            temp_min_y = j
            temp_min_y_coord = (i, j)

      #           Middle Condition        #
      for j in range(int(mask_img.shape[0] * threshold), int(mask_img.shape[0] * (1 - threshold))):
        if mask_img[j][i] == 255:
          if j > white_max_y:
            white_max_y = j
            max_y_coord = (i, j)
          if j < white_min_y:
            white_min_y = j
            min_y_coord = (i, j)

      #     Check Bot Condition   #
      for j in range(int(mask_img.shape[0] * (1 - threshold)), mask_img.shape[0]):
        # if mask_img[j][i] == 255 and j < thr_min_y:
          # thr_min_y = j
          # thr_min_y_coord = (i, j)
        if mask_img[j][i] == 255:
          bot_white_cnt += 1
          if j > temp_max_y:
            temp_max_y = j
            temp_max_y_coord = (i, j)

    if top_white_cnt / (mask_img.shape[1] * int(mask_img.shape[0] * threshold)) > 0.5:
      top_parallel = True
      min_y_coord = temp_min_y_coord

    if bot_white_cnt / (mask_img.shape[1] * int(mask_img.shape[0] * threshold)) > 0.5:
      #     바닥은 평행 이동 사용하지 않는다.    #
      # bot_parallel = True
      max_y_coord = temp_max_y_coord

    return min_x_coord, max_x_coord, min_y_coord, max_y_coord, top_parallel, bot_parallel

def crop_and_warp(refer, vl_img, src_4p, dst_4p):

  #       Crop      #
  refered = np.asarray(refer)[:vl_img.shape[0], :vl_img.shape[1]]

  # plt.imshow(refered)
  # plt.title('refered')
  # plt.show()

  # compute the perspective transform matrix and then apply it
  matrix = cv2.getPerspectiveTransform(src_4p, dst_4p)
  refered = cv2.warpPerspective(refered, matrix, (refered.shape[1], refered.shape[0]))

  return refered
```

```
#     Refering for one image data   #
def synthesize(org_image, refer_image, vpd, scale_factor=6, show_img=False):

  #       Layout Part       #
  # img = Image.open(layout_path + image)
  # # print(type(img))
  # img_np = np.invert(np.asarray(img))
  # # print(img_np.max(), img_np.min())
  # ret, thr = cv2.threshold(img_np, 254, 255, cv2.THRESH_BINARY_INV)

  #       Mask Part         #
  org_color_np, org_np = np.split(np.asarray(org_image), 2, axis=1)
  # org_np = np.asarray(org)
  # gray = cv2.cvtColor(org_np, cv2.COLOR_RGB2GRAY)
  # print(org_np.shape)
  # ret, thr = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY_INV)

  print(image)
  # plt.subplot(131)
  # plt.imshow(org)
  # plt.show()
  img_size = (org_np.shape[1], org_np.shape[0])

  print('scale_factor :', scale_factor)

  # refer = Image.open(refer_path)
  refer = np.asarray(refer_image)

  #     refer_size와 img_size가 동일하거나 refer_size가 작은 경우를 고려해야한다.     #
  refer = np.tile(refer, (scale_factor, scale_factor, 1))
  size_ratio = math.floor(min((refer.shape[0] / (org_np.shape[0] * 1.5)), (refer.shape[1] / (org_np.shape[1] * 1.5))))
  refer = Image.fromarray(refer).resize((int(refer.shape[1] / size_ratio), int(refer.shape[0] / size_ratio)))

  org_color_np2 = org_color_np.copy()
  org_np = org_np.astype(np.uint8)
  org_np2 = org_np.copy()
  org_np3 = org_np.copy()

  start = time.time()

  #                   Find Best Vline_list                #
  kernel = np.ones((10, 10), np.uint8)

  org_np2_morp = cv2.morphologyEx(org_np2, cv2.MORPH_CLOSE, kernel)
  org_np2_copy = org_np2_morp.copy()
  # org_np2_copy_gray = cv2.cvtColor(org_np2_copy, cv2.COLOR_RGB2GRAY)

  ret, thr_org_np2 = cv2.threshold(org_np2_copy, 127, 255, cv2.THRESH_BINARY)
  thr_org_np2 = cv2.morphologyEx(thr_org_np2, cv2.MORPH_CLOSE, kernel)
  thr_org_np2_copy = thr_org_np2.copy()

  edge_org_np2 = cv2.Canny(org_np2_morp, 20, 100)
  edge_org_np2_copy = edge_org_np2.copy()
  edge_org_np2_copy2 = cv2.Canny((org_np2_copy / 255. * org_np2_copy).astype(np.uint8), 20, 60)

  reg_xs = get_vline_points_inborder(vpd, edge_org_np2_copy2)
  print('#            Used vl_list for vline work          #')
  print('edge_org_np2_copy2')
  print()

  #                                     Find Best Hline                                      #
  #     vline 이 존재하지 않는 경우도 고려해야한다.     #
  if len(reg_xs) == 0:
    reg_xs.append([0, 0])
    # reg_ys.append([0, org_np2.shape[0]])

  #         Divide Session by vline       #
  for reg_index, reg_x in enumerate(reg_xs):

    #     In a Session     #
    #     1.  find vanishing point    #
    #     2.  Find 3 points (vp, top & bottom points)     #
    #     3.  Do warfine and attach to the black plane      #

    print("#      Session Status      #")
    print('reg_index, reg_x :', reg_index, reg_x)

    #     0.  crop by vline     #
          #   Find Max_x, min_x, (Max_y, min_y = org.shape[0], 0)
          #   1.    우편에 한해서 max_x = 우편 vline max_x & min_x = 현재 vline min_x
          #   2.    좌편에 한해서 max_x = 현재 Max_x & min_x = 좌편 vline min_x

    #      vline 별로 양옆으로 작업을 하면 len(vline) = 1의 작업을 반복할 필요가 없어진다.    #
    iter = False
    while True:

      #                   We need Max, min x & y                #
      # four_inters = list()
      find_pair = True
      # centroid_inters = all_centroid_inters[inters_i]

      if not iter:

        #       오른쪽 끝 vline 이면        #
        if reg_index == len(reg_xs) - 1:
          print('rightest vline')
          #   1.    우편에 한해서 max_x = 우편 vline max_x & min_x = 현재 vline min_x
          max_x = org_np.shape[1]
          min_x = np.min(reg_x)

        else:
          print('middle vline')
          next_reg_x = reg_xs[reg_index + 1]
          max_x = np.max(next_reg_x)
          if np.min(reg_x) < 0:
            min_x = 0
          else:
            min_x = np.min(reg_x)

      #     i = 0 에 한해서만 왼쪽으로도 refering 진행, 나머지는 오른쪽으로만     #
      else:
          #   2.    좌편에 한해서 max_x = 현재 Max_x & min_x = 좌편 vline min_x
          max_x = np.max(reg_x)
          min_x = 0

      print('min_x, max_x :', min_x, max_x)
      # max_y = org_np.shape[0]
      # min_y = 0

      #     Make Session      #
      session = org_np2[:, int(min_x):int(max_x)]
      # plt.imshow(session)
      # plt.show()

      if (max_x - min_x) > 50:
        length_thresh = 50
      else:
        length_thresh = 20
        if max_x - min_x < 20:
            #     i != 0 인 경우 break     #
          if reg_index == 0 and np.sum(reg_xs) != 0 and not iter:
            iter = True
            print('iter :', iter)
            continue
          else:
            break

      principal_point = None
      focal_length = 1300 # 1102.79
      seed = None
      vpd = VPDetection(length_thresh, principal_point, focal_length, seed)

      try:
        vpd.find_vps(session)
        # vps = vpd.vps_2D
        vl_img, vl_list = vpd.create_debug_VP_image(show_vl=True)

        #       Closing Morphing by OpenCV      #
        kernel = np.ones((10, 10), np.uint8)
        session = cv2.morphologyEx(session, cv2.MORPH_CLOSE, kernel)

        #     Before Canny    #
        session_prob = session / 255.

        #     Multiply mask map with Edge => Erase Unnecessary vl line    #
        #   multiply one time   #
        session2 = session_prob * (session)
        edge_session2 = cv2.Canny(session2.astype(np.uint8), 20, 60)
        #       two time      #
        # session2 = session_prob**2 * (session)
        # edge_session2 = cv2.Canny(session2.astype(np.uint8), 20, 60)

        vpd.find_vps(edge_session2)
        # vps = vpd.vps_2D
        vl_edge_img2, vl_edge_list2 = vpd.create_debug_VP_image(show_vl=True)

      except Exception as e:
        print("Error in vpd Sessions Zone :", e)
        print()
          #     i != 0 인 경우 break     #
        if reg_index == 0 and np.sum(reg_xs) != 0 and not iter:
          iter = True
          print('iter :', iter)
          continue

        else:
          break

      #         What is best vl_list      #
      vl_list = vl_edge_list2
      print('#            Used vl_list for hline work          #')
      print('vl_edge_list2')
      print()

      h_lines = list()
      # print('vps :', vps)

      top_vl = list()
      bot_vl = list()

      left_vl_list = list()
      right_vl_list = list()

      left_angle = list()
      right_angle = list()

      for vl in vl_list:
        x0, y0, x1, y1 = vl
        slope = (y1 - y0) / float(x1 - x0)
        angle = math.degrees(math.atan(slope))

        if abs(angle) < 70:
          if (y0 + y1) / 2 < vl_img.shape[0] / 2:
            # print(y0, y1)
            top_vl.append(vl)
            if angle < 0:
              left_vl_list.append(vl)
              left_angle.append(abs(angle))
            else:
              right_vl_list.append(vl)
              right_angle.append(abs(angle))
          else:
            bot_vl.append(vl)
            if angle > 0:
              left_vl_list.append(vl)
              left_angle.append(abs(angle))
            else:
              right_vl_list.append(vl)
              right_angle.append(abs(angle))
          # cv2.line(skl_copy, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 2,
          #                  cv2.LINE_AA)
          h_lines.append(vl)
        #   regression(vl_img, (x0, x1), (y0, y1), color=(0,255,255),axis=1)

      print('#        Original vl list        #')
      print('len(left_vl_list) :', len(left_vl_list))
      print('len(right_vl_list) :', len(right_vl_list))
      print()

      #             소실점 방향 선택            #
      #       original vl_list 는 remove 가 존재하기 때문에 copy_ version 사용한다.       #
      copy_left_vl_list = left_vl_list.copy()
      copy_right_vl_list = right_vl_list.copy()
      print('len(copy_left_vl_list) :', len(copy_left_vl_list))
      print('len(copy_right_vl_list) :', len(copy_right_vl_list))

      if len(copy_left_vl_list) >= len(copy_right_vl_list):
        direction = 'left'
        vl_list, copy_vl_list, angle = left_vl_list, copy_left_vl_list, left_angle
      else:
        direction = 'right'
        vl_list, copy_vl_list, angle = right_vl_list, copy_right_vl_list, right_angle

      print("#            Remove Outliered Angle in Hlines           #")
      remove_outlier_angle(vl_list, copy_vl_list, angle)
      # remove_outlier_angle(right_vl_list, copy_right_vl_list, right_angle)
      print()

      print('len(left_vl_list) :', len(left_vl_list))
      print('len(right_vl_list) :', len(right_vl_list))
      print()

      # if len(left_vl_list) == 0 and len(right_vl_list) == 0:
      if len(vl_list) == 0:
        #     i != 0 인 경우 break     #
        if reg_index == 0 and not iter:
          iter = True
          print('iter :', iter)
          continue

        else:
          break

      #                       Find External vn_line                     #
      # h_border_top = Line((0, 0), (vl_img.shape[1], 0))
      # h_border_bot = Line((0, vl_img.shape[0]), (vl_img.shape[1], vl_img.shape[0]))
      l2 = Line((0, 0), (0, vl_img.shape[0]))
      l3 = Line((vl_img.shape[1], 0), (vl_img.shape[1], vl_img.shape[0]))

      ex_top_vl, ex_bot_vl = get_hline_points_inborder(vl_img, l2, l3, vl_list, top_vl, bot_vl)
      # ex_left_top_vl, ex_left_bot_vl = get_hline_points_inborder(vl_img, l2, l3, left_vl_list, top_vl, bot_vl)
      # ex_right_top_vl, ex_right_bot_vl = get_hline_points_inborder(vl_img, l2, l3, right_vl_list, top_vl, bot_vl)

      print()
      print('#          Extended vl list left / right  TB Condition        #')
      print('len(ex_top_vl) :', len(ex_top_vl))
      print('len(ex_bot_vl) :', len(ex_bot_vl))
      # print('len(ex_left_top_vl) :', len(ex_left_top_vl))
      # print('len(ex_left_bot_vl) :', len(ex_left_bot_vl))
      # print('len(ex_right_top_vl) :', len(ex_right_top_vl))
      # print('len(ex_right_bot_vl) :', len(ex_right_bot_vl))
      print()

      #       find min max Line     #
      toppest_vl, bottest_vl = toppest_bottest_vl(vl_img, ex_top_vl, ex_bot_vl, direction)
      # right_toppest_vl, right_bottest_vl = toppest_bottest_vl(vl_img, ex_right_top_vl, ex_right_bot_vl, 'right')
      # left_toppest_vl, left_bottest_vl = toppest_bottest_vl(vl_img, ex_left_top_vl, ex_left_bot_vl, 'left')

      #       Figure out min_y & max_y of thr_session       #
      # print('thr_session.shape :', thr_session.shape)
      gray = cv2.cvtColor(session, cv2.COLOR_RGB2GRAY)
      ret, thr_session = cv2.threshold(session, 127, 255, cv2.THRESH_BINARY)
      thr_session = cv2.morphologyEx(thr_session, cv2.MORPH_CLOSE, kernel)
      thr_session_gray = cv2.cvtColor(thr_session, cv2.COLOR_RGB2GRAY)
      print('thr_session_gray.shape :', thr_session_gray.shape)

      min_x_coord, max_x_coord, min_y_coord, max_y_coord, top_parallel, _ = top_bot_mask(thr_session_gray)

      print('max_x_coord, min_x_coord :', max_x_coord, min_x_coord)
      print('max_y_coord, min_y_coord :', max_y_coord, min_y_coord)

      try:

        #     None 이라면 대칭이동을 통해 만들어주어야한다.     #
        if len(ex_bot_vl) == 0:
          if toppest_vl is not None:
            bottest_vl = line_mirroring(toppest_vl, max_y_coord, l2, l3)
            print('bottest_vl :', bottest_vl)

        elif len(ex_top_vl) == 0:
          if bottest_vl is not None:
            toppest_vl = line_mirroring(bottest_vl, min_y_coord, l2, l3)
            print('toppest_vl :', toppest_vl)

        # if len(ex_right_bot_vl) == 0:
        #   if right_toppest_vl is not None:
        #     right_bottest_vl = line_mirroring(right_toppest_vl, max_y_coord, l2, l3)
        #     print('right_bottest_vl :', right_bottest_vl)

        # elif len(ex_right_top_vl) == 0:
        #   if right_bottest_vl is not None:
        #     right_toppest_vl = line_mirroring(right_bottest_vl, min_y_coord, l2, l3)
        #     print('right_toppest_vl :', right_toppest_vl)

        # if len(ex_left_bot_vl) == 0:
        #   if left_toppest_vl is not None:
        #     left_bottest_vl = line_mirroring(left_toppest_vl, max_y_coord, l2, l3)
        #     print('left_bottest_vl :', left_bottest_vl)

        # elif len(ex_left_top_vl) == 0:
        #   if left_bottest_vl is not None:
        #     left_toppest_vl = line_mirroring(left_bottest_vl, min_y_coord, l2, l3)
        #     print('left_toppest_vl :', left_toppest_vl)

        for (x1, y1, x0, y0) in [toppest_vl, bottest_vl]:
          cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
        # for (x1, y1, x0, y0) in [right_toppest_vl, right_bottest_vl]:
        #   cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
        # for (x1, y1, x0, y0) in [left_toppest_vl, left_bottest_vl]:
        #   cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (255, 255, 0), 3, cv2.LINE_AA)
      except Exception as e:
        print('Error in drawing top, bot -est lines :', e)

        #     i != 0 인 경우 break     #
        if reg_index == 0 and not iter:
          iter = True
          print('iter :', iter)
          continue

        else:
          break

      #               Choose 4 points             #
      tl, tr, br, bl = choose_4points(toppest_vl, bottest_vl, min_x_coord, l2, l3, top_parallel)

      # if len(copy_left_vl_list) >= len(copy_right_vl_list):
      #   #       toppest & bottest vl should be on the min_y & max_y of white mask       #
      #   tl, tr, br, bl = choose_4points(left_toppest_vl, left_bottest_vl, min_x_coord, top_parallel)
      # else:
      #   tl, tr, br, bl = choose_4points(right_toppest_vl, right_bottest_vl, min_x_coord, top_parallel)

      #       Refering      #
      #   tl, tr, br, bl    #
      #     refer를 위해 src_x => 0 으로 맞춰준다.    #
      src = np.array([
          [0, 0],
          [vl_img.shape[1], 0],
          [vl_img.shape[1], vl_img.shape[0]],
          [0, vl_img.shape[0]]], dtype = "float32")
      dst = np.array([list(tl),
                      list(tr),
                      list(br),
                      list(bl)], dtype = "float32")

      print()
      print("#          Warping Points        #")
      print('src :', src)
      print('dst :', dst)

      refered = crop_and_warp(refer, vl_img, src, dst)

      # plt.subplot(121)
      # plt.imshow(vl_img)
      # plt.title('Session_vl_img')
      # # plt.xlim(np.min(vps[:, [0]]), np.max(vps[:, [0]]))
      # # plt.ylim(np.min(vps[:, [1]]), np.max(vps[:, [1]]))
      # plt.subplot(122)
      # plt.imshow(refered)
      # plt.show()

      org_color_np2[:, int(min_x):int(max_x)] = refered

      #     i != 0 인 경우 break     #
      if reg_index == 0 and np.sum(reg_xs) != 0 and not iter:
        iter = True
        print('iter :', iter)

      else:
        break

  #         Refer의 검은 부분은 original image로 채운다.       #
  org_color_np2 = np.where(org_color_np2 == 0, org_color_np, org_color_np2)

  print('elapsed time :', time.time() - start)
  print()

  # print('np.max(org_np) :', np.max(org_np))
  org_np = org_np / 255.
  refer = org_color_np2
  # plt.show()

  #     1.wall_mask * reference color image + (1 - wall_mask) * original_image)    #
  syn_ = org_np * (refer) + (1 - org_np) * org_color_np
  syn = org_np * syn_ + (1 - org_np) * org_color_np

  #       Brightness Preservation      #
  org_hsv = cv2.cvtColor(np.uint8(org_color_np), cv2.COLOR_RGB2HSV)
  syn_hsv = cv2.cvtColor(np.uint8(syn), cv2.COLOR_RGB2HSV)

  org_h, org_s, org_v = cv2.split(org_hsv)
  syn_h, syn_s, syn_v = cv2.split(syn_hsv)

  mask_map = org_np[:, :, 0]
  # print(mask_map.shape)

  w = org_v / 255.
  w = w / 1.87
  syn_v2 = mask_map * syn_v + (1 - mask_map) * org_v
  syn_v2 = (1 - w) * syn_v2 + w * org_v
  syn_v2 = syn_v2.astype(np.uint8)

  final_syn_hsv = cv2.merge([syn_h, syn_s, syn_v2])
  final_syn = cv2.cvtColor(final_syn_hsv, cv2.COLOR_HSV2RGB)

  if show_img:

    plt.figure(figsize=(15, 10))
    plt.subplot(131)
    plt.imshow(org_np3)

    plt.subplot(132)
    plt.imshow(org_color_np2)

    plt.subplot(133)
    plt.imshow(final_syn)
    plt.show()

  return final_syn
```

```
if __name__ == '__main__':

  org_path = 'datasets/lsun/images/' # -> original image
  layout_path = 'drn_d_105_024_val_ms/images/' # -> layout image (result image frome above model file)
  refer_path = '../refer_data/wall/myxkehu1kfzggursepnk0tfiyps8zbs5umvzv8d92r6hhxejgawebwsufssgov5q-.jpg'

  object_ = 'wall'
  mask_path = '../test_data/' + object_ + '/'
  # layout_path = '../test_layout_result/' + object_ + '/'
  # layout_path = '../mask_layout_data/' + object_ + '/'
  # layout_list = os.listdir(layout_path)

  refer_path = './refer_data/' + object_ + '/'

  length_thresh = 70
  principal_point = None
  focal_length = 1300 # 1102.79
  seed = 1300
  vpd = VPDetection(length_thresh, principal_point, focal_length, seed)

  for image in os.listdir(mask_path):

    for refer in os.listdir(refer_path):

      try:

        image = '720x480_20191004110915620_aVLsnTL8a4.png'

        org = Image.open(mask_path + image)
        refer = Image.open(refer_path + refer)
        synthesize(org, refer, vpd)

      except Exception as e:
        print('Error in load images :', e)

      # break

```

```
6.486232280731201
elapsed time : 6.074836015701294
elapsed time : 6.072072744369507
elapsed time : 6.072662591934204
elapsed time : 6.140763998031616
---
elapsed time : 4.42145848274231
elapsed time : 4.432448148727417
elapsed time : 4.47941255569458
---
elapsed time : 2.750857353210449
elapsed time : 2.7681655883789062
elapsed time : 2.7948927879333496
elapsed time : 2.882319927215576

```

Layout Estimation Func Final without AI_origin.ipynb