# Deep Learning Bootcamp November 2017, GPU Computing for Data Scientists

## 18  PyTorch NUMER.AI  Deep Learning Binary Classification using BCELoss

Web: https://www.meetup.com/Tel-Aviv-Deep-Learning-Bootcamp/events/241762893/

Notebooks:  On GitHub

*Shlomo Kashani*

# Introduction

- This tutorial was written in order to demonstrate a **fully working** example of a PyTorch NN on a real world use case, namely a Binary Classification problem. If you are interested in the sk-learn version of this problem please refer to: https://github.com/QuantScientist/deep-ml-meetups/tree/master/hacking-kaggle/python/numer-ai

- For the scientific foundation behind Binary Classification and Logistic Regression, refer to: https://github.com/QuantScientist/deep-ml-meetups/blob/master/data-science-interviews/imp-ans.pdf

- Every step, from reading the CSV into numpy arrays, converting to GPU based tensors, training and validation, are meant to aid newcomers in their first steps in PyTorch.

- Additionally, commonly used Kaggle metrics such as ROC_AUC and LOG_LOSS are logged and plotted both for the training set as well as for the validation set.

- Thus, the NN architecture is naive and by any means **unoptimized**. Hopefully, I will improve it over time and I am working on a second CNN based version of the same problem.

## Data
- Download from https://numer.ai/leaderboard

# PyTorch Imports

```python
# !pip install pycuda
%reset -f
# %%timeit

import torch
from torch.autograd import Variable
import numpy as np
import pandas
import numpy as np
import pandas as pd
from sklearn import cross_validation
from sklearn import metrics
from sklearn.metrics import roc_auc_score, log_loss, roc_auc_score, roc_curve, auc
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn import metrics
from sklearn.metrics import roc_auc_score, log_loss, roc_auc_score, roc_curve, auc
from sklearn.cross_validation import StratifiedKFold, ShuffleSplit, cross_val_score, train_test_split
import logging
import numpy
import numpy as np
from __future__ import print_function
from __future__ import division
import math
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import os
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torchvision import transforms
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from sklearn.preprocessing import MultiLabelBinarizer
import time
from sklearn.preprocessing import PolynomialFeatures
import pandas as pd
import numpy as np
import scipy
%matplotlib inline
from pylab import rcParams
rcParams['figure.figsize'] = (6, 6)      # setting default size of plots
import tensorflow as tf
print("tensorflow:" + tf.__version__)
!set "KERAS_BACKEND=tensorflow"
import torch
import sys
print('__Python VERSION:', sys.version)
print('__pyTorch VERSION:', torch.__version__)
print('__CUDA VERSION')
from subprocess import call
print('__CUDNN VERSION:', torch.backends.cudnn.version())
print('__Number CUDA Devices:', torch.cuda.device_count())
print('__Devices')

# !pip install http://download.pytorch.org/whl/cu75/torch-0.2.0.post1-cp27-cp27mu-manylinux1_x86_64.whl
# !pip install torchvision
# ! pip install cv2
# import cv2

print("OS: ", sys.platform)
print("Python: ", sys.version)
print("PyTorch: ", torch.__version__)
print("Numpy: ", np.__version__)

handler=logging.basicConfig(level=logging.INFO)
lgr = logging.getLogger(__name__)
%matplotlib inline

# !pip install psutil
import psutil
def cpuStats():
        print(sys.version)
        print(psutil.cpu_percent())
        print(psutil.virtual_memory())  # physical memory usage
        pid = os.getpid()
        py = psutil.Process(pid)
        memoryUse = py.memory_info()[0] / 2. ** 30  # memory use in GB...I think
        print('memory GB:', memoryUse)

cpuStats()
```

    /usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
      "This module will be removed in 0.20.", DeprecationWarning)

    tensorflow:1.2.1
    __Python VERSION: 2.7.12 (default, Nov 19 2016, 06:48:10)
    [GCC 5.4.0 20160609]
    __pyTorch VERSION: 0.2.0+42448cf
    __CUDA VERSION
    __CUDNN VERSION: None
    __Number CUDA Devices: 1
    __Devices
    OS:  linux2
    Python:  2.7.12 (default, Nov 19 2016, 06:48:10)
    [GCC 5.4.0 20160609]
    PyTorch:  0.2.0+42448cf
    Numpy:  1.13.1
    2.7.12 (default, Nov 19 2016, 06:48:10)
    [GCC 5.4.0 20160609]
    0.0
    svmem(total=67469099008, available=63134949376, percent=6.4, used=3730501632, free=61150445568, active=4437622784, inactive=1223413760, buffers=284803072, cached=2303348736, shared=55218176)
    memory GB: 0.221778869629

#  CUDA

```python
# %%timeit
use_cuda = torch.cuda.is_available()
# use_cuda = False

FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
Tensor = FloatTensor

lgr.info("USE CUDA=" + str (use_cuda))

# ! watch -n 0.1 'ps f -o user,pgrp,pid,pcpu,pmem,start,time,command -p `lsof -n -w -t /dev/nvidia*`'
# sudo apt-get install dstat #install dstat
# sudo pip install nvidia-ml-py #install Python NVIDIA Management Library
# wget https://raw.githubusercontent.com/datumbox/dstat/master/plugins/dstat_nvidia_gpu.py
# sudo mv dstat_nvidia_gpu.py /usr/share/dstat/ #move file to the plugins directory of dstat
```

    INFO:__main__:USE CUDA=True

#  Global params

```python
# NN params
DROPOUT_PROB = 0.75
N_EPOCHS = 50
BATCH_SIZE = 4
LR = 0.005
MOMENTUM= 0.9
PIN_MEMORY=use_cuda # True IF CUDA

# Data params
TARGET_VAR= 'target'
TOURNAMENT_DATA_CSV = 'numerai_tournament_data.csv'
TRAINING_DATA_CSV = 'numerai_training_data.csv'
BASE_FOLDER = 'numerai/'

# fix seed
seed=17*19
np.random.seed(seed)
torch.manual_seed(seed)
if use_cuda:
    torch.cuda.manual_seed(seed)
```

#  Load a CSV file for Binary classification (numpy)

```python
# %%timeit
df_train = pd.read_csv(BASE_FOLDER + TRAINING_DATA_CSV)
df_train.head(5)
```

      id
      era
      data_type
      feature1
      feature2
      feature3
      feature4
      feature5
      feature6
      feature7
      ...
      feature13
      feature14
      feature15
      feature16
      feature17
      feature18
      feature19
      feature20
      feature21
      target

      0
      135682
      era1
      train
      0.53352
      0.64336
      0.46577
      0.53001
      0.55734
      0.45773
      0.41169
      ...
      0.51224
      0.50484
      0.41929
      0.50954
      0.47383
      0.48797
      0.38373
      0.46233
      0.33341
      0

      1
      110546
      era1
      train
      0.54196
      0.81576
      0.46632
      0.62320
      0.52427
      0.64378
      0.55662
      ...
      0.52643
      0.63809
      0.67121
      0.49421
      0.45291
      0.46932
      0.54445
      0.30997
      0.19023
      0

      2
      76047
      era1
      train
      0.49158
      0.69131
      0.57816
      0.54010
      0.43064
      0.49986
      0.61902
      ...
      0.43310
      0.72286
      0.76257
      0.36600
      0.55330
      0.56566
      0.67528
      0.34960
      0.25721
      1

      3
      66098
      era1
      train
      0.54519
      0.42473
      0.63472
      0.39003
      0.37485
      0.43810
      0.59557
      ...
      0.41658
      0.63417
      0.50189
      0.40883
      0.58705
      0.63785
      0.56225
      0.55989
      0.58642
      0

      4
      88227
      era1
      train
      0.44307
      0.74076
      0.52210
      0.56543
      0.51125
      0.66457
      0.42263
      ...
      0.45851
      0.58805
      0.49860
      0.48023
      0.52606
      0.53253
      0.38361
      0.43829
      0.25014
      0

5 rows × 25 columns

#  Feature enrichement
- This would be usually not required when using NN's; it is here for demonstration purposes.

```python
def genBasicFeatures(inDF):
    print('Generating basic features ...')
    df_copy=inDF.copy(deep=True)
    magicNumber=21
    feature_cols = list(inDF.columns)

    inDF['x_mean'] = np.mean(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_median'] = np.median(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_std'] = np.std(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_skew'] = scipy.stats.skew(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_kurt'] = scipy.stats.kurtosis(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_var'] = np.var(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_max'] = np.max(df_copy.ix[:, 0:magicNumber], axis=1)
    inDF['x_min'] = np.min(df_copy.ix[:, 0:magicNumber], axis=1)

    return inDF

def addPolyFeatures(inDF, deg=2):
    print('Generating poly features ...')
    df_copy=inDF.copy(deep=True)
    poly=PolynomialFeatures(degree=deg)
    p_testX = poly.fit(df_copy)
    # AttributeError: 'PolynomialFeatures' object has no attribute 'get_feature_names'
    target_feature_names = ['x'.join(['{}^{}'.format(pair[0],pair[1]) for pair in tuple if pair[1]!=0]) for tuple in [zip(df_copy.columns,p) for p in poly.powers_]]
    df_copy = pd.DataFrame(p_testX.transform(df_copy),columns=target_feature_names)

    return df_copy
```

#  Train / Validation / Test Split
- Numerai provides a data set that is allready split into train, validation and test sets.

```python
# Train, Validation, Test Split
def loadDataSplit():
    df_train = pd.read_csv(BASE_FOLDER + TRAINING_DATA_CSV)
    # TOURNAMENT_DATA_CSV has both validation and test data provided by NumerAI
    df_test_valid = pd.read_csv(BASE_FOLDER + TOURNAMENT_DATA_CSV)

    answers_1_SINGLE = df_train[TARGET_VAR]
    df_train.drop(TARGET_VAR, axis=1,inplace=True)
    df_train.drop('id', axis=1,inplace=True)
    df_train.drop('era', axis=1,inplace=True)
    df_train.drop('data_type', axis=1,inplace=True)

    # Add polynomial features
    df_train=genBasicFeatures(df_train)
#     df_train = addPolyFeatures(df_train)

    df_train.to_csv(BASE_FOLDER + TRAINING_DATA_CSV + 'clean.csv', header=False,  index = False)
    df_train= pd.read_csv(BASE_FOLDER + TRAINING_DATA_CSV + 'clean.csv', header=None, dtype=np.float32)
    df_train = pd.concat([df_train, answers_1_SINGLE], axis=1)
    feature_cols = list(df_train.columns[:-1])
#     print (feature_cols)
    target_col = df_train.columns[-1]
    trainX, trainY = df_train[feature_cols], df_train[target_col]

    # TOURNAMENT_DATA_CSV has both validation and test data provided by NumerAI
    # Validation set
    df_validation_set=df_test_valid.loc[df_test_valid['data_type'] == 'validation']
    df_validation_set=df_validation_set.copy(deep=True)
    answers_1_SINGLE_validation = df_validation_set[TARGET_VAR]
    df_validation_set.drop(TARGET_VAR, axis=1,inplace=True)
    df_validation_set.drop('id', axis=1,inplace=True)
    df_validation_set.drop('era', axis=1,inplace=True)
    df_validation_set.drop('data_type', axis=1,inplace=True)

   # Add polynomial features
    df_validation_set=genBasicFeatures(df_validation_set)
#     df_validation_set = addPolyFeatures(df_validation_set)

    df_validation_set.to_csv(BASE_FOLDER + TRAINING_DATA_CSV + '-validation-clean.csv', header=False,  index = False)
    df_validation_set= pd.read_csv(BASE_FOLDER + TRAINING_DATA_CSV + '-validation-clean.csv', header=None, dtype=np.float32)
    df_validation_set = pd.concat([df_validation_set, answers_1_SINGLE_validation], axis=1)
    feature_cols = list(df_validation_set.columns[:-1])

    target_col = df_validation_set.columns[-1]
    valX, valY = df_validation_set[feature_cols], df_validation_set[target_col]

    # Test set for submission (not labeled)
    df_test_set = pd.read_csv(BASE_FOLDER + TOURNAMENT_DATA_CSV)
#     df_test_set=df_test_set.loc[df_test_valid['data_type'] == 'live']
    df_test_set=df_test_set.copy(deep=True)
    df_test_set.drop(TARGET_VAR, axis=1,inplace=True)
    tid_1_SINGLE = df_test_set['id']
    df_test_set.drop('id', axis=1,inplace=True)
    df_test_set.drop('era', axis=1,inplace=True)
    df_test_set.drop('data_type', axis=1,inplace=True)

    # Add polynomial features
    df_test_set=genBasicFeatures(df_test_set)
#     df_test_set = addPolyFeatures(df_test_set)

    feature_cols = list(df_test_set.columns) # must be run here, we dont want the ID
#     print (feature_cols)
    df_test_set = pd.concat([tid_1_SINGLE, df_test_set], axis=1)
    testX = df_test_set[feature_cols].values

    return trainX, trainY, valX, valY, testX, df_test_set
```

```python
# %%timeit
trainX, trainY, valX, valY, testX, df_test_set = loadDataSplit()
# X, y = loadDataSplit(999)

# # Number of features for the input layer
N_FEATURES=trainX.shape[1]
# print (trainX.head(3))
# print (df_test_set.head(3))
print (trainX.shape)
print (trainY.shape)
print (valX.shape)
print (valY.shape)
print (testX.shape)
print (df_test_set.shape)
```

    Generating basic features ...

    /usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:7: DeprecationWarning:
    .ix is deprecated. Please use
    .loc for label based indexing or
    .iloc for positional indexing

    See the documentation here:
    http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated
      import sys

    Generating basic features ...
    Generating basic features ...
    (108405, 29)
    (108405,)
    (16686, 29)
    (16686,)
    (45647, 29)
    (45647, 30)

#  Create PyTorch GPU tensors from numpy arrays

- Note how we transfrom the np arrays

```python
# Convert the np arrays into the correct dimention and type
# Note that BCEloss requires Float in X as well as in y
def XnumpyToTensor(x_data_np):
    x_data_np = np.array(x_data_np.values, dtype=np.float32)
    print(x_data_np.shape)
    print(type(x_data_np))

    if use_cuda:
        lgr.info ("Using the GPU")
        X_tensor = Variable(torch.from_numpy(x_data_np).cuda()) # Note the conversion for pytorch
    else:
        lgr.info ("Using the CPU")
        X_tensor = Variable(torch.from_numpy(x_data_np)) # Note the conversion for pytorch

    print(type(X_tensor.data)) # should be 'torch.cuda.FloatTensor'
    print(x_data_np.shape)
    print(type(x_data_np))
    return X_tensor

# Convert the np arrays into the correct dimention and type
# Note that BCEloss requires Float in X as well as in y
def YnumpyToTensor(y_data_np):
    y_data_np=y_data_np.reshape((y_data_np.shape[0],1)) # Must be reshaped for PyTorch!
    print(y_data_np.shape)
    print(type(y_data_np))

    if use_cuda:
        lgr.info ("Using the GPU")
    #     Y = Variable(torch.from_numpy(y_data_np).type(torch.LongTensor).cuda())
        Y_tensor = Variable(torch.from_numpy(y_data_np)).type(torch.FloatTensor).cuda()  # BCEloss requires Float
    else:
        lgr.info ("Using the CPU")
    #     Y = Variable(torch.squeeze (torch.from_numpy(y_data_np).type(torch.LongTensor)))  #
        Y_tensor = Variable(torch.from_numpy(y_data_np)).type(torch.FloatTensor)  # BCEloss requires Float

    print(type(Y_tensor.data)) # should be 'torch.cuda.FloatTensor'
    print(y_data_np.shape)
    print(type(y_data_np))
    return Y_tensor
```

# The NN model

### MLP model
- A multilayer perceptron is a logistic regressor where instead of feeding the input to the logistic regression you insert a intermediate layer, called the hidden layer, that has a nonlinear activation function (usually tanh or sigmoid) . One can use many such hidden layers making the architecture deep.

- Here we define a simple MLP structure. We map the input feature vector to a higher space, then later gradually decrease the dimension, and in the end into a 1-dimension space. Because we are calculating the probability of each genre independently, after the final layer we need to use a sigmoid layer.

###  Initial weights selection

- There are many ways to select the initial weights to a neural network architecture. A common initialization scheme is random initialization, which sets the biases and weights of all the nodes in each hidden layer randomly.

- Before starting the training process, an initial value is assigned to each variable. This is done by pure randomness, using for example a uniform or Gaussian distribution. But if we start with weights that are too small, the signal could decrease so much that it is too small to be useful. On the other side, when the parameters are initialized with high values, the signal can end up to explode while propagating through the network.

- In consequence, a good initialization can have a radical effect on how fast the network will learn useful patterns.For this purpose, some best practices have been developed. One famous example used is **Xavier initialization**. Its formulation is based on the number of input and output neurons and uses sampling from a uniform distribution with zero mean and all biases set to zero.

- In effect (according to theory) initializing the weights of the network to values that would be closer to the optimal, and therefore require less epochs to train.

### References:
* **`nninit.xavier_uniform(tensor, gain=1)`** - Fills `tensor` with values according to the method described in ["Understanding the difficulty of training deep feedforward neural networks" - Glorot, X. and Bengio, Y.](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf), using a uniform distribution.
* **`nninit.xavier_normal(tensor, gain=1)`** - Fills `tensor` with values according to the method described in ["Understanding the difficulty of training deep feedforward neural networks" - Glorot, X. and Bengio, Y.](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf), using a normal distribution.
* **`nninit.kaiming_uniform(tensor, gain=1)`** - Fills `tensor` with values according to the method described in ["Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" - He, K. et al.](https://arxiv.org/abs/1502.01852) using a uniform distribution.
* **`nninit.kaiming_normal(tensor, gain=1)`** - Fills `tensor` with values according to the method described in ["Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" - He, K. et al.]

```python
# p is the probability of being dropped in PyTorch
# At each layer, DECREASE dropout
dropout = torch.nn.Dropout(p=1 - (DROPOUT_PROB +0.20))

# class Net(torch.nn.Module):
#     def __init__(self, n_feature, n_hidden, n_output,initKernel='uniform'):
#         super(Net, self).__init__()
#         self.hidden = torch.nn.Linear(n_feature, n_hidden)   # hidden layer
#         self.out = torch.nn.Linear(n_hidden, n_output)   # output layer

#         # xavier initializer
#         if initKernel == 'uniform':
#             nn.init.xavier_uniform(self.hidden.weight, gain=np.sqrt(2.0))
#         else:
#             nn.init.kaiming_normal(self.hidden.weight)

#     def forward(self, x):
#         x = F.relu(self.hidden(x))      # activation function for hidden layer
#         x = self.out(x)
#         return F.sigmoid(x)

class Net2(nn.Module):
    def __init__(self, n_feature, n_hidden, n_output,initKernel='uniform'):
        super(Net2, self).__init__()
        self.dis = nn.Sequential(
            nn.Linear(n_feature, n_hidden),
            dropout,
            nn.LeakyReLU(0.1),
            nn.Linear(n_hidden, n_hidden),
            dropout,
            nn.LeakyReLU(0.1),
            nn.Linear(n_hidden, 1),
            dropout,
            nn.Sigmoid()
        )
    def forward(self, x):
        x = self.dis(x)
        return x

hiddenLayer1Size=1024
hiddenLayer2Size=int(hiddenLayer1Size/8)
hiddenLayer3Size=int(hiddenLayer1Size/16)
hiddenLayer4Size=int(hiddenLayer1Size/32)
hiddenLayer5Size=int(hiddenLayer1Size/64)

# # Hypothesis using sigmoid
linear1=torch.nn.Linear(N_FEATURES, hiddenLayer1Size, bias=True)
torch.nn.init.xavier_uniform(linear1.weight)

linear2=torch.nn.Linear(hiddenLayer1Size, hiddenLayer2Size)
torch.nn.init.xavier_uniform(linear2.weight)

linear3=torch.nn.Linear(hiddenLayer2Size, hiddenLayer3Size)
torch.nn.init.xavier_uniform(linear3.weight)

linear4=torch.nn.Linear(hiddenLayer3Size, hiddenLayer4Size)
torch.nn.init.xavier_uniform(linear4.weight)

linear5=torch.nn.Linear(hiddenLayer4Size, hiddenLayer5Size)
torch.nn.init.xavier_uniform(linear5.weight)

linear6=torch.nn.Linear(hiddenLayer5Size, 1)
torch.nn.init.xavier_uniform(linear6.weight)

sigmoid = torch.nn.Sigmoid()
tanh=torch.nn.Tanh()
relu=torch.nn.LeakyReLU()

net = torch.nn.Sequential(linear1,dropout,tanh,nn.BatchNorm1d(hiddenLayer1Size),
                          linear2,dropout,tanh,
                          linear3,dropout,relu,
                          linear4,dropout,tanh,
                          linear5,dropout,relu,
                          linear6,sigmoid
                          )

# net = Net(n_feature=N_FEATURES, n_hidden=1024, n_output=1)   # define the network
# net = Net2(n_feature=N_FEATURES, n_hidden=2048, n_output=1)   # define the network

lgr.info(net)  # net architecture
```

    INFO:__main__:Sequential (
      (0): Linear (29 -> 1024)
      (1): Dropout (p = 0.05)
      (2): Tanh ()
      (3): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True)
      (4): Linear (1024 -> 128)
      (5): Dropout (p = 0.05)
      (6): Tanh ()
      (7): Linear (128 -> 64)
      (8): Dropout (p = 0.05)
      (9): LeakyReLU (0.01)
      (10): Linear (64 -> 32)
      (11): Dropout (p = 0.05)
      (12): Tanh ()
      (13): Linear (32 -> 16)
      (14): Dropout (p = 0.05)
      (15): LeakyReLU (0.01)
      (16): Linear (16 -> 1)
      (17): Sigmoid ()
    )

## Print the full net architecture

```python
# Taken from https://stackoverflow.com/questions/42480111/model-summary-in-pytorch/42616812
from torch.nn.modules.module import _addindent
import torch
import numpy as np
def torch_summarize(model, show_weights=True, show_parameters=True):
    """Summarizes torch model by showing trainable parameters and weights."""
    tmpstr = model.__class__.__name__ + ' (\n'
    for key, module in model._modules.items():
        # if it contains layers let call it recursively to get params and weights
        if type(module) in [
            torch.nn.modules.container.Container,
            torch.nn.modules.container.Sequential
        ]:
            modstr = torch_summarize(module)
        else:
            modstr = module.__repr__()
        modstr = _addindent(modstr, 2)

        params = sum([np.prod(p.size()) for p in module.parameters()])
        weights = tuple([tuple(p.size()) for p in module.parameters()])

        tmpstr += '  (' + key + '): ' + modstr
        if show_weights:
            tmpstr += ', weights={}'.format(weights)
        if show_parameters:
            tmpstr +=  ', parameters={}'.format(params)
        tmpstr += '\n'

    tmpstr = tmpstr + ')'
    return tmpstr

lgr.info(torch_summarize(net))
```

    INFO:__main__:Sequential (
      (0): Linear (29 -> 1024), weights=((1024L, 29L), (1024L,)), parameters=30720
      (1): Dropout (p = 0.05), weights=(), parameters=0
      (2): Tanh (), weights=(), parameters=0
      (3): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True), weights=((1024L,), (1024L,)), parameters=2048
      (4): Linear (1024 -> 128), weights=((128L, 1024L), (128L,)), parameters=131200
      (5): Dropout (p = 0.05), weights=(), parameters=0
      (6): Tanh (), weights=(), parameters=0
      (7): Linear (128 -> 64), weights=((64L, 128L), (64L,)), parameters=8256
      (8): Dropout (p = 0.05), weights=(), parameters=0
      (9): LeakyReLU (0.01), weights=(), parameters=0
      (10): Linear (64 -> 32), weights=((32L, 64L), (32L,)), parameters=2080
      (11): Dropout (p = 0.05), weights=(), parameters=0
      (12): Tanh (), weights=(), parameters=0
      (13): Linear (32 -> 16), weights=((16L, 32L), (16L,)), parameters=528
      (14): Dropout (p = 0.05), weights=(), parameters=0
      (15): LeakyReLU (0.01), weights=(), parameters=0
      (16): Linear (16 -> 1), weights=((1L, 16L), (1L,)), parameters=17
      (17): Sigmoid (), weights=(), parameters=0
    )

#  Loss and Optimizer

###  BCELoss
- In addition, we will calculate the binary cross entropy loss (BCELoss). Luckily we have one loss function already present. For details please checkout http://pytorch.org/docs/master/nn.html.

- ** NOTE this BCELoss may not be numerical stable, although it's fine during my training process.**

### Optimization

- if return F.log_softmax(x) then loss = F.nll_loss(output, target) (MNIST)
- print(nn.BCEWithLogitsLoss()(o, t)) is equivalent to print(nn.BCELoss()(sigmoid(o), t))

```python
# ! pip install sympy
import sympy as sp
sp.interactive.printing.init_printing(use_latex=True)
from IPython.display import display, Math, Latex
maths = lambda s: display(Math(s))
latex = lambda s: display(Latex(s))

#the loss function is as follows:
maths("\mathbf{Loss Function:} J(x, z) = -\sum_k^d[x_k \log z_k + (1-x_k)log(1-z_k)]")
```

$$\mathbf{Loss Function:} J(x, z) = -\sum_k^d[x_k \log z_k + (1-x_k)log(1-z_k)]$$

```python
# optimizer = torch.optim.SGD(net.parameters(), lr=0.02)
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
# optimizer = optim.SGD(net.parameters(), lr=LR, momentum=MOMENTUM, weight_decay=5e-4)

#L2 regularization can easily be added to the entire model via the optimizer
optimizer = torch.optim.Adam(net.parameters(), lr=LR,weight_decay=5e-4) #  L2 regularization
# optimizer = torch.optim.Adagrad(net.parameters(), lr=1e-6, weight_decay=5e-4)
# loss_func = torch.nn.CrossEntropyLoss()  # the target label is NOT an one-hotted
# loss_func = torch.nn.NLLLoss()
loss_func=torch.nn.BCELoss() # Binary cross entropy: http://pytorch.org/docs/nn.html#bceloss
# http://andersonjo.github.io/artificial-intelligence/2017/01/07/Cost-Functions/
# use_cuda=True
if use_cuda:
    lgr.info ("Using the GPU")
    net.cuda()
    loss_func.cuda()
#     cudnn.benchmark = True
    #net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))

lgr.info (optimizer)
lgr.info (loss_func)
```

    INFO:__main__:Using the GPU
    INFO:__main__:<torch.optim.adam.Adam object at 0x7ff6645bca50>
    INFO:__main__:BCELoss (
    )

#  Training in batches  + Measuring the performance of the deep learning model

```python
import time
start_time = time.time()
epochs=250 # change to 1500 for better results
all_losses = []

X_tensor_train= XnumpyToTensor(trainX)
Y_tensor_train= YnumpyToTensor(trainY)

print(type(X_tensor_train.data), type(Y_tensor_train.data)) # should be 'torch.cuda.FloatTensor'

# From here onwards, we must only use PyTorch Tensors
for step in range(epochs):
    out = net(X_tensor_train)                 # input x and predict based on x
    cost = loss_func(out, Y_tensor_train)     # must be (1. nn output, 2. target), the target label is NOT one-hotted

    optimizer.zero_grad()   # clear gradients for next train
    cost.backward()         # backpropagation, compute gradients
    optimizer.step()        # apply gradients

    if step % 50 == 0:
        loss = cost.data[0]
        all_losses.append(loss)
        print(step, cost.data.cpu().numpy())
        # RuntimeError: can't convert CUDA tensor to numpy (it doesn't support GPU arrays).
        # Use .cpu() to move the tensor to host memory first.
        prediction = (net(X_tensor_train).data).float() # probabilities
#         prediction = (net(X_tensor).data > 0.5).float() # zero or one
#         print ("Pred:" + str (prediction)) # Pred:Variable containing: 0 or 1
#         pred_y = prediction.data.numpy().squeeze()
        pred_y = prediction.cpu().numpy().squeeze()
        target_y = Y_tensor_train.cpu().data.numpy()

        tu = ((pred_y == target_y).mean(),log_loss(target_y, pred_y),roc_auc_score(target_y,pred_y ))
        print ('ACC={}, LOG_LOSS={}, ROC_AUC={} '.format(*tu))

end_time = time.time()
print ('{} {:6.3f} seconds'.format('GPU:', end_time-start_time))

%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(all_losses)
plt.show()

false_positive_rate, true_positive_rate, thresholds = roc_curve(target_y,pred_y)
roc_auc = auc(false_positive_rate, true_positive_rate)

plt.title('LOG_LOSS=' + str(log_loss(target_y, pred_y)))
plt.plot(false_positive_rate, true_positive_rate, 'b', label='AUC = %0.6f' % roc_auc)
plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([-0.1, 1.2])
plt.ylim([-0.1, 1.2])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
```

#  Performance of the deep learning model on the Validation set

```python
net.eval()
# Validation data
print (valX.shape)
print (valY.shape)

X_tensor_val= XnumpyToTensor(valX)
Y_tensor_val= YnumpyToTensor(valY)

print(type(X_tensor_val.data), type(Y_tensor_val.data)) # should be 'torch.cuda.FloatTensor'

predicted_val = (net(X_tensor_val).data).float() # probabilities
# predicted_val = (net(X_tensor_val).data > 0.5).float() # zero or one
pred_y = predicted_val.cpu().numpy()
target_y = Y_tensor_val.cpu().data.numpy()

print (type(pred_y))
print (type(target_y))

tu = (str ((pred_y == target_y).mean()),log_loss(target_y, pred_y),roc_auc_score(target_y,pred_y ))
print ('\n')
print ('acc={} log_loss={} roc_auc={} '.format(*tu))

false_positive_rate, true_positive_rate, thresholds = roc_curve(target_y,pred_y)
roc_auc = auc(false_positive_rate, true_positive_rate)

plt.title('LOG_LOSS=' + str(log_loss(target_y, pred_y)))
plt.plot(false_positive_rate, true_positive_rate, 'b', label='AUC = %0.6f' % roc_auc)
plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([-0.1, 1.2])
plt.ylim([-0.1, 1.2])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()

# print (pred_y)
```

#  Submission on Test set

```python
# testX, df_test_set
# df[df.columns.difference(['b'])]
# trainX, trainY, valX, valY, testX, df_test_set = loadDataSplit()

print (df_test_set.shape)
columns = ['id', 'probability']
df_pred=pd.DataFrame(data=np.zeros((0,len(columns))), columns=columns)
df_pred.id.astype(int)

for index, row in df_test_set.iterrows():
    rwo_no_id=row.drop('id')
#     print (rwo_no_id.values)
    x_data_np = np.array(rwo_no_id.values, dtype=np.float32)
    if use_cuda:
        X_tensor_test = Variable(torch.from_numpy(x_data_np).cuda()) # Note the conversion for pytorch
    else:
        X_tensor_test = Variable(torch.from_numpy(x_data_np)) # Note the conversion for pytorch

    X_tensor_test=X_tensor_test.view(1, trainX.shape[1]) # does not work with 1d tensors
    predicted_val = (net(X_tensor_test).data).float() # probabilities
    p_test =   predicted_val.cpu().numpy().item() # otherwise we get an array, we need a single float

    df_pred = df_pred.append({'id':row['id'].astype(int), 'probability':p_test},ignore_index=True)

df_pred.head(5)
```

    (45647, 30)

      id
      probability

      0
      97040.0
      0.519708

      1
      65399.0
      0.517178

      2
      147258.0
      0.518780

      3
      129573.0
      0.519850

      4
      134978.0
      0.522069

#  Create a CSV with the ID's and the coresponding probabilities.

```python
df_pred.id=df_pred.id.astype(int)

def savePred(df_pred, loss):
#     csv_path = 'pred/p_{}_{}_{}.csv'.format(loss, name, (str(time.time())))
    csv_path = 'pred/pred_{}_{}.csv'.format(loss, (str(time.time())))
    df_pred.to_csv(csv_path, columns=('id', 'probability'), index=None)
    print (csv_path)

savePred (df_pred, log_loss(target_y, pred_y))
```

    pred/pred_0.693924313567_1504720448.33.csv

```python

```

day 02 PyTORCH and PyCUDA/PyTorch/18-PyTorch-NUMER.AI-Binary-Classification-BCELoss.ipynb


Table of Contents

1  Partial Dependence Plot

1.1  Individual Conditional Expectation (ICE) Plot

1.2  Implementation

2  Reference

```python
# code for loading the format for the notebook
import os

# path : store the current path to convert back to it later
path = os.getcwd()
os.chdir(os.path.join('..', '..', 'notebook_format'))

from formats import load_style
load_style(css_style = 'custom2.css', plot_style = False)
```

```python
os.chdir(path)

# 1. magic for inline plot
# 2. magic to print version
# 3. magic so that the notebook will reload external python modules
# 4. magic to enable retina (high resolution) plots
# https://gist.github.com/minrk/3301035
%matplotlib inline
%load_ext watermark
%load_ext autoreload
%autoreload 2
%config InlineBackend.figure_format = 'retina'

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pathlib import Path
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

%watermark -a 'Ethen' -d -t -v -p numpy,pandas,matplotlib,sklearn
```

    Ethen 2018-01-24 16:15:24

    CPython 3.5.2
    IPython 6.2.1

    numpy 1.13.3
    pandas 0.21.1
    matplotlib 2.1.0
    sklearn 0.19.1

# Partial Dependence Plot

During the talk, [Youtube: PyData - Random Forests Best Practices for the Business World](https://www.youtube.com/watch?v=E7VLE-U07x0&index=51&list=PLGVZCDnMOq0oqs6RTJk4zZde86DZrgnzm), one of the best practices that the speaker mentioned when using tree-based models is to check for directional relationships. When using non-linear machine learning algorithms, such as popular tree-based models random forest and gradient boosted trees, it can be hard to understand the relations between predictors and model outcome as they do not give us handy coefficients like linear-based models. For example, in terms of random forest, all we get is the feature importance. Although based on that information, we can tell which feature is significantly influencing the outcome based on the importance calculation, it does not inform us in which direction is the predictor influencing outcome. In this notebook, we'll be exploring **Partial dependence plot (PDP)**, a model agnostic technique that gives us an approximate directional influence for a given feature that was used in the model. Note much of the explanation is "borrowed" from the blog post at the following link, [Blog: Introducing PDPbox](https://towardsdatascience.com/introducing-pdpbox-2aa820afd312), this documentation aims to improve upon it by giving a cleaner implementation.

**Partial dependence plot (PDP)** aims to visualize the marginal effect of a given predictor towards the model outcome by plotting out the average model outcome in terms of different values of the predictor. Let's first gain some intuition of how it works with a made up example. Assume we have a data set that only contains three data points and three features (A, B, C) as shown below.

If we wish to see how feature A is influencing the prediction Y, what PDP does is to generate a new data set as follow. (here we assume that feature A only has three unique values: A1, A2, A3)

We then perform the prediction as usual with this new set of data. As we can imagine, PDP would generate **num_rows * num_grid_points** (here, the number of grid point equals the number of unique values of the target feature, more on this later) number of predictions and average them for each unique value of Feature A.

In the end, PDP would only plot out the average predictions for each unique value of our target feature.

Let's now formalize this idea with some notation. The partial dependence function is defined as:

$$
\begin{align}
\hat{f}_{x_S}(x_S) = E_{x_C} \left[ f(x_S, x_C) \right]
\end{align}
$$

The term $x_S$ denotes the set of features for which the partial dependence function should be plotting and $x_C$ are all other features that were used in the machine learning model $f$. In other words, if there were $p$ predictors, $S$ is a subset of our $p$ predictors, $S \subset \left\{ x_1, x_2, \ldots, x_p \right\}$, $C$ would be complementing $S$ such that $S \cup C = \left\{x_1, x_2, \ldots, x_p\right\}$. The function above is then estimated by calculating averages in the training data, which is also known as Monte Carlo method:

$$
\begin{align}
\hat{f}_{x_S}(x_S) = \frac{1}{n} \sum_{i=1}^n f(x_S, x_{Ci})
\end{align}
$$

Where $\left\{x_{C1}, x_{C2}, \ldots, x_{CN}\right\}$ are the values of $X_C$ occurring over all observations in the training data. In other words, in order to calculate the partial dependence of a given variable (or variables), the entire training set must be utilized for every set of joint values. For classification, where the machine learning model outputs probabilities, the partial dependence function displays the probability for a certain class given different values for features $x_s$, a straightforward way to handle multi-class problems is to plot one line per class.

## Individual Conditional Expectation (ICE) Plot

As an extension of a PDP, ICE plot visualizes the relationship between a feature and the predicted responses for each observation. While a PDP visualizes the averaged relationship between features and predicted responses, a set of ICE plots disaggregates the averaged information and visualizes an individual dependence for each observation. Hence, instead of only plotting out the average predictions, ICEbox displays all individual lines. (three lines in total in this case)

The authors of the [Paper: A. Goldstein, A. Kapelner, J. Bleich, E. Pitkin Peeking Inside the Black Box: Visualizing Statistical Learning with Plots of Individual Conditional Expectation](https://arxiv.org/abs/1309.6392) claims with everything displayed in its raw state, any interesting discovers wouldn’t be shielded because of the averaging inherented with PDP. A vivid example from the paper is shown below:

In this example, if we only look at the PDP in Figure b, we would think that on average, the feature X2 is not meaningfully associated with the our target response variable Y. However, if judging from the scatter plot showed in Figure a, this conclusion is plainly wrong. Now if we were to plot out the individual estimated conditional expectation curves, everything becomes more obvious.

After having an understand of the procedure for PDP and ICE plot, we can observe that:

- PDP is a global method, it takes into account all instances and makes a statement about the global relationship of a feature with the predicted outcome.
- One of the main advantage of PDP is that it can be used to interpret the result of any "black box" learning methods.
- PDP can be quite computationally expensive when the data set becomes large.
- Owing to the limitations of computer graphics, and human perception, the size of the subsets $x_S$ must be small (l ≈ 1,2,3). There are of course a large number of such subsets, but only those chosen from among the usually much smaller set of highly relevant predictors are likely to be informative.
- PDP can obfuscate relationship that comes from interactions. PDPs show us how the average relationship between feature $x_S$ and $\hat{y}$ looks like. This works well only in cases where the interactions between $x_S$ and the remaining features $x_C$ are weak. In cases where interactions do exist, the ICE plot may give a lot more insight of the underlying relationship.

## Implementation

We'll be using the [titanic dataset](https://www.kaggle.com/c/titanic/data) (details of the dataset is listed in the link) to test our implementation.

```python
# we download the training data and store it
# under the `data` directory
data_dir = Path('data')
data_path = data_dir / 'train.csv'
data = pd.read_csv(data_path)
print('dimension: ', data.shape)
print('features: ', data.columns)
data.head()
```

    dimension:  (891, 12)
    features:  Index(['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp',
           'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked'],
          dtype='object')

      PassengerId
      Survived
      Pclass
      Name
      Sex
      Age
      SibSp
      Parch
      Ticket
      Fare
      Cabin
      Embarked

      0
      1
      0
      3
      Braund, Mr. Owen Harris
      male
      22.0
      1
      0
      A/5 21171
      7.2500
      NaN
      S

      1
      2
      1
      1
      Cumings, Mrs. John Bradley (Florence Briggs Th...
      female
      38.0
      1
      0
      PC 17599
      71.2833
      C85
      C

      2
      3
      1
      3
      Heikkinen, Miss. Laina
      female
      26.0
      0
      0
      STON/O2. 3101282
      7.9250
      NaN
      S

      3
      4
      1
      1
      Futrelle, Mrs. Jacques Heath (Lily May Peel)
      female
      35.0
      1
      0
      113803
      53.1000
      C123
      S

      4
      5
      0
      3
      Allen, Mr. William Henry
      male
      35.0
      0
      0
      373450
      8.0500
      NaN
      S

```python
# some naive feature engineering
data['Age'] = data['Age'].fillna(data['Age'].median())
data['Embarked'] = data['Embarked'].fillna('S')
data['Sex'] = data['Sex'].apply(lambda x: 1 if x == 'male' else 0)
data = pd.get_dummies(data, columns = ['Embarked'])

# features/columns that are used
label = data['Survived']
features = [
    'Pclass', 'Sex',
    'Age', 'SibSp',
    'Parch', 'Fare',
    'Embarked_C', 'Embarked_Q', 'Embarked_S']
data = data[features]

X_train, X_test, y_train, y_test = train_test_split(
    data, label, test_size = 0.2, random_state = 1234, stratify = label)
```

```python
# fit a baseline random forest model and show its top 2 most important features
rf = RandomForestClassifier(n_estimators = 50, random_state = 1234)
rf.fit(X_train, y_train)

print('top 2 important features:')
imp_index = np.argsort(rf.feature_importances_)
print(features[imp_index[-1]])
print(features[imp_index[-2]])
```

    top 2 important features:
    Fare
    Sex

Aforementioned, tree-based models lists out the top important features, but it is not clear whether they have a positive or negative impact on the result. This is where tools such as partial dependence plots can aid us communicate the results better to others.

```python
from partial_dependence import PartialDependenceExplainer
plt.rcParams['figure.figsize'] = 16, 9

# we specify the feature name and its type to fit the partial dependence
# result, after fitting the result, we can call .plot to visualize it
# since this is a binary classification model, when we call the plot
# method, we tell it which class are we targeting, in this case 1 means
# the passenger did indeed survive (more on centered argument later)
pd_explainer = PartialDependenceExplainer(estimator = rf, verbose = 0)
pd_explainer.fit(data, feature_name = 'Sex', feature_type = 'cat')
pd_explainer.plot(centered = False, target_class = 1)
plt.show()
```

Hopefully, we can agree that the partial dependence plot makes intuitive sense, as for the categorical feature `Sex`, 1 indicates that the passenger was a male. And we know that during the titanic accident, the majority of the survivors were female passenger, thus the plot is telling us male passengers will on average have around 40% chance lower of surviving when compared with female passengers. Also instead of only plotting the "partial dependence" plot, the plot also fills between the standard deviation range. This is essentially borrowing the idea from ICE plot that only plotting the average may obfuscate the relationship.

Centered plot can be useful when we are not interested in seeing the absolute change of a predicted value, but rather the difference in prediction compared to a fixed point of the feature range.

```python
# centered = True is actually the default
pd_explainer.plot(centered = True, target_class = 1)
plt.show()
```

We can perform the same process for numerical features such as `Fare`. We know that more people from the upper class survived, and people from the upper class generally have to pay more Fare to get onboard the titanic. The partial dependence plot below also depicts this trend.

```python
pd_explainer.fit(data, feature_name = 'Fare', feature_type = 'num')
pd_explainer.plot(target_class = 1)
plt.show()
```

If you prefer to create your own visualization, you can call the `results_` attribute to access the partial dependence result. And for those that are interested in the implementation details, the code can be obtained at the following [link](https://github.com/ethen8181/machine-learning/tree/master/model_selection/partial_dependence/partial_dependence.py).

We'll conclude our discussion on parital dependence plot by providing a link to another blog that showcases this method's usefulness in ensuring the behavior of the new machine learning model does intuitively and logically match our intuition and does not differ significantly from a baseline model. [Blog: Using Partial Dependence to Compare Sort Algorithms](http://techblog.hotwire.com/2016/06/13/partial-dependence-compare-sort/)

# Reference

- [Blog: Introducing PDPbox](https://towardsdatascience.com/introducing-pdpbox-2aa820afd312)
- [Online Book: Partial Dependence Plot (PDP)](https://christophm.github.io/interpretable-ml-book/pdp.html)
- [Mathworks Documentation: plotPartialDependence](https://www.mathworks.com/help/stats/regressiontree.plotpartialdependence.html?requestedDomain=true#mw_79dadf51-f451-45a9-a801-2e9ccec37aae)
- [Github: PDPbox - python partial dependence plot toolbox](https://github.com/SauceCat/PDPbox)


# ECE-6524 / CS-6524 Deep Learning
# Assignment 3

In this assignment, **you need to complete the Yolo loss function, and train an object detector. Yay!**

This assignment is inspired and adapted from [UIUC CS498](http://slazebni.cs.illinois.edu/fall18/assignment3_part2.html)
## Submission guideline

1. Click the Save button at the top of the Jupyter Notebook.
2. Please make sure to have entered your Virginia Tech PID below.
3. Select Cell -> All Output -> Clear. This will clear all the outputs from all cells (but will keep the content of cells).
4. Select Cell -> Run All. This will run all the cells in order.
5. Once you've rerun everything, select File -> Download as -> PDF via LaTeX
6. Look at the PDF file and make sure all your solutions are displayed correctly there.
7. Zip the all the files along with this notebook (Please don't include the data)
8. Name your PDF file as Assignment2_[YOUR ID NUMBER].
9. Submit your zipped file and the PDF **INDEPENDENTLY**.
10. **PLEASE DO NOT ZIP YOUR DATASET. ONLY NOTEBOOK/CODE/PDF.**

**While you are encouraged to discuss with your peers, all work submitted is expected to be your own. If you use any information from other resources (e.g. online materials), you are required to cite it below you VT PID. Any violation will result in a 0 mark for the assignment.**

813808996### Please Write Your VT PID Here: 813808996

---

### Reference (if any): https://github.com/xiongzihua/pytorch-YOLO-v1

In this homework, you would need to use **Python 3.6+** along with the following packages:
```
1. pytorch 1.2
2. torchvision
3. numpy
4. matplotlib
5. tqdm (for better, cuter progress bar. Yay!)
```
To install pytorch, please follow the instructions on the [Official website](https://pytorch.org/). In addition, the [official document](https://pytorch.org/docs/stable/) could be very helpful when you want to find certain functionalities.

Note that, on a high-end GPU, it sill takes 3-4 hours to train. **SO START EARLY. IT'S IMPOSSIBLE TO FINISH IT AT THE LAST MINUTE!**

```python
from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/Colab Notebooks/Assignment_3
```

    Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
    /content/drive/My Drive/Colab Notebooks/Assignment_3

```python
import os
import random

import cv2
import numpy as np

import torch
from torch.utils.data import DataLoader
from torchvision import models

from resnet_yolo import resnet50
#from yolo_loss import YoloLoss
from dataset import VocDetectorDataset
from eval_voc import evaluate
from predict import predict_image
from config import VOC_CLASSES, COLORS
from kaggle_submission import output_submission_csv
import matplotlib.pyplot as plt
from tqdm import tqdm

%matplotlib inline
%load_ext autoreload
%autoreload 2
```

## Initialization

```python
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
```

# You Only Look Once: Unified, Real-Time Object Detection [100 pts]
In this assignment, you need to implement the loss function and train the **YOLO object detector** (specfically, YOLO-v1). Here we provide a list of recommend readings for you:
- [YOLO original paper](http://slazebni.cs.illinois.edu/fall18/lec09_detection.pdf) (recommended)
- [Great post about YOLO](https://medium.com/adventures-with-deep-learning/yolo-v1-part-1-cfb47135f81f) on Medium
- [Differences between YOLO, YOLOv2 and YOLOv3
](https://medium.com/@jonathan_hui/real-time-object-detection-with-yolo-yolov2-28b1b93e2088)
- [Great explanation of the Yolo Loss function](https://stats.stackexchange.com/questions/287486/yolo-loss-function-explanation)
- [YOLO on SNL, suggested by UIUC CS498](https://www.youtube.com/watch?v=xZGahvrep3o)

We adopt a variant of YOLO, which:
1. Use pretrained ResNet50 classifier as detector backbone. The pretrained model is offered in `torchvision.models`.
2. Instead of using a $7\times7$ detection grid, we use $14\times14$ to get a more finegrained detection.

In general, the backbone models are usually pretrained on ImageNet dataset (> 1 million images) with numerous classes. As a result, having these pretrained backbone can greatly shorten the required training time, as well as improve the performance. **But still, it takes at least 3-4 hours to train, not to mention that you might need to debug after one training run. So START EARLY, DON'T GO #YOLO!**

You are supposed to get a reasonable detector (like the ... above?) after training the model correctly.

```python
# YOLO network hyperparameters
B = 2  # number of bounding box predictions per cell
S = 14  # width/height of network output grid (larger than 7x7 from paper since we use a different network)
```

## Load the pretrained ResNet classifier
Load the pretrained classifier. By default, it would use the pretrained model provided by `Pytorch`.

```python
load_network_path = None
pretrained = True

# use to load a previously trained network
if load_network_path is not None:
    print('Loading saved network from {}'.format(load_network_path))
    net = resnet50().to(device)
    net.load_state_dict(torch.load(load_network_path))
else:
    print('Load pre-trained model')
    net = resnet50(pretrained=pretrained).to(device)
```

    Load pre-trained model

Some basic hyperparameter settings that you probably don't have to tune.

```python
learning_rate = 0.001
num_epochs = 30
batch_size = 12

# Yolo loss component coefficients (as given in Yolo v1 paper)
lambda_coord = 5
lambda_noobj = 0.5
```

## Implement the YOLO-v1 loss [50 pts]
Now, you have to implement the `YoloLoss` for training your object detector. Please read closely to the [YOLO original paper](http://slazebni.cs.illinois.edu/fall18/lec09_detection.pdf) so that you can implement it.

In general, there are 4 components in the YOLO loss. Consider that we have our prediction grid of size$(N, S, S, 5B+c)$ ( (x, y, w, h, C) for each bounding box, and c is the number of classes), where $N$ is the batch size, $S$ is the grid size, $B$ is the number of bounding boxes. We have :
1. Bounding box regression loss on the bounding box$(x, y, w, h)$
    - $l_{coord}=\sum_{i=0}^{S^2}\sum_{j=0}^B\mathbb{1}^{obj}_{ij}\left[(x_i-\hat{x}_i)^2+(y_i-\hat{y}_i)^2\right]$ + $\sum_{i=0}^{S^2}\sum_{j=0}^B\mathbb{1}^{obj}_{ij}\left[(\sqrt{w_i}-\sqrt{\hat{w}_i})^2+(\sqrt{h_i}-\sqrt{\hat{h}_i})^2\right]$
    - $\mathbb{1}^{obj}_{ij}$: equals to 1 when object appears in cell $i$, and the bounding box $j$ is responsible for the prediction. 0 otherwise.
2. Contain object loss on the confidence prediction $c$ (only calculate for those boxes that actually have objects)
    - $l_{contain}=\sum_{i=0}^{S^2}\sum_{j=0}^B\mathbb{1}^{obj}_{ij}(C_i-\hat{C}_i)^2$
    - $C_i$ the predicted confidence score for cell $i$ from predicted box $j$
    - For each grid cell, you only calculate the contain object loss for the predicted bounding box that has maximum overlap (iou) with the gruond truth box.
    - We say that this predicted box with maximum iou is **responsible** for the prediction.
3. No object loss on the confidence prediction $c$ (only calculate for those boxes that don't have objects)
    - $l_{noobj}=\sum_{i=0}^{S^2}\sum_{j=0}^B\mathbb{1}^{noobj}_{ij}(C_i-\hat{C}_i)^2$
    - $\mathbb{1}^{obj}_{ij}$: equals to 1 when **no object appears** in cell $i$.
4. Classification error loss.
    - $l_{class}=\sum_{i=0}^{S^2}\mathbb{1}_i^{obj}\sum_{c\in classes}\left(p_i(c)-\hat{p_i}(c)\right)^2$
    - $p_i(c)$ is the predicted score for class $c$

Putting them together, we get the yolo loss:
\begin{equation}
yolo=\lambda_{coord}l_{coord}+l_{contain}+\lambda_{noobj}l_{noobj}+l_{class}
\end{equation}
where $\lambda$ are hyperparameters. We have provided detailed comments to gudie you through implementing the loss. So now, please complete the YoloLoss in the code block below. **If you have any problem with regard to implementation, post and discuss it on Piazza.**

```python
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class YoloLoss(nn.Module):
    def __init__(self,S,B,l_coord,l_noobj):
        super(YoloLoss,self).__init__()
        self.S = S
        self.B = B
        self.l_coord = l_coord
        self.l_noobj = l_noobj

    def compute_iou(self, box1, box2):
        '''Compute the intersection over union of two set of boxes, each box is [x1,y1,x2,y2].
        Args:
          box1: (tensor) bounding boxes, sized [N,4].
          box2: (tensor) bounding boxes, sized [M,4].
        Return:
          (tensor) iou, sized [N,M].
        '''
        N = box1.size(0)
        M = box2.size(0)

        lt = torch.max(
            box1[:,:2].unsqueeze(1).expand(N,M,2),  # [N,2] -> [N,1,2] -> [N,M,2]
            box2[:,:2].unsqueeze(0).expand(N,M,2),  # [M,2] -> [1,M,2] -> [N,M,2]
        )

        rb = torch.min(
            box1[:,2:].unsqueeze(1).expand(N,M,2),  # [N,2] -> [N,1,2] -> [N,M,2]
            box2[:,2:].unsqueeze(0).expand(N,M,2),  # [M,2] -> [1,M,2] -> [N,M,2]
        )

        wh = rb - lt  # [N,M,2]
        wh[wh<0] = 0  # clip at 0
        inter = wh[:,:,0] * wh[:,:,1]  # [N,M]

        area1 = (box1[:,2]-box1[:,0]) * (box1[:,3]-box1[:,1])  # [N,]
        area2 = (box2[:,2]-box2[:,0]) * (box2[:,3]-box2[:,1])  # [M,]
        area1 = area1.unsqueeze(1).expand_as(inter)  # [N,] -> [N,1] -> [N,M]
        area2 = area2.unsqueeze(0).expand_as(inter)  # [M,] -> [1,M] -> [N,M]

        iou = inter / (area1 + area2 - inter)
        return iou

    def get_class_prediction_loss(self, classes_pred, classes_target):
        """
        Parameters:
        classes_pred : (tensor) size (batch_size, S, S, 20)
        classes_target : (tensor) size (batch_size, S, S, 20)

        Returns:
        class_loss : scalar
        """

        ##### CODE #####
        class_loss = F.mse_loss(classes_pred,classes_target,size_average=False)

        return class_loss

    def get_regression_loss(self, box_pred_response, box_target_response):
        """
        Parameters:
        box_pred_response : (tensor) size (-1, 5)
        box_target_response : (tensor) size (-1, 5)
        Note : -1 corresponds to ravels the tensor into the dimension specified
        See : https://pytorch.org/docs/stable/tensors.html#torch.Tensor.view_as

        Returns:
        reg_loss : scalar

        """

        ##### CODE #####
        reg_loss = F.mse_loss(box_pred_response[:,:2],box_target_response[:,:2],size_average=False) + F.mse_loss(torch.sqrt(box_pred_response[:,2:4]),torch.sqrt(box_target_response[:,2:4]),size_average=False)

        return reg_loss

    def get_contain_object_loss(self, box_pred_response, box_target_response_iou):
        """
        Parameters:
        box_pred_response : (tensor) size ( -1 , 5)
        box_target_response_iou : (tensor) size ( -1 , 5)
        Note : -1 corresponds to ravels the tensor into the dimension specified
        See : https://pytorch.org/docs/stable/tensors.html#torch.Tensor.view_as

        Returns:
        contain_loss : scalar

        """

        ##### CODE #####
        contain_loss = F.mse_loss(box_pred_response[:,4],box_target_response_iou[:,4],size_average=False)

        return contain_loss

    def get_no_object_loss(self, target_tensor, pred_tensor, no_object_mask):
        """
        Parameters:
        target_tensor : (tensor) size (batch_size, S , S, 30)
        pred_tensor : (tensor) size (batch_size, S , S, 30)
        no_object_mask : (tensor) size (batch_size, S , S)

        Returns:
        no_object_loss : scalar

        Hints:
        1) Create 2 tensors no_object_prediction and no_object_target which only have the
        values which have no object.
        2) Have another tensor no_object_prediction_mask of the same size such that
        mask with respect to both confidences of bounding boxes set to 1.
        3) Create 2 tensors which are extracted from no_object_prediction and no_object_target using
        the mask created above to find the loss.
        """

        ##### CODE #####
        no_object_prediction = pred_tensor[no_object_mask].view(-1, 30)
        no_object_target = target_tensor[no_object_mask].view(-1, 30)
        no_object_prediction_mask = torch.cuda.ByteTensor(no_object_prediction.size())
        no_object_prediction_mask.zero_()
        no_object_prediction_mask[:, 4] = 1
        no_object_prediction_mask[:, 9] = 1
        noo_pred_c = no_object_prediction[no_object_prediction_mask]
        noo_target_c = no_object_target[no_object_prediction_mask]
        no_object_loss = F.mse_loss(noo_pred_c, noo_target_c, reduction='sum')

        return no_object_loss

    def find_best_iou_boxes(self, box_target, box_pred):
        """
        Parameters:
        box_target : (tensor)  size (-1, 5)
        box_pred : (tensor) size (-1, 5)
        Note : -1 corresponds to ravels the tensor into the dimension specified
        See : https://pytorch.org/docs/stable/tensors.html#torch.Tensor.view_as

        Returns:
        box_target_iou: (tensor)
        contains_object_response_mask : (tensor)

        Hints:
        1) Find the iou's of each of the 2 bounding boxes of each grid cell of each image.
        2) Set the corresponding contains_object_response_mask of the bounding box with the max iou
        of the 2 bounding boxes of each grid cell to 1.
        3) For finding iou's use the compute_iou function
        4) Before using compute preprocess the bounding box coordinates in such a way that
        if for a Box b the coordinates are represented by [x, y, w, h] then
        x, y = x/S - 0.5*w, y/S - 0.5*h ; w, h = x/S + 0.5*w, y/S + 0.5*h
        Note: Over here initially x, y are the center of the box and w,h are width and height.
        We perform this transformation to convert the correct coordinates into bounding box coordinates.
        5) Set the confidence of the box_target_iou of the bounding box to the maximum iou

        """

        ##### CODE #####
        coo_response_mask = torch.cuda.ByteTensor(box_target.size())
        coo_response_mask.zero_()
        coo_not_response_mask = torch.cuda.ByteTensor(box_target.size())
        coo_not_response_mask.zero_()

        box_target_iou = torch.zeros(box_target.size()).cuda()

        for i in range(0,box_target.size()[0],2): #choose the best iou box
            box1 = box_pred[i:i+2]
            box1_xyxy = Variable(torch.FloatTensor(box1.size()))
            box1_xyxy[:,:2] = box1[:,:2]/14. -0.5*box1[:,2:4]
            box1_xyxy[:,2:4] = box1[:,:2]/14. +0.5*box1[:,2:4]
            box2 = box_target[i].view(-1,5)
            box2_xyxy = Variable(torch.FloatTensor(box2.size()))
            box2_xyxy[:,:2] = box2[:,:2]/14. -0.5*box2[:,2:4]
            box2_xyxy[:,2:4] = box2[:,:2]/14. +0.5*box2[:,2:4]
            iou = self.compute_iou(box1_xyxy[:,:4],box2_xyxy[:,:4]) #[2,1]
            max_iou,max_index = iou.max(0)
            max_index = max_index.data.cuda()

            coo_response_mask[i+max_index]=1
            coo_not_response_mask[i+1-max_index]=1

            #####
            # we want the confidence score to equal the
            # intersection over union (IOU) between the predicted box
            # and the ground truth
            #####
            box_target_iou[i+max_index,torch.LongTensor([4]).cuda()] = (max_iou).data.cuda()
        box_target_iou = Variable(box_target_iou).cuda()

        return box_target_iou, coo_response_mask

    def forward(self, pred_tensor,target_tensor):
        '''
        pred_tensor: (tensor) size(batchsize,S,S,Bx5+20=30)
                      where B - number of bounding boxes this grid cell is a part of = 2
                            5 - number of bounding box values corresponding to [x, y, w, h, c]
                                where x - x_coord, y - y_coord, w - width, h - height, c - confidence of having an object
                            20 - number of classes

        target_tensor: (tensor) size(batchsize,S,S,30)

        Returns:
        Total Loss
        '''

        N = pred_tensor.size(0)
        total_loss = None

        # Create 2 tensors contains_object_mask and no_object_mask
        # of size (Batch_size, S, S) such that each value corresponds to if the confidence of having
        # an object > 0 in the target tensor.

        ##### CODE #####
        batch_size = pred_tensor.size()[0]

        contains_object_mask = target_tensor[:, :, :, 4] > 0
        no_object_mask = target_tensor[:, :, :, 4] == 0
        contains_object_mask = contains_object_mask.unsqueeze(-1).expand_as(target_tensor)
        no_object_mask = no_object_mask.unsqueeze(-1).expand_as(target_tensor)

        # Create a tensor contains_object_pred that corresponds to
        # to all the predictions which seem to confidence > 0 for having an object
        # Then, split this tensor into 2 tensors :
        # 1) bounding_box_pred : Contains all the Bounding box predictions (x, y, w, h, c) of all grid
        #                        cells of all images
        # 2) classes_pred : Contains all the class predictions for each grid cell of each image
        # Hint : Use contains_object_mask

        ##### CODE #####
        contains_object_pred = pred_tensor[contains_object_mask].view(-1, 30)
        bounding_box_pred = contains_object_pred[:, :10].contiguous().view(-1, 5)
        classes_pred = contains_object_pred[:, 10:]

        ##### CODE #####

        # Similarly, create 2 tensors bounding_box_target and classes_target
        # using the contains_object_mask.

        ##### CODE #####
        contains_object_target = target_tensor[contains_object_mask].view(-1, 30)
        bounding_box_target = contains_object_target[:, :10].contiguous().view(-1, 5)
        classes_target = contains_object_target[:, 10:]

        #Compute the No object loss here
        # Instruction: finish your get_no_object_loss

        ##### CODE #####
        no_object_loss = self.get_no_object_loss(pred_tensor,target_tensor,no_object_mask)

        # Compute the iou's of all bounding boxes and the mask for which bounding box
        # of 2 has the maximum iou the bounding boxes for each grid cell of each image.
        # Instruction: finish your find_best_iou_boxes and use it.
        ##### CODE #####

        box_target_iou, coo_response_mask = self.find_best_iou_boxes(bounding_box_target, bounding_box_pred)

        # Create 3 tensors :
        # 1) box_prediction_response - bounding box predictions for each grid cell which has the maximum iou
        # 2) box_target_response_iou - bounding box target ious for each grid cell which has the maximum iou
        # 3) box_target_response -  bounding box targets for each grid cell which has the maximum iou
        # Hint : Use coo_response_mask

        ##### CODE #####
        box_pred_response = bounding_box_pred[coo_response_mask].view(-1,5)
        box_target_response_iou = box_target_iou[coo_response_mask].view(-1,5)
        box_target_response = bounding_box_target[coo_response_mask].view(-1,5)

        # Find the class_loss, containing object loss and regression loss

        ##### CODE #####

        class_loss = self.get_class_prediction_loss(classes_pred, classes_target)

        contain_loss = self.get_contain_object_loss(box_pred_response, box_target_response_iou)

        reg_loss = self.get_regression_loss(box_pred_response, box_target_response)

        total_loss = self.l_coord*reg_loss + contain_loss + self.l_noobj*no_object_loss + class_loss

        return total_loss / N
```

```python
criterion = YoloLoss(S, B, lambda_coord, lambda_noobj)
optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, weight_decay=5e-4)
```

## Reading Pascal Data

Since Pascal is a small dataset (5000 in train+val) we have combined the train and val splits to train our detector. This is not typically a good practice, but we will make an exception in this case to be able to get reasonable detection results with a comparatively small object detection dataset. Use `download_data.sh` to download the dataset.

The train dataset loader also using a variety of data augmentation techniques including random shift, scaling, crop, and flips. Data augmentation is slightly more complicated for detection dataset since the bounding box annotations must be kept consistent through the transformations.

Since the output of the dector network we train is a $(S, S, 5B+c)$ tensor, we use an encoder to convert the original bounding box coordinates into relative grid bounding box coordinates corresponding to the the expected output. We also use a decoder which allows us to convert the opposite direction into image coordinate bounding boxes.

```python
#!sh download_data.sh
```

```python
file_root_train = 'VOCdevkit_2007/VOC2007/JPEGImages/'
annotation_file_train = 'voc2007.txt'

train_dataset = VocDetectorDataset(root_img_dir=file_root_train,dataset_file=annotation_file_train,train=True, S=S)
train_loader = DataLoader(train_dataset,batch_size=batch_size,shuffle=True,num_workers=4)
print('Loaded %d train images' % len(train_dataset))
```

    Initializing dataset
    Loaded 5011 train images

```python
file_root_test = 'VOCdevkit_2007/VOC2007test/JPEGImages/'
annotation_file_test = 'voc2007test.txt'

test_dataset = VocDetectorDataset(root_img_dir=file_root_test,dataset_file=annotation_file_test,train=False, S=S)
test_loader = DataLoader(test_dataset,batch_size=batch_size,shuffle=False,num_workers=4)
print('Loaded %d test images' % len(test_dataset))
```

    Initializing dataset
    Loaded 4950 test images

```python
import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter('ignore')
```

## Train detector
Now, train your detector.

```python
best_test_loss = np.inf

for epoch in range(num_epochs):
    net.train()

    # Update learning rate late in training
    if epoch == 30 or epoch == 40:
        learning_rate /= 10.0

    for param_group in optimizer.param_groups:
        param_group['lr'] = learning_rate

    print('\n\nStarting epoch %d / %d' % (epoch + 1, num_epochs))
    print('Learning Rate for this epoch: {}'.format(learning_rate))

    total_loss = 0.

    for i, (images, target) in enumerate(tqdm(train_loader, total=len(train_loader))):
        images, target = images.to(device), target.to(device)

        pred = net(images)
        loss = criterion(pred,target)
        total_loss += loss.item()

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    print('Epoch [%d/%d], average_loss: %.4f'
            % (epoch+1, num_epochs, total_loss / (i+1)))

    # evaluate the network on the test data
    with torch.no_grad():
        test_loss = 0.0
        net.eval()
        for i, (images, target) in enumerate(tqdm(test_loader, total=len(test_loader))):
            images, target = images.to(device), target.to(device)

            pred = net(images)
            loss = criterion(pred,target)
            test_loss += loss.item()
        test_loss /= len(test_loader)

    if best_test_loss > test_loss:
        best_test_loss = test_loss
        print('Updating best test loss: %.5f' % best_test_loss)
        torch.save(net.state_dict(),'best_detector.pth')

    torch.save(net.state_dict(),'detector.pth')

```

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 1 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:21<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [1/30], average_loss: 8.1792

    100%|██████████| 413/413 [09:19<00:00,  1.13it/s]

    Updating best test loss: 4.99687

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 2 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [2/30], average_loss: 4.6620

    100%|██████████| 413/413 [03:11<00:00,  2.57it/s]

    Updating best test loss: 4.42587

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 3 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [3/30], average_loss: 4.2238

    100%|██████████| 413/413 [03:11<00:00,  2.58it/s]

    Updating best test loss: 4.09016

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 4 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [4/30], average_loss: 3.8739

    100%|██████████| 413/413 [03:11<00:00,  2.56it/s]

    Updating best test loss: 3.90351

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 5 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [5/30], average_loss: 3.6740

    100%|██████████| 413/413 [03:11<00:00,  2.52it/s]

    Updating best test loss: 3.70457

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 6 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [6/30], average_loss: 3.4790

    100%|██████████| 413/413 [03:11<00:00,  2.58it/s]

    Updating best test loss: 3.55289

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 7 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.12s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [7/30], average_loss: 3.3607

    100%|██████████| 413/413 [03:11<00:00,  2.58it/s]

    Updating best test loss: 3.51387

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 8 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [8/30], average_loss: 3.2288

    100%|██████████| 413/413 [03:11<00:00,  2.57it/s]

    Updating best test loss: 3.34477

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 9 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [9/30], average_loss: 3.0711

    100%|██████████| 413/413 [03:11<00:00,  2.53it/s]

    Updating best test loss: 3.27449

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 10 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [10/30], average_loss: 2.9570

    100%|██████████| 413/413 [03:11<00:00,  2.59it/s]

    Updating best test loss: 3.13985

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 11 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [11/30], average_loss: 2.8817

    100%|██████████| 413/413 [03:11<00:00,  2.59it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 12 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [12/30], average_loss: 2.7810

    100%|██████████| 413/413 [03:12<00:00,  2.57it/s]

    Updating best test loss: 3.07331

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 13 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.12s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [13/30], average_loss: 2.7408

    100%|██████████| 413/413 [03:11<00:00,  2.57it/s]

    Updating best test loss: 3.06498

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 14 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [14/30], average_loss: 2.6689

    100%|██████████| 413/413 [03:10<00:00,  2.56it/s]

    Updating best test loss: 2.97575

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 15 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [15/30], average_loss: 2.6475

    100%|██████████| 413/413 [03:11<00:00,  2.57it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 16 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [16/30], average_loss: 2.6009

    100%|██████████| 413/413 [03:11<00:00,  2.56it/s]

    Updating best test loss: 2.92822

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 17 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [17/30], average_loss: 2.5544

    100%|██████████| 413/413 [03:11<00:00,  2.58it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 18 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [18/30], average_loss: 2.4802

    100%|██████████| 413/413 [03:11<00:00,  2.59it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 19 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [19/30], average_loss: 2.4652

    100%|██████████| 413/413 [03:12<00:00,  2.59it/s]

    Updating best test loss: 2.89087

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 20 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [20/30], average_loss: 2.4429

    100%|██████████| 413/413 [03:12<00:00,  2.57it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 21 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [21/30], average_loss: 2.3739

    100%|██████████| 413/413 [03:12<00:00,  2.58it/s]

    Updating best test loss: 2.87360

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 22 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.12s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [22/30], average_loss: 2.3780

    100%|██████████| 413/413 [03:13<00:00,  2.55it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 23 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [23/30], average_loss: 2.3283

    100%|██████████| 413/413 [03:12<00:00,  2.54it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 24 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [24/30], average_loss: 2.3183

    100%|██████████| 413/413 [03:12<00:00,  2.58it/s]

    Updating best test loss: 2.84059

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 25 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [25/30], average_loss: 2.2958

    100%|██████████| 413/413 [03:12<00:00,  2.57it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 26 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [26/30], average_loss: 2.2287

    100%|██████████| 413/413 [03:12<00:00,  2.58it/s]

    Updating best test loss: 2.83978

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 27 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.11s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [27/30], average_loss: 2.2409

    100%|██████████| 413/413 [03:12<00:00,  2.59it/s]
      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 28 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [28/30], average_loss: 2.1832

    100%|██████████| 413/413 [03:11<00:00,  2.57it/s]

    Updating best test loss: 2.81783

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 29 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:19<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [29/30], average_loss: 2.0975

    100%|██████████| 413/413 [03:12<00:00,  2.53it/s]

    Updating best test loss: 2.74238

      0%|          | 0/418 [00:00<?, ?it/s]

    Starting epoch 30 / 30
    Learning Rate for this epoch: 0.001

    100%|██████████| 418/418 [08:18<00:00,  1.10s/it]
      0%|          | 0/413 [00:00<?, ?it/s]

    Epoch [30/30], average_loss: 2.0500

    100%|██████████| 413/413 [03:12<00:00,  2.55it/s]

    Updating best test loss: 2.73623

Finally, I have 28 epoch. There are 5 more epoches left.

# View example predictions

Now, take a glance at how your detector works:

```python
net.eval()
net.load_state_dict(torch.load('best_detector.pth'))
# select random image from train set
image_name = random.choice(train_dataset.fnames)
image = cv2.imread(os.path.join(file_root_train, image_name))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
threshold = 0.1
print('predicting...')
print(image.shape)
result = predict_image(net, image_name, root_img_directory=file_root_train, threshold=threshold)
for left_up, right_bottom, class_name, _, prob in result:
    color = COLORS[VOC_CLASSES.index(class_name)]
    cv2.rectangle(image, left_up, right_bottom, color, 2)
    label = class_name + str(round(prob, 2))
    text_size, baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.4, 1)
    p1 = (left_up[0], left_up[1] - text_size[1])
    cv2.rectangle(image, (p1[0] - 2 // 2, p1[1] - 2 - baseline), (p1[0] + text_size[0], p1[1] + text_size[1]),
                  color, -1)
    cv2.putText(image, label, (p1[0], p1[1] + baseline), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, 8)

plt.figure(figsize = (15,15))
plt.imshow(image)

```

## Evaluate on Test [50 pts]

To evaluate detection results we use mAP (mean of average precision over each class), You are expected to get an map of at least 49.

```python
test_aps = evaluate(net, test_dataset_file=annotation_file_test, threshold=threshold)
```

    ---Evaluate model on test samples---

    100%|██████████| 4950/4950 [05:34<00:00, 14.32it/s]

    ---class aeroplane ap 0.45382926818827185---
    ---class bicycle ap 0.6133697250842431---
    ---class bird ap 0.40451988083730284---
    ---class boat ap 0.29503429020794214---
    ---class bottle ap 0.18594144803978246---
    ---class bus ap 0.6019470911545964---
    ---class car ap 0.6250108236861638---
    ---class cat ap 0.6546296063528672---
    ---class chair ap 0.2543201768177243---
    ---class cow ap 0.5053320083359611---
    ---class diningtable ap 0.3327077093852767---
    ---class dog ap 0.5858280143227454---
    ---class horse ap 0.6745579682702006---
    ---class motorbike ap 0.5467768098517196---
    ---class person ap 0.5417891772434396---
    ---class pottedplant ap 0.2222798902248187---
    ---class sheep ap 0.4375255366566567---
    ---class sofa ap 0.3593103592786684---
    ---class train ap 0.5611824811669688---
    ---class tvmonitor ap 0.38011590420242436---
    ---map 0.4618004084653887---