```python
%load_ext autoreload
%autoreload 2
%matplotlib inline
```

# Deep Dive into Normalization

```python
from pathlib import Path
from IPython.core.debugger import set_trace
from fastai import datasets
import pickle, gzip, math, matplotlib as mpl
import matplotlib.pyplot as plt
import torch
from torch import nn, optim, tensor
from torch.nn import init
import torch.nn.functional as F
from torch.utils.data import DataLoader, SequentialSampler, RandomSampler
```

```python
# Importing and setting seaborn for improved plots
#import seaborn as sns; sns.set(style='white')
# Importing partials module
from functools import partial
```

## Initial Setup - Taking Previous NBs Into Account

**This is just to illustrate the amount of work that goes into building a customized DL library for experimentation and model building.**

**The best method is to utilize the auto-export script and import all the necessary classes and modules.**

### Exports of NB1

```python
import operator

def test(a,b,cmp,cname=None):
    if cname is None: cname=cmp.__name__
    assert cmp(a,b),f"{cname}:\n{a}\n{b}"

def test_eq(a,b): test(a,b,operator.eq,'==')

from pathlib import Path
from IPython.core.debugger import set_trace
from fastai import datasets
import pickle, gzip, math, torch, matplotlib as mpl
import matplotlib.pyplot as plt
from torch import tensor

MNIST_URL='http://deeplearning.net/data/mnist/mnist.pkl'

def near(a,b): return torch.allclose(a, b, rtol=1e-3, atol=1e-5)
def test_near(a,b): test(a,b,near)
```

### Exports of NB2

```python
def get_data():
    path = datasets.download_data(MNIST_URL, ext='.gz')
    with gzip.open(path, 'rb') as f:
        ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding='latin-1')
    return map(tensor, (x_train,y_train,x_valid,y_valid))

def normalize(x, m, s): return (x-m)/s

def test_near_zero(a,tol=1e-3): assert a.abs()<tol, f"Near zero: {a}"

from torch.nn import init

def mse(output, targ): return (output.squeeze(-1) - targ).pow(2).mean()

from torch import nn
```

### Exports of NB3

```python
import torch.nn.functional as F

def accuracy(out, yb): return (torch.argmax(out, dim=1)==yb).float().mean()

from torch import optim

class Dataset():
    def __init__(self, x, y): self.x,self.y = x,y
    def __len__(self): return len(self.x)
    def __getitem__(self, i): return self.x[i],self.y[i]

from torch.utils.data import DataLoader, SequentialSampler, RandomSampler

def get_dls(train_ds, valid_ds, bs, **kwargs):
    return (DataLoader(train_ds, batch_size=bs, shuffle=True, **kwargs),
            DataLoader(valid_ds, batch_size=bs*2, **kwargs))
```

### Exports of NB4

```python
class DataBunch():
    def __init__(self, train_dl, valid_dl, c=None):
        self.train_dl,self.valid_dl,self.c = train_dl,valid_dl,c

    @property
    def train_ds(self): return self.train_dl.dataset

    @property
    def valid_ds(self): return self.valid_dl.dataset

def get_model(data, lr=0.5, nh=50):
    m = data.train_ds.x.shape[1]
    model = nn.Sequential(nn.Linear(m,nh), nn.ReLU(), nn.Linear(nh,data.c))
    return model, optim.SGD(model.parameters(), lr=lr)

class Learner():
    def __init__(self, model, opt, loss_func, data):
        self.model,self.opt,self.loss_func,self.data = model,opt,loss_func,data

import re

_camel_re1 = re.compile('(.)([A-Z][a-z]+)')
_camel_re2 = re.compile('([a-z0-9])([A-Z])')

def camel2snake(name):
    s1 = re.sub(_camel_re1, r'\1_\2', name)
    return re.sub(_camel_re2, r'\1_\2', s1).lower()

class Callback():
    _order=0
    def set_runner(self, run): self.run=run
    def __getattr__(self, k): return getattr(self.run, k)
    @property
    def name(self):
        name = re.sub(r'Callback$', '', self.__class__.__name__)
        return camel2snake(name or 'callback')

from typing import *

def listify(o):
    if o is None: return []
    if isinstance(o, list): return o
    if isinstance(o, str): return [o]
    if isinstance(o, Iterable): return list(o)
    return [o]

class AvgStats():
    def __init__(self, metrics, in_train): self.metrics,self.in_train = listify(metrics),in_train

    def reset(self):
        self.tot_loss,self.count = 0.,0
        self.tot_mets = [0.] * len(self.metrics)

    @property
    def all_stats(self): return [self.tot_loss.item()] + self.tot_mets
    @property
    def avg_stats(self): return [o/self.count for o in self.all_stats]

    def __repr__(self):
        if not self.count: return ""
        return f"{'train' if self.in_train else 'valid'}: {self.avg_stats}"

    def accumulate(self, run):
        bn = run.xb.shape[0]
        self.tot_loss += run.loss * bn
        self.count += bn
        for i,m in enumerate(self.metrics):
            self.tot_mets[i] += m(run.pred, run.yb) * bn

```

```python

```

### Exports of NB5

```python
def create_learner(model_func, loss_func, data):
    return Learner(*model_func(data), loss_func, data)

def get_model_func(lr=0.5): return partial(get_model, lr=lr)

def annealer(f):
    def _inner(start, end): return partial(f, start, end)
    return _inner

@annealer
def sched_lin(start, end, pos): return start + pos*(end-start)

@annealer
def sched_cos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2
@annealer
def sched_no(start, end, pos):  return start
@annealer
def sched_exp(start, end, pos): return start * (end/start) ** pos

#This monkey-patch is there to be able to plot tensors
torch.Tensor.ndim = property(lambda x: len(x.shape))

def combine_scheds(pcts, scheds):
    assert sum(pcts) == 1.
    pcts = tensor([0] + listify(pcts))
    assert torch.all(pcts >= 0)
    pcts = torch.cumsum(pcts, 0)
    def _inner(pos):
        idx = (pos >= pcts).nonzero().max()
        actual_pos = (pos-pcts[idx]) / (pcts[idx+1]-pcts[idx])
        return scheds[idx](actual_pos)
    return _inner

class Recorder(Callback):
    def begin_fit(self):
        self.lrs = [[] for _ in self.opt.param_groups]
        self.losses = []

    def after_batch(self):
        if not self.in_train: return
        for pg,lr in zip(self.opt.param_groups,self.lrs): lr.append(pg['lr'])
        self.losses.append(self.loss.detach().cpu())

    def plot_lr  (self, pgid=-1): plt.plot(self.lrs[pgid])
    def plot_loss(self, skip_last=0): plt.plot(self.losses[:len(self.losses)-skip_last])

class ParamScheduler(Callback):
    _order=1
    def __init__(self, pname, sched_funcs): self.pname,self.sched_funcs = pname,sched_funcs

    def begin_fit(self):
        if not isinstance(self.sched_funcs, (list,tuple)):
            self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)

    def set_param(self):
        assert len(self.opt.param_groups)==len(self.sched_funcs)
        for pg,f in zip(self.opt.param_groups,self.sched_funcs):
            pg[self.pname] = f(self.n_epochs/self.epochs)

    def begin_batch(self):
        if self.in_train: self.set_param()

def pg_dicts(pgs): return [{'params':o} for o in pgs]

```

### Exports of NB5b

```python
class Callback():
    _order=0
    def set_runner(self, run): self.run=run
    def __getattr__(self, k): return getattr(self.run, k)

    @property
    def name(self):
        name = re.sub(r'Callback$', '', self.__class__.__name__)
        return camel2snake(name or 'callback')

    def __call__(self, cb_name):
        f = getattr(self, cb_name, None)
        if f and f(): return True
        return False

class TrainEvalCallback(Callback):
    def begin_fit(self):
        self.run.n_epochs=0.
        self.run.n_iter=0

    def after_batch(self):
        if not self.in_train: return
        self.run.n_epochs += 1./self.iters
        self.run.n_iter   += 1

    def begin_epoch(self):
        self.run.n_epochs=self.epoch
        self.model.train()
        self.run.in_train=True

    def begin_validate(self):
        self.model.eval()
        self.run.in_train=False

class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass

class Runner():
    def __init__(self, cbs=None, cb_funcs=None):
        self.in_train = False
        cbs = listify(cbs)
        for cbf in listify(cb_funcs):
            cb = cbf()
            setattr(self, cb.name, cb)
            cbs.append(cb)
        self.stop,self.cbs = False,[TrainEvalCallback()]+cbs

    @property
    def opt(self):       return self.learn.opt
    @property
    def model(self):     return self.learn.model
    @property
    def loss_func(self): return self.learn.loss_func
    @property
    def data(self):      return self.learn.data

    def one_batch(self, xb, yb):
        try:
            self.xb,self.yb = xb,yb
            self('begin_batch')
            self.pred = self.model(self.xb)
            self('after_pred')
            self.loss = self.loss_func(self.pred, self.yb)
            self('after_loss')
            if not self.in_train: return
            self.loss.backward()
            self('after_backward')
            self.opt.step()
            self('after_step')
            self.opt.zero_grad()
        except CancelBatchException: self('after_cancel_batch')
        finally: self('after_batch')

    def all_batches(self, dl):
        self.iters = len(dl)
        try:
            for xb,yb in dl: self.one_batch(xb, yb)
        except CancelEpochException: self('after_cancel_epoch')

    def fit(self, epochs, learn):
        self.epochs,self.learn,self.loss = epochs,learn,tensor(0.)

        try:
            for cb in self.cbs: cb.set_runner(self)
            self('begin_fit')
            for epoch in range(epochs):
                self.epoch = epoch
                if not self('begin_epoch'): self.all_batches(self.data.train_dl)

                with torch.no_grad():
                    if not self('begin_validate'): self.all_batches(self.data.valid_dl)
                self('after_epoch')

        except CancelTrainException: self('after_cancel_train')
        finally:
            self('after_fit')
            self.learn = None

    def __call__(self, cb_name):
        res = False
        for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name) and res
        return res

class AvgStatsCallback(Callback):
    def __init__(self, metrics):
        self.train_stats,self.valid_stats = AvgStats(metrics,True),AvgStats(metrics,False)

    def begin_epoch(self):
        self.train_stats.reset()
        self.valid_stats.reset()

    def after_loss(self):
        stats = self.train_stats if self.in_train else self.valid_stats
        with torch.no_grad(): stats.accumulate(self.run)

    def after_epoch(self):
        print(self.train_stats)
        print(self.valid_stats)

class Recorder(Callback):
    def begin_fit(self):
        self.lrs = [[] for _ in self.opt.param_groups]
        self.losses = []

    def after_batch(self):
        if not self.in_train: return
        for pg,lr in zip(self.opt.param_groups,self.lrs): lr.append(pg['lr'])
        self.losses.append(self.loss.detach().cpu())

    def plot_lr  (self, pgid=-1): plt.plot(self.lrs[pgid])
    def plot_loss(self, skip_last=0): plt.plot(self.losses[:len(self.losses)-skip_last])

    def plot(self, skip_last=0, pgid=-1):
        losses = [o.item() for o in self.losses]
        lrs    = self.lrs[pgid]
        n = len(losses)-skip_last
        plt.xscale('log')
        plt.plot(lrs[:n], losses[:n])

class ParamScheduler(Callback):
    _order=1
    def __init__(self, pname, sched_funcs): self.pname,self.sched_funcs = pname,sched_funcs

    def begin_fit(self):
        if not isinstance(self.sched_funcs, (list,tuple)):
            self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)

    def set_param(self):
        assert len(self.opt.param_groups)==len(self.sched_funcs)
        for pg,f in zip(self.opt.param_groups,self.sched_funcs):
            pg[self.pname] = f(self.n_epochs/self.epochs)

    def begin_batch(self):
        if self.in_train: self.set_param()

class LR_Find(Callback):
    _order=1
    def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
        self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr
        self.best_loss = 1e9

    def begin_batch(self):
        if not self.in_train: return
        pos = self.n_iter/self.max_iter
        lr = self.min_lr * (self.max_lr/self.min_lr) ** pos
        for pg in self.opt.param_groups: pg['lr'] = lr

    def after_step(self):
        if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
            raise CancelTrainException()
        if self.loss < self.best_loss: self.best_loss = self.loss
```

### Exports of NB6

```python
# Enable CUDA
#device = torch.device('cuda', 0)
#torch.cuda.set_device(device)
```

```python
torch.set_num_threads(2)

def normalize_to(train, valid):
    m,s = train.mean(),train.std()
    return normalize(train, m, s), normalize(valid, m, s)

class Lambda(nn.Module):
    def __init__(self, func):
        super().__init__()
        self.func = func

    def forward(self, x): return self.func(x)

def flatten(x):      return x.view(x.shape[0], -1)

class CudaCallback(Callback):
    def begin_fit(self): self.model.cuda()
    def begin_batch(self): self.run.xb,self.run.yb = self.xb.cuda(),self.yb.cuda()

class BatchTransformXCallback(Callback):
    _order=2
    def __init__(self, tfm): self.tfm = tfm
    def begin_batch(self): self.run.xb = self.tfm(self.xb)

def view_tfm(*size):
    def _inner(x): return x.view(*((-1,)+size))
    return _inner

def get_runner(model, data, lr=0.6, cbs=None, opt_func=None, loss_func = F.cross_entropy):
    if opt_func is None: opt_func = optim.SGD
    opt = opt_func(model.parameters(), lr=lr)
    learn = Learner(model, opt, loss_func, data)
    return learn, Runner(cb_funcs=listify(cbs))

def children(m): return list(m.children())

class Hook():
    def __init__(self, m, f): self.hook = m.register_forward_hook(partial(f, self))
    def remove(self): self.hook.remove()
    def __del__(self): self.remove()

def append_stats(hook, mod, inp, outp):
    if not hasattr(hook,'stats'): hook.stats = ([],[])
    means,stds = hook.stats
    if mod.training:
        means.append(outp.data.mean())
        stds .append(outp.data.std())

class ListContainer():
    def __init__(self, items): self.items = listify(items)
    def __getitem__(self, idx):
        try: return self.items[idx]
        except TypeError:
            if isinstance(idx[0],bool):
                assert len(idx)==len(self) # bool mask
                return [o for m,o in zip(idx,self.items) if m]
            return [self.items[i] for i in idx]
    def __len__(self): return len(self.items)
    def __iter__(self): return iter(self.items)
    def __setitem__(self, i, o): self.items[i] = o
    def __delitem__(self, i): del(self.items[i])
    def __repr__(self):
        res = f'{self.__class__.__name__} ({len(self)} items)\n{self.items[:10]}'
        if len(self)>10: res = res[:-1]+ '...]'
        return res

from torch.nn import init

class Hooks(ListContainer):
    def __init__(self, ms, f): super().__init__([Hook(m, f) for m in ms])
    def __enter__(self, *args): return self
    def __exit__ (self, *args): self.remove()
    def __del__(self): self.remove()

    def __delitem__(self, i):
        self[i].remove()
        super().__delitem__(i)

    def remove(self):
        for h in self: h.remove()

def get_cnn_layers(data, nfs, layer, **kwargs):
    nfs = [1] + nfs
    return [layer(nfs[i], nfs[i+1], 5 if i==0 else 3, **kwargs)
            for i in range(len(nfs)-1)] + [
        nn.AdaptiveAvgPool2d(1), Lambda(flatten), nn.Linear(nfs[-1], data.c)]

def conv_layer(ni, nf, ks=3, stride=2, **kwargs):
    return nn.Sequential(
        nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride), GeneralRelu(**kwargs))

class GeneralRelu(nn.Module):
    def __init__(self, leak=None, sub=None, maxv=None):
        super().__init__()
        self.leak,self.sub,self.maxv = leak,sub,maxv

    def forward(self, x):
        x = F.leaky_relu(x,self.leak) if self.leak is not None else F.relu(x)
        if self.sub is not None: x.sub_(self.sub)
        if self.maxv is not None: x.clamp_max_(self.maxv)
        return x

def init_cnn(m, uniform=False):
    f = init.kaiming_uniform_ if uniform else init.kaiming_normal_
    for l in m:
        if isinstance(l, nn.Sequential):
            f(l[0].weight, a=0.1)
            l[0].bias.data.zero_()

def get_cnn_model(data, nfs, layer, **kwargs):
    return nn.Sequential(*get_cnn_layers(data, nfs, layer, **kwargs))

def get_learn_run(nfs, data, lr, layer, cbs=None, opt_func=None, uniform=False, **kwargs):
    model = get_cnn_model(data, nfs, layer, **kwargs)
    init_cnn(model, uniform=uniform)
    return get_runner(model, data, lr=lr, cbs=cbs, opt_func=opt_func)

from IPython.display import display, Javascript
def nb_auto_export():
    display(Javascript("""{
const ip = IPython.notebook
if (ip) {
    ip.save_notebook()
    console.log('a')
    const s = `!python notebook2script.py ${ip.notebook_name}`
    if (ip.kernel) { ip.kernel.execute(s) }
}
}"""))
```

## Building the ConvNet

```python
x_train, y_train, x_valid, y_valid = get_data()
# Normalizing
x_train, x_valid = normalize_to(x_train, x_valid)
# Build Datasets
train_ds, valid_ds = Dataset(x_train, y_train), Dataset(x_valid, y_valid)

# Arch
nh, bs = 50, 512
c = y_train.max().item()+1
loss_func = F.cross_entropy

# Create databunch
data = DataBunch(*get_dls(train_ds, valid_ds, bs), c)
```

```python
# Transforming
mnist_view = view_tfm(1, 28, 28)
# Callbacks
cbfs = [Recorder,
       partial(AvgStatsCallback, accuracy),
       CudaCallback,
       partial(BatchTransformXCallback, mnist_view)]
```

```python
nfs = [8, 16, 32, 64, 64]
```

```python
learn, run = get_learn_run(nfs, data, lr=0.4,
                           layer=conv_layer, cbs=cbfs)
```

```python
%time run.fit(2, learn)
```

    train: [1.6192525, tensor(0.4702, device='cuda:0')]
    valid: [0.25496630859375, tensor(0.9239, device='cuda:0')]
    train: [0.310026640625, tensor(0.9089, device='cuda:0')]
    valid: [0.15269254150390624, tensor(0.9537, device='cuda:0')]
    CPU times: user 3.1 s, sys: 307 ms, total: 3.4 s
    Wall time: 2.87 s

## Custom BatchNorm

```python
class BatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        # NB: pytorch bn mom is opposite of what you'd expect
        self.mom, self.eps = mom, eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('vars',  torch.ones(1,nf,1,1))
        self.register_buffer('means', torch.zeros(1,nf,1,1))

    def update_stats(self, x):
        m = x.mean((0,2,3), keepdim=True)
        v = x.var ((0,2,3), keepdim=True)
        self.means.lerp_(m, self.mom)
        self.vars.lerp_(v, self.mom)
        return m,v

    def forward(self, x):
        if self.training:
            with torch.no_grad(): m,v = self.update_stats(x)
        else:
            m,v = self.means,self.vars

        x = (x-m) / (v+self.eps).sqrt()
        return x*self.mults + self.adds
```

```python
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    # No bias needed in case of BN
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(BatchNorm(nf)) #Custom BN
    return nn.Sequential(*layers)
```

```python
# Initalize
def init_cnn_(m, f):
    if isinstance(m, nn.Conv2d):
        f(m.weight, a=0.1)
        if getattr(m, 'bias', None) is not None: m.bias.data.zero_()
    for l in m.children():
        init_cnn_(l, f)

def init_cnn(m, uniform=False):
    f = init.kaiming_uniform_ if uniform else init.kaiming_normal_
    init_cnn_(m, f)

def get_learn_run(nfs, data, lr, layer, cbs=None,
                  opt_func=None, uniform=False, **kwargs):
    model = get_cnn_model(data, nfs, layer, **kwargs)
    init_cnn(model, uniform=uniform)
    return get_runner(model, data, lr=lr, cbs=cbs, opt_func=opt_func)
```

Using this during training to observe how it helps in keeping the activation means to 0 and std to 1.

```python
learn, run = get_learn_run(nfs, data, lr=0.8, layer=conv_layer, cbs=cbfs)
```

```python
with Hooks(learn.model, append_stats) as hooks:
    run.fit(1, learn)
    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks[:-1]:
        ms,ss = h.stats
        ax0.plot(ms[:10])
        ax1.plot(ss[:10])
        h.remove()
    plt.legend(range(6));

    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks[:-1]:
        ms,ss = h.stats
        ax0.plot(ms)
        ax1.plot(ss)
```

After applying BatchNorm using linear interpolation, we can see the improvement in our standard deviations and means.

```python
learn, run = get_learn_run(nfs, data, 1.0, conv_layer, cbs=cbfs)
```

```python
%time run.fit(5, learn)
```

    train: [0.26607884765625, tensor(0.9183, device='cuda:0')]
    valid: [0.2518082275390625, tensor(0.9188, device='cuda:0')]
    train: [0.08462037109375, tensor(0.9741, device='cuda:0')]
    valid: [0.07879784545898437, tensor(0.9751, device='cuda:0')]
    train: [0.0603748388671875, tensor(0.9812, device='cuda:0')]
    valid: [0.11693089599609376, tensor(0.9633, device='cuda:0')]
    train: [0.0490041064453125, tensor(0.9846, device='cuda:0')]
    valid: [0.06964392700195313, tensor(0.9784, device='cuda:0')]
    train: [0.0403047900390625, tensor(0.9866, device='cuda:0')]
    valid: [0.0707970458984375, tensor(0.9793, device='cuda:0')]
    CPU times: user 5.89 s, sys: 35.9 ms, total: 5.92 s
    Wall time: 4.93 s

### Comparison to Built-in BatchNorm

```python
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(nn.BatchNorm2d(nf, eps=1e-5, momentum=0.1))
    return nn.Sequential(*layers)
```

```python
learn, run = get_learn_run(nfs, data, 1.0, conv_layer, cbs=cbfs)
```

```python
%time run.fit(5, learn)
```

    train: [0.23452498046875, tensor(0.9251, device='cuda:0')]
    valid: [0.08072618408203125, tensor(0.9748, device='cuda:0')]
    train: [0.06257984375, tensor(0.9802, device='cuda:0')]
    valid: [0.1142558837890625, tensor(0.9642, device='cuda:0')]
    train: [0.0423348681640625, tensor(0.9868, device='cuda:0')]
    valid: [0.07078927001953125, tensor(0.9797, device='cuda:0')]
    train: [0.03153028076171875, tensor(0.9903, device='cuda:0')]
    valid: [0.057710650634765624, tensor(0.9821, device='cuda:0')]
    train: [0.02128148681640625, tensor(0.9938, device='cuda:0')]
    valid: [0.05555340576171875, tensor(0.9835, device='cuda:0')]
    CPU times: user 5.61 s, sys: 11.9 ms, total: 5.62 s
    Wall time: 4.63 s

### Adding the Scheduler

Adding learning rate annealing:

```python
sched = combine_scheds([0.3, 0.7], [sched_lin(0.6, 2.), sched_lin(2., 0.1)])
```

```python
learn, run = get_learn_run(nfs, data, 0.9, conv_layer, cbs=cbfs +
                           [partial(ParamScheduler, 'lr', sched)])
```

```python
run.fit(10, learn)
```

    train: [0.242242890625, tensor(0.9275, device='cuda:0')]
    valid: [0.1109781005859375, tensor(0.9654, device='cuda:0')]
    train: [0.07245439453125, tensor(0.9776, device='cuda:0')]
    valid: [0.07864197387695313, tensor(0.9756, device='cuda:0')]
    train: [0.05993212890625, tensor(0.9809, device='cuda:0')]
    valid: [0.10018829345703124, tensor(0.9698, device='cuda:0')]
    train: [0.03866418701171875, tensor(0.9874, device='cuda:0')]
    valid: [0.05641264038085937, tensor(0.9833, device='cuda:0')]
    train: [0.02331113037109375, tensor(0.9927, device='cuda:0')]
    valid: [0.05662333984375, tensor(0.9840, device='cuda:0')]
    train: [0.01354578125, tensor(0.9962, device='cuda:0')]
    valid: [0.048208865356445314, tensor(0.9855, device='cuda:0')]
    train: [0.0076075390625, tensor(0.9980, device='cuda:0')]
    valid: [0.04530145263671875, tensor(0.9874, device='cuda:0')]
    train: [0.004427695922851562, tensor(0.9991, device='cuda:0')]
    valid: [0.04266683349609375, tensor(0.9886, device='cuda:0')]
    train: [0.0028356549072265625, tensor(0.9997, device='cuda:0')]
    valid: [0.0436294677734375, tensor(0.9886, device='cuda:0')]
    train: [0.0023242259216308594, tensor(0.9998, device='cuda:0')]
    valid: [0.043790444946289066, tensor(0.9885, device='cuda:0')]

## Additional Norms

### Layer Norm

General equation for a norm layer with learnable affine:

$$y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta$$

The key difference with BN is:
1. Don't keep moving averages.
2. Don't average over the batch dimension but over the hidden dimension. This makes it independent of batch size.
3. Instead of (0, 2, 3), we now have (1, 2, 3).
4. Not nearly as good as BN, but works well enough on RNNs since we can't use BN in that scenario.

```python
# This is the code implementation of the equation above
class LayerNorm (nn.Module):
    __constants__ = ['eps']
    def __init__(self, eps=1e-5):
        super().__init__()
        self.eps = eps
        self.mult = nn.Parameter(tensor(1.))
        self.add = nn.Parameter(tensor(0.))

    def forward(self, x):
        m = x.mean((1, 2, 3), keepdim=True)
        v = x.var((1, 2, 3), keepdim=True)
        x = (x - m) / (v+self.eps).sqrt()
        return x*self.mult + self.add
```

```python
def conv_layer_norm(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(LayerNorm())
    return nn.Sequential(*layers)
```

```python
learn, run = get_learn_run(nfs, data, 0.8, conv_layer_norm, cbs=cbfs)
```

```python
%time run.fit(3, learn)
```

    train: [nan, tensor(0.1380, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    CPU times: user 4.09 s, sys: 16.1 ms, total: 4.1 s
    Wall time: 3.51 s

### Instance Norm

|\begin{equation}\label{eq:bnorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{i}}{\sqrt{\sigma_i^2 + \epsilon}},
    \quad
    \mu_i = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_i^2 = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_i)^2.
\end{equation}

In order to combine the effects of instance-specific normalization and batch normalization, the authors propose to replace the latter by the *instance normalization* (also known as *contrast normalization*) layer:

\begin{equation}\label{eq:inorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{ti}}{\sqrt{\sigma_{ti}^2 + \epsilon}},
    \quad
    \mu_{ti} = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_{ti}^2 = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_{ti})^2.
\end{equation}

This used for style transfer and **NOT** for image classification.

A graphical depiction of the different types of norms :

```python
class InstanceNorm(nn.Module):
    __constants__ = ['eps']
    def __init__(self, nf, eps=1e-0):
        super().__init__()
        self.eps = eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))

    def forward(self, x):
        m = x.mean((2,3), keepdim=True)
        v = x.var ((2,3), keepdim=True)
        res = (x-m) / ((v+self.eps).sqrt())
        return res*self.mults + self.adds
```

```python
def conv_instance_norm(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(InstanceNorm(nf))
    return nn.Sequential(*layers)
```

```python
learn, run = get_learn_run(nfs, data, 0.1, conv_instance_norm, cbs=cbfs)
```

```python
%time run.fit(3, learn)
```

    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    CPU times: user 4.5 s, sys: 19.6 ms, total: 4.52 s
    Wall time: 3.92 s

## Addressing the Issue of Small Batch Sizes

### Problem:

Computing the statistics, i.e. mean and std. deviation, for a BatchNorm Layer on a small batch size gives us a standard deviation very close to 0 due to the lack of sufficient numbers of samples.

```python
data = DataBunch(*get_dls(train_ds, valid_ds, 2), c)
```

```python
def conv_layer(ni, nf, ks=3, stride=2 , bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(nn.BatchNorm2d(nf, eps=1e-5, momentum=0.1))
    return nn.Sequential(*layers)
```

```python
learn, run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs)
```

```python
%time run.fit(2, learn)
```

    train: [2.3357021875, tensor(0.1712, device='cuda:0')]
    valid: [288141.4912, tensor(0.1925, device='cuda:0')]
    train: [2.32542328125, tensor(0.1792, device='cuda:0')]
    valid: [38715509.9648, tensor(0.2931, device='cuda:0')]
    CPU times: user 1min 54s, sys: 775 ms, total: 1min 55s
    Wall time: 1min 54s

Extremely small batch sizes also add to the compute overhead!The performance is abysmal!!

### Solution: Running Batch Norm

As indicated in the lessons, the solution is to use Running BatchNorm, which employs smoother running mean and variance for the mean and std dev.

```python
class RunningBatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        self.mom, self.eps = mom, eps
        self.mults = nn.Parameter(torch.ones(nf, 1, 1))
        self.adds = nn.Parameter(torch.ones(nf, 1, 1))
        self.register_buffer('sums', torch.zeros(1, nf, 1, 1))
        self.register_buffer('sqrs', torch.zeros(1, nf, 1, 1))
        self.register_buffer('batch', torch.tensor(0.))
        self.register_buffer('count', torch.tensor(0.))
        self.register_buffer('step', torch.tensor(0.))
        self.register_buffer('dbias', torch.tensor(0.))

    def update_stats(self, x):
        bs, nc, *_ = x.shape
        self.sums.detach_()
        self.sqrs.detach_()
        dims = (0, 2, 3)
        s = x.sum(dims, keepdim=True)
        ss = (x*x).sum(dims, keepdim=True)
        c = self.count.new_tensor(x.numel() / nc)
        mom1 = 1 - (1-self.mom)/math.sqrt(bs-1)
        self.mom1 = self.dbias.new_tensor(mom1)
        self.sums.lerp_(s, self.mom1)
        self.sqrs.lerp_(ss, self.mom1)
        self.count.lerp_(c, self.mom1)
        self.dbias = self.dbias*(1-self.mom1) + self.mom1
        self.batch += bs
        self.step += 1

    def forward(self, x):
        if self.training: self.update_stats(x)
        sums = self.sums
        sqrs = self.sqrs
        c = self.count
        if self.step < 100:
            sums = sums / self.dbias
            sqrs = sqrs / self.dbias
            c    = c    / self.dbias
        means = sums / c
        vars = (sqrs / c).sub_(means*means)
        if bool(self.batch < 20): vars.clamp_min_(0.01)
        x = (x - means).div_((vars.add_(self.eps)).sqrt())
        return x.mul_(self.mults).add_(self.adds)
```

```python
# Lets apply the Running BatchNorm to a new Conv learner
def conv_running_bn(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride,
                       bias=not bn), GeneralRelu(**kwargs)]
    if bn: layers.append(RunningBatchNorm(nf))
    return nn.Sequential(*layers)
```

```python
learn, run = get_learn_run(nfs, data, 0.4, conv_running_bn, cbs=cbfs)
```

```python
%time run.fit(1, learn)
```

    train: [0.63196484375, tensor(0.8098, device='cuda:0')]
    valid: [17.473784375, tensor(0.9446, device='cuda:0')]
    CPU times: user 2min 18s, sys: 387 ms, total: 2min 19s
    Wall time: 2min 18s

### Maxing out the performance in a single epoch

With a more reasonable batch size...

```python
data = DataBunch(*get_dls(train_ds, valid_ds, 32), c)
```

```python
learn, run = get_learn_run(nfs, data, 0.9, conv_running_bn, cbs=cbfs +
                          [partial(ParamScheduler, 'lr', sched_lin(1., 0.2))])
```

```python
%time run.fit(1, learn)

```

    train: [0.246381328125, tensor(0.9263, device='cuda:0')]
    valid: [372.079275, tensor(0.9726, device='cuda:0')]
    CPU times: user 9.3 s, sys: 28 ms, total: 9.33 s
    Wall time: 9.12 s

```python
# Changing batch size
data = DataBunch(*get_dls(train_ds, valid_ds, 128), c)
```

```python
#cbfs.append(LR_Find)
```

```python
learn, run = get_learn_run(nfs, data, 0.85, conv_running_bn, cbs=cbfs +
                          [partial(ParamScheduler, 'lr', sched_lin(0.9, 0.10))])
```

```python
%time run.fit(1, learn)
```

    train: [0.2676537109375, tensor(0.9299, device='cuda:0')]
    valid: [0.09938456420898438, tensor(0.9758, device='cuda:0')]
    CPU times: user 2.42 s, sys: 7.91 ms, total: 2.42 s
    Wall time: 2.22 s

# Layerwise Sequential Unit Variance (LSUV)

Managing to keep the unit variances of our layer outputs in check as the model trains can prove to be quite a "fiddley" task, especially if we're adding dropout, or changing activation functions. These variations in outputs get exponentially worse as the model trains over multiple epochs.

LSUV shifts this burden to the computer itself.

```python
# Redefining our architecture
nh, bs = 50, 512
data = DataBunch(*get_dls(train_ds, valid_ds, bs), c)
```

```python
# Recreating our ConvLayer class
class ConvLayer(nn.Module):
    # Adding a subtraction hyper parameter
    def __init__(self, ni, nf, ks=3, stride=2, sub=0., **kwargs):
        super().__init__()
        self.conv = nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True)
        self.relu = GeneralRelu(sub=sub, **kwargs)

    def forward(self, x): return self.relu(self.conv(x))

    @property
    def bias(self): return -self.relu.sub
    @bias.setter
    def bias(self, v): self.relu.sub = -v
    @property
    def weight(self): return self.conv.weight

```

Create a learner and runner, without really worrying about how it initializes...

```python
learn, run = get_learn_run(nfs, data, 0.6, ConvLayer, cbs=cbfs)
```

```python
run.fit(2, learn)
```

    train: [1.433776875, tensor(0.5219, device='cuda:0')]
    valid: [0.226480224609375, tensor(0.9305, device='cuda:0')]
    train: [0.30987103515625, tensor(0.9050, device='cuda:0')]
    valid: [0.1582620849609375, tensor(0.9539, device='cuda:0')]

With the initial performance noted, let's recreate the model, this time with LSUV, and we will define a function which grabs a single mini-batch.

```python
learn, run = get_learn_run(nfs, data, 0.6, ConvLayer, cbs=cbfs)
```

```python
def get_batch(dl, runer):
    run.xb, run.yb = next(iter(dl))
    for cb in run.cbs: cb.set_runner(run)
    run('begin_batch')
    return run.xb, run.yb

```

```python
xb, yb = get_batch(data.train_dl, run)
```

Now that we have our mini-batch, we will use a function which (using recursion) only gives us the outputs of the convolutional layers.

```python
def find_modules(m, cond):
    if cond(m): return [m]
    return sum([find_modules(o, cond) for o in m.children()], [])

def is_lin_layer(l):
    lin_layers = (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear, nn.ReLu)
    return isinstance(l, lin_layers)
```

```python
mods = find_modules(learn.model, lambda o: isinstance(o, ConvLayer))
```

```python
mods
```

    [ConvLayer(
       (conv): Conv2d(1, 8, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2))
       (relu): GeneralRelu()
     ), ConvLayer(
       (conv): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
       (relu): GeneralRelu()
     ), ConvLayer(
       (conv): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
       (relu): GeneralRelu()
     ), ConvLayer(
       (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
       (relu): GeneralRelu()
     ), ConvLayer(
       (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
       (relu): GeneralRelu()
     )]

Adding another helper function to grab the mean and std of the output of a hooked layer.

```python
def append_stat(hook, mod, inp, outp):
    d = outp.data
    hook.mean, hook.std = d.mean().item(), d.std().item()
```

```python
mdl = learn.model.cuda()
```

```python
with Hooks(mods, append_stat) as hooks:
    mdl(xb)
    for hook in hooks: print(hook.mean, hook.std)
```

    0.3897072970867157 0.6319916248321533
    0.33694127202033997 0.5548509955406189
    0.2168125957250595 0.3948383331298828
    0.21095244586467743 0.3277290463447571
    0.1627117097377777 0.2220781147480011

Here, our means are too high and the std. devs. are not close to 1. Therefore, we will adjust the bias terms to make the means 0 and then std. devs. must be adjusted to 1 (with a threshold of 1e-3).

```python
def lsuv_module(m, xb):
    h = Hook(m, append_stat)

    # mdl(xb) is not None exists to pass xb through mdl while computing
    # all activations in order to update the hooks.

    while mdl(xb) is not None and abs(h.mean) > 1e-3: m.bias -= h.mean
    while mdl(xb) is not None and abs(h.std-1)> 1e-3: m.weight.data /= h.std

    h.remove()
    return h.mean, h.std
```

Executing the initialization on all conv layers in order...

```python
for m in mods: print(lsuv_module(m, xb))
```

    (0.22692637145519257, 1.0000001192092896)
    (0.11001376807689667, 1.0)
    (0.15807662904262543, 0.9999999403953552)
    (0.1515551060438156, 1.0000001192092896)
    (0.2991049587726593, 1.0)

Now that our means and std. devs. are much more acceptable, the model will begin training on much better grounds.

```python
%time run.fit(2, learn)
```

    train: [0.4551003125, tensor(0.8555, device='cuda:0')]
    valid: [0.1386651123046875, tensor(0.9577, device='cuda:0')]
    train: [0.113353916015625, tensor(0.9646, device='cuda:0')]
    valid: [0.10336754150390624, tensor(0.9667, device='cuda:0')]
    CPU times: user 2.37 s, sys: 0 ns, total: 2.37 s
    Wall time: 1.97 s

**Note** LSUV is very useful for more complex and deeper architectures where initializations are hard to optimize from the get go.

```python

```


# 60 minutos sobre XGBoost

> My confession: I (over)use GBM. When in doubt, use GBM.

-- Owen Zhang, 2015;[ _"Winning Data Science Competitions"_ @ NYC Data Science Academy](https://www.youtube.com/watch?v=LgLcfZjNF44)

## Israel Zúñiga de la Mora | 01/Mar/202 @ Saturdays.ai Mty

# Objetivo de esta plática:
Aprender el uso básico de la biblioteca de Machine Lerning XGBoost para Gradient Boosting en Python.

# Que no veremos hoy (AKA: the proof is left as an exercise to the reader):
- Uso avanzado de XGBoost:
    - Selección de features por puntuación (XGBoost + Scikit-Learn's `SelectFromModel` class)
    - Monitoreo de performance en entrenamiento y _Early Stopping_
    - Tuning: Multithreading, Gradient Boosting Estimators, Learning Rate
    - Stochastic Gradient Boosting
    - Serialización de modelos (Pickle/Joblib)
- Uso y/o despliegue a Producción en AWS/GCP/Azure/Docker/Kubeflow/Polyaxon/MLFlow
- Comparación con otras implementaciones similares (Microsoft's LightGBM, Yandex's CatBoost, H2O's GBM)
- Formalidad en Teoría Matemática/CompSci sobre algoritmos implementados:
    - [Kaggle Winning Solution Xgboost algorithm -- Let us learn from its author](https://www.slideshare.net/ShangxuanZhang/kaggle-winning-solution-xgboost-algorithm-let-us-learn-from-its-author)
    - [Original XGBoost Paper](https://arxiv.org/pdf/1603.02754v3.pdf)
        - [Slides(Tianqi Chen)](https://homes.cs.washington.edu/~tqchen/data/pdf/BoostedTree.pdf)
    - [Documentation](https://xgboost.readthedocs.io/en/latest/)

# Outline

- Teoría
    - Decision Trees
    - Gradient Boosting
    - Tree Boosting
- XGBoost
    - Qué es XGBoost?
    - Características principales
    - Por qué usar XGBoost?
- Nuestro primero modelo 🧶🐈
    - Instalación (aunque este stand-up no es un tutorial)
    - Descripción del problema
    - Carga de datos
    - Entrenamiento del modelo
    - Generación de predicciones ($\hat{Y}$)
- Visualización de árboles de decisión e imporancia de features
    - Feature Importance plot
    - Tree Graph
- Preparación de datos para GB
- (si da tiempo: Evaluación de Modelos)

# Decision Trees

- [A visual introduction to machine learning](http://www.r2d3.us/visual-intro-to-machine-learning-part-1/)
- [The Mechanics of Machine Learning: How Machine Learning Works](https://mlbook.explained.ai/intro.html)

# Gradient Boosting

GB es una de las técnicas más populares en ML para ganar competencias de Kaggle sobre datos estructurados (y con el feature engineering adecuado), entrenar sobre datasets en los que no tenemos _business expertise_ para descubrir variables con poder predictivo o solucionar casi cualquier problema de aprendiza supervisado y ganar dinero rápido.

## Origen de _Boosting_
La idea fue articulada por Michael Kearns sobre un clasificador débil (weaker learner\*) que puede convertirse en eficiente sobre el tiempo:

> ... an efficient algorithm for converting relatively poor hypotheses into very good hypotheses — Thoughts on Hypothesis Boosting, 1988.

Hypothesis boosting was the idea of filtering observations,
leaving those observations that the weak learner can handle
and focusing on developing new weak learns to handle the remaining difficult observations.

## AdaBoosting

AdaBoost y otros algoritmos relacionados son propuestos como un marco de aprendizaje estadístico por Leo Breiman como ARCing Algorithms:

> Arcing is an acronym for Adaptive Reweighting and Combining. Each step in an arcing algorithm consists of a weighted minimization followed by a recomputation of [the classifiers] and [weighted input].  — Prediction Games and Arching Algorithms, 1999

## AdaBoosting

Después, es extendido en 2001 por Jerome H. Friedman y llamado _Gradient Boosting Machine_

Se establece el GB como un problema de optimización donde la función objetivo es minimizar la pérdida del modelo al agregar clasificadores débiles mediante gradient descent/descenso de gradiente. Este algoritmo describe un modelo aditivo por etapas:

Un nuevo clasificador débil es agregado a la vez a una estructura de clasificadores que no se modifican.

## AdaBoosting

Figura: Wang, Zhuo & Zhang, Jintao & Verma, Naveen. (2015). Realizing Low-Energy Classification Systems by Implementing Matrix Multiplication Directly Within an ADC. IEEE Transactions on Biomedical Circuits and Systems. 9. 1-1. 10.1109/TBCAS.2015.2500101.

## Entonces cómo funciona Gradient Boosting?

Se involucran tres elementos:

- Una función de pérdida a ser optimizada
- Un clasificador débil que hace prediccciones
- Un modelo aditivo que incrementa sus clasificadores para minimizar la función de pérdida

### Gradient Boosting

- **Weak learner**: es un clasificador que sin importar la distribución del dataset de entrenamiento, siempre será mejor que una elección al azar, cuando intenta etiquetar ($\hat{Y}$) los datos de una observación ($X$).

### Gradient Boosting

- Regularization: Mide la complejidad del modelo
- Training Loss: Mide que tan bien se ajusta nuestro modelo a los datos

### Gradient Boosting

- **Training loss functions**: Métodos para la evaluación de que tan bien un algoritmo en particular está modelando las observaciones dadas.
Si la predicción $\hat{Y}$ se desvía demasiado del resultado actual ${Y}$ la función de pérdida otorgará un valor muy alto. Gradualmente, con la ayuda de una función de optimización, la función de pérdida aprende a reducir el error en la predicción)

### Gradient Boosting

- *Modelo Aditivo*: Es un modelo donde se añaden árboles a la vez, y los existentes no se modifican. Se usa gradient descent para minimizar la pérdida cuando se agrega un nuevo arbol.

> One way to produce a weighted combination of classifiers which optimizes [the cost] is by gradient descent in function space. — Boosting Algorithms as Gradient Descent in Function Space, 1999. Llew Mason

Normalmente se usa el descenso de gradiente para minimizar parámetros, p. ej. el coeficiente un una ecuación de regresión o los pesos de una NN. Después de calcular el error o la pérdida, los pesos internos se actualizan para minimizar ese error.
En vez de parametros, los clasificadores débiles (decision trees). Después de calcular la pérdidad, para calcular el descenso de gradiente, debemos añadir un arbol para minimizar la pérdida (seguir el gradiente). Este approach es conocido como functional gradient descent.

## Mejoras al Gradient Boosting

Dado que GB es un algoritmo agresivo, puede sobre-entrenar en el dataset. Usando funciones de regularización que penalizen a la implementación del algoritmo:

- Constantes para árboles
- Contracción (shrinkage/learning rate)
- Random Sampling (Stochastic GB)
- Penalized Gradient Boosting (Regularized Gradient Boosting:  𝐿1 / 𝐿2 regularization of weights )

También hay dos técnicas de regularización adicionales para evitar el overfitting. El primero es la contracción (shrinkage), que agrega un factor de escala a los nuevos pesos después de cada paso de refuerzo. Esto evita que algunos árboles individuales dominen el conjunto, dejando espacio para mejorar. La segunda técnica es el submuestreo de características/features (feature subsampling), donde se muestrea una fracción de las observaciones en cada paso, para aumentar la variedad de árboles construidos. El submuestreo de características también aumenta la velocidad de cálculo, permitiendo que XGBoost se escale de manera eficiente.

\begin{equation}\label{eq:loss}
\begin{split}
\mathcal{L}(\phi) =& \sum_{i} l( \hat{y}_i, y_i ) + \sum_{k}\Omega( f_{k} ) \\
 \mbox{ donde }& \Omega(f) = \gamma T + \frac{1}{2} \lambda \|w\|^2
 \end{split}
\end{equation}

L1 regularization of weights. L2 regularization of weights.
The additional regularization term helps to smooth the final learnt weights to avoid over-fitting. Intuitively, the regularized objective will tend to select a model employing simple and predictive functions.

# Tree Boosting

## Regularized learning objective

Como en la mayoría de modelos, minimizar la pérdida (la diferencia entre la predicción $\hat{Y}$ y la etiqueta ${Y}$ ) es la meta principal y queremos que el modelo sea lo más exacto posible. También es importante agregar un término de regularización ($\Omega$) para evitar el Overfitting/sobre-entrenamiento.  Expresamos la función de objetivo de la siguiente forma

$$
\begin{equation}\label{eq:loss}
\begin{split}
\mathcal{L}(\phi) =& \sum_{i} l( \hat{y}_i, y_i ) + \sum_{k}\Omega( f_{k} ) \\
 \end{split}
\end{equation}
$$

Here $l$ is a differentiable convex loss function that measures the difference between the prediction $\hat{y}_i$ and the target $y_i$. The second term $\Omega$ penalizes the complexity of the model (i.e., the regression tree functions).
The additional regularization term helps to smooth the final learnt weights to avoid over-fitting.
Intuitively, the regularized objective will tend to select a model employing simple and predictive functions.
A similar regularization technique has been used in Regularized greedy forest~(RGF)~ model.
Our objective and the corresponding learning algorithm is simpler than RGF and easier to parallelize.
When the regularization parameter is set to zero, the objective falls back to the traditional gradient tree boosting.

También hay dos técnicas de regularización adicionales para evitar el sobreajuste. El primero es la contracción (shrinkage), que agrega un factor de escala a los nuevos pesos después de cada paso de refuerzo. Esto evita que algunos árboles individuales dominen el conjunto, dejando espacio para mejorar. La segunda técnica es el submuestreo de características/features (feature subsampling), donde se muestrea una fracción de las observaciones en cada paso, para aumentar la variedad de árboles construidos. El submuestreo de características también aumenta la velocidad de cálculo, permitiendo que XGBoost se escale de manera eficiente.

\begin{equation}\label{eq:loss}
\begin{split}
\mathcal{L}(\phi) =& \sum_{i} l( \hat{y}_i, y_i ) + \sum_{k}\Omega( f_{k} ) \\
 \mbox{ donde }& \Omega(f) = \gamma T + \frac{1}{2} \lambda \|w\|^2
 \end{split}
\end{equation}

# Tree Boosting

## Gradient tree boosting

Cuando entrenamos un modelo, es común optimizarlo tomando el gradiente. Sin embargo, esto es difícil de realizar en un conjunto de árboles. En su lugar, usamos una estrategia aditiva y agregamos un árbol nuevo a la vez, de modo que cada árbol que agreguemos optimice el objetivo. Esto nos da una función de puntuación que podemos usar para evaluar la calidad de una estructura de árbol.

En pocas palabras, $g_i$ y $h_i$ se ordenan en sus hojas apropiadas, se suman y luego se puntúan.

# Tree Boosting

Ahora que tenemos un método para evaluar la calidad de la estructura de árbol, necesitamos aprender la estructura del árbol. Es imposible enumerar/completar esto en un solo paso para todos los árboles posibles, por lo que se optimiza un nivel de árbol a la vez. Comenzando con una división en una hoja, calculamos la ganancia, que viene dada por:

\begin{equation}\label{eq:gain}
      \mathcal{L}_{split} =\frac{1}{2} \left[\frac{(\sum_{i\in I_L} g_i)^2}{\sum_{i\in I_L} h_i + \lambda}+\frac{(\sum_{i\in I_R} g_i)^2}{\sum_{i\in I_R} h_i + \lambda} - \frac{(\sum_{i\in I} g_i)^2}{\sum_{i\in I} h_i + \lambda}\right] - \gamma
\end{equation}

---
Donde $I_j=\{i|q(x_i)=j\}$ es el conjunto de instancias de la hoja  $j$

# XGBoost: eXtreme Gradient Boosting

XGBoost es una implementación **Open Source** de Gradient Boosting Machines/Gradient Boosted Decision Trees creada por Tianqi Chen, en co-autoría con Carlos Guestrin, (del grupo DMLC https://dmlc.github.io/people.html, también creadores de MXNet para Deep Learning). Siendo pensada desde el origen para entrenar con velocidad de cómputo y performance.

Escrita en C++, está disponible para los SO más populares (Windows, OS X, Linux) con las siguientes interfaces:

- Command Line Inferface (CLI)
- C++
- Python
- R
- Julia
- JVM: Java (Hadoop), Scala (Spark)

## XGBoost Features

## Model Features
Soporta tres maneras (implementaciones de algoritmos) de Gradient Boosting:

- Gradient Boosting/Gradient Boosting Machine (Learning Rate)
- Stochastic Gradient Boosting (sub-sampling: row & column)
- Regularized Gradient Boosting ($L^1$, $L^2$)

## System Features
Uso de algoritmos y estructuras de datos optimizados para exprimir nuestro hardware:

- Paralelización: Uso de todos los cores durante entrenamiento (construcción de árboles)
- Cómputo Distribuido (HPC: OpenMP, Spark, Yarn, CUDA/NCCL, Dask, Dask+Kubernetes): Para entrenamiento de modelos en clusters de cómputo
- Out-of-Core Computing: Para datasets que no caben en memoria RAM

## Algorithm Features

- Manejo automático de valores faltantes (Sparsity-aware Split Finding Algorithm)
- Continued Training: Entrenamiento incremental sobre modelos previos

## Por qué usar XGBoost?

En resumen, usamos XGBoost por las siguientes razones:
- Fácil de usar e instalar
- Eficiencia
    - Rápido
    - Parelizalización sin dolor en una sola computadora o en cluster \*
- Exactitud
    - Buenos resultados para la mayoría de datasets (Pruning, Gradient Optimization)
- Personalización:
    - Paramétros
    - Cambios en la función Objetivo y evaluación

----
\* Sabemos lo que has llorado al implementar en tu lenguaje favorito:

- Concurrency (Threading, Async IO/Cooperative Multitasking)
- Parallelism (Multiprocessing).

### Velocidad de entrenamiento (I)

#### [Benchmarking Random Forest Implementations, Szilard Pafka. 2015](http://datascience.la/benchmarking-random-forest-implementations/)

> In our experiments, random forests with 500 trees have been trained in each tool with default hyper-parameter values. The training times and AUC as a function of the dataset size are plotted in the figures below (with more details available on Github).
------

> I also tried xgboost, a popular library for boosting which is capable to build random forests as well. It is fast, memory efficient and of high accuracy. Note the different shapes of the AUC and runtime vs dataset sizes for H2O and xgboost, however.

-----

### Velocidad de entrenamiento (II)

#### [Xgboost: A scalable tree boosting system. T Chen, C Guestrin](https://arxiv.org/abs/1603.02754)

> Figure 12: Comparison of different distributed systems on 32 EC2 nodes for 10 iterations on different subset of criteo data. XGBoost runs more 10x than spark per iteration and 2.2x as H2O’s optimized version (However, H2O is slow in loading the data, getting worse end-to-end time). Note that spark suffers from drastic slow down when running out of memory. XGBoost runs faster and scales smoothly to the full 1.7 billion examples with given resources by utilizing out-of-core computation.
------

# Nuestro primero modelo 🧶🐈
    - Instalación (aunque este stand-up no es un tutorial)
    - Descripción del problema
    - Carga de datos
    - Entrenamiento del modelo
    - Generación de predicciones

## Instalación:

### Anaconda:
`$ conda install -c conda-forge xgboost`

### PIP:
`$ pip3 install xgboost`

#### Para compilación (GCC / Mingw) y soporte de GPU (Linux): https://xgboost.readthedocs.io/en/latest/build.html

## Instalación (de complementos):

- NumPy
- SciPy
- Pandas
- Scikit-Learn
- Matplotlib
- Graphviz (`$ pip install graphviz` / `$ conda install graphviz && conda install python-graphviz`)

## Descripción del problema

### Usando Aprendizaje Supervisado (clasificación) y el dataset Digits de NIST, entrenaremos un modelo que reconozca digitos escritos a mano.

- 5620 registros por 64 atributos ( cada registro representa una imagen de 8x8 pixeles en el rango de 0 a 16
- 10 clases únicas

# The process of building a ML product

# Typical Machine Learning Workflow (I)

# Typical Machine Learning Workflow (II)

### Carga de datos (y paquetes)

```python
# Numeric
import numpy as np

# DataFrame Support
import pandas as pd

#ML Algos
from xgboost import XGBClassifier

# ML toolkit

from sklearn.model_selection import train_test_split, KFold, GridSearchCV
from sklearn.metrics import mean_absolute_error, mean_squared_error, median_absolute_error, r2_score
from sklearn.metrics import confusion_matrix

# Feature Viz
from xgboost import to_graphviz, plot_tree, plot_importance

# Graphing
import matplotlib.pyplot as plt
%matplotlib nbagg
```

```python
# The digits dataset
digits = datasets.load_digits()

# To apply a classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
data = digits.images.reshape((n_samples, -1))
```

```python
# split data into train and test sets
seed = 42
test_size = 0.33
# Split data into train and test subsets
X_train, X_test, y_train, y_test = train_test_split(
    data, digits.target, test_size=0.5, shuffle=False, random_state=seed)
```

# Entrenamiento del modelo (con un solo core)

```python
# Create a classifier: XGBoostClassifier!
classifier = XGBClassifier(nthread=1)
```

```python
# We learn the digits on the 33% of the digits
classifier.fit(X_train, y_train)
```

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
                  nthread=None, objective='multi:softprob', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1)

# Entrenamiento del modelo (con un todos los cores disponibles)

```python
# Create a classifier: XGBoostClassifier!
classifier = XGBClassifier(n_jobs=-1)
```

```python
# We learn the digits on the 33% of the digits
%time classifier.fit(X_train, y_train)
```

    CPU times: user 3.12 s, sys: 13.9 ms, total: 3.14 s
    Wall time: 3.14 s

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
                  nthread=None, objective='multi:softprob', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1)

## Generación de predicciones ($\hat{Y}$)

```python
# Now predict the value of the digit on the rest:
predicted = classifier.predict(X_test)
```

```python
_, axes = plt.subplots(2, 4)
images_and_labels = list(zip(digits.images, digits.target))
for ax, (image, label) in zip(axes[0, :], images_and_labels[:4]):
    ax.set_axis_off()
    ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
    ax.set_title('Training: %i' % label)

images_and_predictions = list(zip(digits.images[n_samples // 2:], predicted))
for ax, (image, prediction) in zip(axes[1, :], images_and_predictions[:4]):
    ax.set_axis_off()
    ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
    ax.set_title('Prediction: %i' % prediction)

plt.show()
```

    <IPython.core.display.Javascript object>

```python
print("Classification report for classifier %s:\n%s\n"
      % (classifier, metrics.classification_report(y_test, predicted)))
disp = metrics.plot_confusion_matrix(classifier, X_test, y_test)
disp.figure_.suptitle("Confusion Matrix")
print("Confusion matrix:\n%s" % disp.confusion_matrix)
```

    Classification report for classifier XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
                  nthread=None, objective='multi:softprob', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1):
                  precision    recall  f1-score   support

               0       0.93      0.99      0.96        88
               1       0.86      0.82      0.84        91
               2       0.96      0.90      0.93        86
               3       0.86      0.82      0.84        91
               4       0.96      0.87      0.91        92
               5       0.89      0.91      0.90        91
               6       0.93      0.95      0.94        91
               7       0.98      0.93      0.95        89
               8       0.86      0.88      0.87        88
               9       0.80      0.93      0.86        92

        accuracy                           0.90       899
       macro avg       0.90      0.90      0.90       899
    weighted avg       0.90      0.90      0.90       899

    <IPython.core.display.Javascript object>

    Confusion matrix:
    [[87  0  0  0  0  0  0  0  1  0]
     [ 0 75  0  3  2  0  0  0  1 10]
     [ 1  0 77  7  0  0  0  0  0  1]
     [ 0  3  0 75  0  3  0  2  8  0]
     [ 4  0  0  0 80  0  4  0  0  4]
     [ 0  1  0  0  0 83  2  0  0  5]
     [ 0  2  2  1  0  0 86  0  0  0]
     [ 0  0  1  0  1  3  0 83  1  0]
     [ 0  6  0  1  0  2  0  0 77  2]
     [ 2  0  0  0  0  2  0  0  2 86]]

## Visualización de árboles de decisión e imporancia de features
    - Feature Importance plot
    - Tree Graph

```python
# Modelo
# 284,807 registros x  31 columnas
dataset = pd.read_csv('creditcard.csv')
dataset.head()
```

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Time</th>
      <th>V1</th>
      <th>V2</th>
      <th>V3</th>
      <th>V4</th>
      <th>V5</th>
      <th>V6</th>
      <th>V7</th>
      <th>V8</th>
      <th>V9</th>
      <th>V10</th>
      <th>V11</th>
      <th>V12</th>
      <th>V13</th>
      <th>V14</th>
      <th>V15</th>
      <th>V16</th>
      <th>V17</th>
      <th>V18</th>
      <th>V19</th>
      <th>V20</th>
      <th>V21</th>
      <th>V22</th>
      <th>V23</th>
      <th>V24</th>
      <th>V25</th>
      <th>V26</th>
      <th>V27</th>
      <th>V28</th>
      <th>Amount</th>
      <th>Class</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.0</td>
      <td>-1.359807</td>
      <td>-0.072781</td>
      <td>2.536347</td>
      <td>1.378155</td>
      <td>-0.338321</td>
      <td>0.462388</td>
      <td>0.239599</td>
      <td>0.098698</td>
      <td>0.363787</td>
      <td>0.090794</td>
      <td>-0.551600</td>
      <td>-0.617801</td>
      <td>-0.991390</td>
      <td>-0.311169</td>
      <td>1.468177</td>
      <td>-0.470401</td>
      <td>0.207971</td>
      <td>0.025791</td>
      <td>0.403993</td>
      <td>0.251412</td>
      <td>-0.018307</td>
      <td>0.277838</td>
      <td>-0.110474</td>
      <td>0.066928</td>
      <td>0.128539</td>
      <td>-0.189115</td>
      <td>0.133558</td>
      <td>-0.021053</td>
      <td>149.62</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>0.0</td>
      <td>1.191857</td>
      <td>0.266151</td>
      <td>0.166480</td>
      <td>0.448154</td>
      <td>0.060018</td>
      <td>-0.082361</td>
      <td>-0.078803</td>
      <td>0.085102</td>
      <td>-0.255425</td>
      <td>-0.166974</td>
      <td>1.612727</td>
      <td>1.065235</td>
      <td>0.489095</td>
      <td>-0.143772</td>
      <td>0.635558</td>
      <td>0.463917</td>
      <td>-0.114805</td>
      <td>-0.183361</td>
      <td>-0.145783</td>
      <td>-0.069083</td>
      <td>-0.225775</td>
      <td>-0.638672</td>
      <td>0.101288</td>
      <td>-0.339846</td>
      <td>0.167170</td>
      <td>0.125895</td>
      <td>-0.008983</td>
      <td>0.014724</td>
      <td>2.69</td>
      <td>0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1.0</td>
      <td>-1.358354</td>
      <td>-1.340163</td>
      <td>1.773209</td>
      <td>0.379780</td>
      <td>-0.503198</td>
      <td>1.800499</td>
      <td>0.791461</td>
      <td>0.247676</td>
      <td>-1.514654</td>
      <td>0.207643</td>
      <td>0.624501</td>
      <td>0.066084</td>
      <td>0.717293</td>
      <td>-0.165946</td>
      <td>2.345865</td>
      <td>-2.890083</td>
      <td>1.109969</td>
      <td>-0.121359</td>
      <td>-2.261857</td>
      <td>0.524980</td>
      <td>0.247998</td>
      <td>0.771679</td>
      <td>0.909412</td>
      <td>-0.689281</td>
      <td>-0.327642</td>
      <td>-0.139097</td>
      <td>-0.055353</td>
      <td>-0.059752</td>
      <td>378.66</td>
      <td>0</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1.0</td>
      <td>-0.966272</td>
      <td>-0.185226</td>
      <td>1.792993</td>
      <td>-0.863291</td>
      <td>-0.010309</td>
      <td>1.247203</td>
      <td>0.237609</td>
      <td>0.377436</td>
      <td>-1.387024</td>
      <td>-0.054952</td>
      <td>-0.226487</td>
      <td>0.178228</td>
      <td>0.507757</td>
      <td>-0.287924</td>
      <td>-0.631418</td>
      <td>-1.059647</td>
      <td>-0.684093</td>
      <td>1.965775</td>
      <td>-1.232622</td>
      <td>-0.208038</td>
      <td>-0.108300</td>
      <td>0.005274</td>
      <td>-0.190321</td>
      <td>-1.175575</td>
      <td>0.647376</td>
      <td>-0.221929</td>
      <td>0.062723</td>
      <td>0.061458</td>
      <td>123.50</td>
      <td>0</td>
    </tr>
    <tr>
      <th>4</th>
      <td>2.0</td>
      <td>-1.158233</td>
      <td>0.877737</td>
      <td>1.548718</td>
      <td>0.403034</td>
      <td>-0.407193</td>
      <td>0.095921</td>
      <td>0.592941</td>
      <td>-0.270533</td>
      <td>0.817739</td>
      <td>0.753074</td>
      <td>-0.822843</td>
      <td>0.538196</td>
      <td>1.345852</td>
      <td>-1.119670</td>
      <td>0.175121</td>
      <td>-0.451449</td>
      <td>-0.237033</td>
      <td>-0.038195</td>
      <td>0.803487</td>
      <td>0.408542</td>
      <td>-0.009431</td>
      <td>0.798278</td>
      <td>-0.137458</td>
      <td>0.141267</td>
      <td>-0.206010</td>
      <td>0.502292</td>
      <td>0.219422</td>
      <td>0.215153</td>
      <td>69.99</td>
      <td>0</td>
    </tr>
  </tbody>
</table>
</div>

```python
# Create X and Y
X = dataset.values[:,0:29]

Y = dataset.values[:,30]

# split data into train and test sets
seed = 42
test_size = 0.33
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=test_size, random_state=seed)
# fit model with training data
model = XGBClassifier(n_jobs=-1)
model.fit(X_train, y_train)
```

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
                  nthread=None, objective='binary:logistic', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1)

```python
# make predictions for test data
y_pred = model.predict(X_test)
predictions = [round(value) for value in y_pred]
# evaluate predictions

from sklearn.metrics import roc_auc_score, accuracy_score

accuracy = accuracy_score(y_test, predictions)
print("Accuracy: %.2f%%" % (accuracy * 100.0))
```

    Accuracy: 99.96%

```python
roc_auc_score(y_test, y_pred)
```

    0.9093480181681094

### Feature importance Plot

```python
plot_importance(model)
```

    <IPython.core.display.Javascript object>

    <matplotlib.axes._subplots.AxesSubplot at 0x1a2b6a42e8>

```python
list(enumerate(dataset.keys()))
```

    [(0, 'Time'),
     (1, 'V1'),
     (2, 'V2'),
     (3, 'V3'),
     (4, 'V4'),
     (5, 'V5'),
     (6, 'V6'),
     (7, 'V7'),
     (8, 'V8'),
     (9, 'V9'),
     (10, 'V10'),
     (11, 'V11'),
     (12, 'V12'),
     (13, 'V13'),
     (14, 'V14'),
     (15, 'V15'),
     (16, 'V16'),
     (17, 'V17'),
     (18, 'V18'),
     (19, 'V19'),
     (20, 'V20'),
     (21, 'V21'),
     (22, 'V22'),
     (23, 'V23'),
     (24, 'V24'),
     (25, 'V25'),
     (26, 'V26'),
     (27, 'V27'),
     (28, 'V28'),
     (29, 'Amount'),
     (30, 'Class')]

### Tree Graph

```python
plot_tree(model)
```

    <IPython.core.display.Javascript object>

    <matplotlib.axes._subplots.AxesSubplot at 0x1a26c3b7f0>

# Preparación de datos para GB

XGBoost solo considera como entrada valores númericos, lo que significa que debemos "codificar" nuestro dataset  mediante:

- Label Encoding (LabelEncoder)
- OneHot Encoding (OneHotEncoder)
- Feature Scaling (StandardScaler

```python
# multiclass classification with Label Encoding
from pandas import read_csv
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import LabelEncoder
from sklearn.datasets import load_iris
# load data
iris = pd.read_csv('iris.csv')
# split data into X and y
X = iris.values[:,0:4]
Y = iris.values[:,4]
# encode string class values as integers
label_encoder = LabelEncoder()
label_encoder = label_encoder.fit(Y)
label_encoded_y = label_encoder.transform(Y)
seed = 7
test_size = 0.33
X_train, X_test, y_train, y_test = train_test_split(X, label_encoded_y, test_size=test_size, random_state=seed)
# fit model on training data
model = XGBClassifier()
model.fit(X_train, y_train)
print(model)
```

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,
                  nthread=None, objective='multi:softprob', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1)

```python
# make predictions for test data
predictions = model.predict(X_test)
# evaluate predictions
accuracy = accuracy_score(y_test, predictions)
print("Accuracy: %.2f%%" % (accuracy * 100.0))
```

    Accuracy: 92.00%

```python
Y
```

    array(['Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',
           'Iris-setosa', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-versicolor', 'Iris-versicolor', 'Iris-versicolor',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica', 'Iris-virginica',
           'Iris-virginica', 'Iris-virginica'], dtype=object)

```python
label_encoded_y
```

    array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])

# Evaluación de modelos

```python
print("Parameter optimization")

import numpy as np

# Mean absolute percentage error
def mean_absolute_percentage_error(estimator, X, y):
    y_pred = estimator.predict(X)
    y_true = y
    return np.mean(np.abs((y_true - y_pred) / y_true))

estimators = [i for i in range(0, 5000, 500)]
gammas = [i/10 for i in range(0, 21, 1)]
max_depths=[6]

scorer = mean_absolute_percentage_error
xgb_grid_params = dict(n_estimators = estimators, gamma = gammas, max_depth=max_depths)
xgb = XGBClassifier()

xgb
```

    Parameter optimization

    XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                  colsample_bynode=1, colsample_bytree=1, gamma=0,
                  learning_rate=0.1, max_delta_step=0, max_depth=3,
                  min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,
                  nthread=None, objective='binary:logistic', random_state=0,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                  silent=None, subsample=1, verbosity=1)

```python
clf = GridSearchCV(xgb,
                   xgb_grid_params,
                   cv = 10,
                   scoring=scorer,
                   n_jobs = -1)

clf
```

    GridSearchCV(cv=10, error_score=nan,
                 estimator=XGBClassifier(base_score=0.5, booster='gbtree',
                                         colsample_bylevel=1, colsample_bynode=1,
                                         colsample_bytree=1, gamma=0,
                                         learning_rate=0.1, max_delta_step=0,
                                         max_depth=3, min_child_weight=1,
                                         missing=None, n_estimators=100, n_jobs=1,
                                         nthread=None, objective='binary:logistic',
                                         random_state=0, reg_alpha=0, reg_lambda=1,
                                         scale_p...
                                         subsample=1, verbosity=1),
                 iid='deprecated', n_jobs=-1,
                 param_grid={'gamma': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,
                                       0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7,
                                       1.8, 1.9, 2.0],
                             'max_depth': [6],
                             'n_estimators': [0, 500, 1000, 1500, 2000, 2500, 3000,
                                              3500, 4000, 4500]},
                 pre_dispatch='2*n_jobs', refit=True, return_train_score=False,
                 scoring=<function mean_absolute_percentage_error at 0x1a426c2d08>,
                 verbose=0)

```python
%time clf.fit(X_train, y_train)
print(clf.best_score_)
print(clf.best_params_)
```

    Parameter optimization
    CPU times: user 11.2 s, sys: 676 ms, total: 11.8 s
    Wall time: 9min 55s
    nan
    {'gamma': 0.0, 'max_depth': 6, 'n_estimators': 0}

# Fin

```python

```


# Text Classification with SpaCy

A common task in NLP is **text classification**. This is "classification" in the conventional machine learning sense, and it is applied to text. Examples include spam detection, sentiment analysis, and tagging customer queries.

In this tutorial, you'll learn text classification with spaCy. The classifier will detect spam messages, a common functionality in most email clients. Here is an overview of the data you'll use:

```python
import pandas as pd

# Loading the spam data
# ham is the label for non-spam messages
spam = pd.read_csv('../input/nlp-course/spam.csv')
spam.head(10)
```

# Bag of Words
Machine learning models don't learn from raw text data. Instead, you need to convert the text to something numeric.

The simplest common representation is a variation of one-hot encoding. You represent each document as a vector of term frequencies for each term in the vocabulary. The vocabulary is built from all the tokens (terms) in the corpus (the collection of documents).

As an example, take the sentences "Tea is life. Tea is love." and "Tea is healthy, calming, and delicious." as our corpus. The vocabulary then is `{"tea", "is", "life", "love", "healthy", "calming", "and", "delicious"}` (ignoring punctuation).

For each document, count up how many times a term occurs, and place that count in the appropriate element of a vector. The first sentence has "tea" twice and that is the first position in our vocabulary, so we put the number 2 in the first element of the vector. Our sentences as vectors then look like

$$
\begin{align}
v_1 &= \left[\begin{matrix} 2 & 2 & 1 & 1 & 0 & 0 & 0 & 0 \end{matrix}\right] \\
v_2 &= \left[\begin{matrix} 1 & 1 & 0 & 0 & 1 & 1 & 1 & 1 \end{matrix}\right]
\end{align}
$$

This is called the **bag of words** representation. You can see that documents with similar terms will have similar vectors. Vocabularies frequently have tens of thousands of terms, so these vectors can be very large.

Another common representation is **TF-IDF (Term Frequency - Inverse Document Frequency)**. TF-IDF is similar to bag of words except that each term count is scaled by the term's frequency in the corpus. Using TF-IDF can potentially improve your models. You won't need it here. Feel free to look it up though!

# Building a Bag of Words model

Once you have your documents in a bag of words representation, you can use those vectors as input to any machine learning model. spaCy handles the bag of words conversion and building a simple linear model for you with the `TextCategorizer` class.

The TextCategorizer is a spaCy **pipe**. Pipes are classes for processing and transforming tokens. When you create a spaCy model with `nlp = spacy.load('en_core_web_sm')`, there are default pipes that perform part of speech tagging, entity recognition, and other transformations. When you run text through a model `doc = nlp("Some text here")`, the output of the pipes are attached to the tokens in the `doc` object. The lemmas for `token.lemma_` come from one of these pipes.

You can remove or add pipes to models. What we'll do here is create an empty model without any pipes (other than a tokenizer, since all models always have a tokenizer). Then, we'll create a TextCategorizer pipe and add it to the empty model.

```python
import spacy

# Create an empty model
nlp = spacy.blank("en")

# Create the TextCategorizer with exclusive classes and "bow" architecture
textcat = nlp.create_pipe(
              "textcat",
              config={
                "exclusive_classes": True,
                "architecture": "bow"})

# Add the TextCategorizer to the empty model
nlp.add_pipe(textcat)
```

Since the classes are either ham or spam, we set `"exclusive_classes"` to `True`. We've also configured it with the bag of words (`"bow"`) architecture. spaCy provides a convolutional neural network architecture as well, but it's more complex than you need for now.

Next we'll add the labels to the model. Here "ham" are for the real messages, "spam" are spam messages.

```python
# Add labels to text classifier
textcat.add_label("ham")
textcat.add_label("spam")
```

# Training a Text Categorizer Model

Next, you'll convert the labels in the data to the form TextCategorizer requires. For each document, you'll create a dictionary of boolean values for each class.

For example, if a text is "ham", we need a dictionary `{'ham': True, 'spam': False}`. The model is looking for these labels inside another dictionary with the key `'cats'`.

```python
train_texts = spam['text'].values
train_labels = [{'cats': {'ham': label == 'ham',
                          'spam': label == 'spam'}}
                for label in spam['label']]
```

Then we combine the texts and labels into a single list.

```python
train_data = list(zip(train_texts, train_labels))
train_data[:3]
```

Now you are ready to train the model. First, create an `optimizer` using `nlp.begin_training()`. spaCy uses this optimizer to update the model. In general it's more efficient to train models in small batches. spaCy provides the `minibatch` function that returns a generator yielding minibatches for training. Finally, the minibatches are split into texts and labels, then used with `nlp.update` to update the model's parameters.

```python
from spacy.util import minibatch

spacy.util.fix_random_seed(1)
optimizer = nlp.begin_training()

# Create the batch generator with batch size = 8
batches = minibatch(train_data, size=8)
# Iterate through minibatches
for batch in batches:
    # Each batch is a list of (text, label) but we need to
    # send separate lists for texts and labels to update().
    # This is a quick way to split a list of tuples into lists
    texts, labels = zip(*batch)
    nlp.update(texts, labels, sgd=optimizer)
```

This is just one training loop (or epoch) through the data. The model will typically need multiple epochs. Use another loop for more epochs, and optionally re-shuffle the training data at the begining of each loop.

```python
import random

random.seed(1)
spacy.util.fix_random_seed(1)
optimizer = nlp.begin_training()

losses = {}
for epoch in range(10):
    random.shuffle(train_data)
    # Create the batch generator with batch size = 8
    batches = minibatch(train_data, size=8)
    # Iterate through minibatches
    for batch in batches:
        # Each batch is a list of (text, label) but we need to
        # send separate lists for texts and labels to update().
        # This is a quick way to split a list of tuples into lists
        texts, labels = zip(*batch)
        nlp.update(texts, labels, sgd=optimizer, losses=losses)
    print(losses)
```

# Making Predictions

Now that you have a trained model, you can make predictions with the `predict()` method. The input text needs to be tokenized with `nlp.tokenizer`. Then you pass the tokens to the predict method which returns scores. The scores are the probability the input text belongs to the classes.

```python
texts = ["Are you ready for the tea party????? It's gonna be wild",
         "URGENT Reply to this message for GUARANTEED FREE TEA" ]
docs = [nlp.tokenizer(text) for text in texts]

# Use textcat to get the scores for each doc
textcat = nlp.get_pipe('textcat')
scores, _ = textcat.predict(docs)

print(scores)
```

The scores are used to predict a single class or label by choosing the label with the highest probability. You get the index of the highest probability with `scores.argmax`, then use the index to get the label string from `textcat.labels`.

```python
# From the scores, find the label with the highest score/probability
predicted_labels = scores.argmax(axis=1)
print([textcat.labels[label] for label in predicted_labels])
```

Evaluating the model is straightforward once you have the predictions. To measure the accuracy, calculate how many correct predictions are made on some test data, divided by the total number of predictions.

# Your Turn
Try it yourself as you **[predict the sentiment of Yelp reviews](#$NEXT_NOTEBOOK_URL$)**.