```
%load_ext autoreload
%autoreload 2

%matplotlib inline
```

```
#export
from exp.nb_06 import *
```

## ConvNet

```
x_train,y_train,x_valid,y_valid = get_data()

x_train,x_valid = normalize_to(x_train,x_valid)
train_ds,valid_ds = Dataset(x_train, y_train),Dataset(x_valid, y_valid)

nh,bs = 50,512
c = y_train.max().item()+1
loss_func = F.cross_entropy

data = DataBunch(*get_dls(train_ds, valid_ds, bs), c)
```

```
mnist_view = view_tfm(1,28,28)
cbfs = [Recorder,
        partial(AvgStatsCallback,accuracy),
        CudaCallback,
        partial(BatchTransformXCallback, mnist_view)]
```

```
nfs = [8,16,32,32]
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_layer, cbs=cbfs)
```

```
%time run.fit(1, learn)
```

    train: [1.63907625, tensor(0.4483, device='cuda:0')]
    valid: [0.56672900390625, tensor(0.8311, device='cuda:0')]
    CPU times: user 2.33 s, sys: 888 ms, total: 3.21 s
    Wall time: 3.21 s

## Batchnorm

### Custom

```
class BatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        # NB: pytorch bn mom is opposite of what you'd expect
        self.mom,self.eps = mom,eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('vars',  torch.zeros(1,nf,1,1))
        self.register_buffer('means', torch.zeros(1,nf,1,1))
        self.steps = 0

    def update_stats(self, x):
        self.steps += 1
        m = x.mean((0,2,3), keepdim=True)
        v = x.var ((0,2,3), keepdim=True)
        self.means.detach_()
        self.vars.detach_()
        mom = 1. if self.steps==1 else self.mom
        self.means.lerp_(m, mom)
        self.vars.lerp_ (v, mom)

    def forward(self, x):
        if self.training: self.update_stats(x)
        x = (x-self.means).div_((self.vars+self.eps).sqrt())
        return x.mul_(self.mults).add_(self.adds)
```

```
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    # No bias needed if using bn
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(BatchNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.9, conv_layer, cbs=cbfs)
```

```
with Hooks(learn.model, append_stats) as hooks:
    run.fit(1, learn)
    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks:
        ms,ss = h.stats
        ax0.plot(ms[:10])
        ax1.plot(ss[:10])
        h.remove()
    plt.legend(range(6));

    fig,(ax0,ax1) = plt.subplots(1,2, figsize=(10,4))
    for h in hooks:
        ms,ss = h.stats
        ax0.plot(ms)
        ax1.plot(ss)
```

```
learn,run = get_learn_run(nfs, data, 1.0, conv_layer, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.4624445703125, tensor(0.8601, device='cuda:0')]
    valid: [0.1636791015625, tensor(0.9560, device='cuda:0')]
    train: [0.147957431640625, tensor(0.9565, device='cuda:0')]
    valid: [0.11624248046875, tensor(0.9665, device='cuda:0')]
    train: [0.111165283203125, tensor(0.9668, device='cuda:0')]
    valid: [0.092507275390625, tensor(0.9730, device='cuda:0')]
    CPU times: user 3.52 s, sys: 476 ms, total: 4 s
    Wall time: 4 s

### Bias corrected

See XL file *mom_bias.xlsx*.

```
class BatchNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        # NB: pytorch bn mom is opposite of what you'd expect
        self.mom,self.eps = mom,eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('vars',  torch.zeros(1,nf,1,1))
        self.register_buffer('means', torch.zeros(1,nf,1,1))
        self.steps = 0

    def update_stats(self, x):
        self.steps += 1
        m = x.mean((0,2,3), keepdim=True)
        v = x.var ((0,2,3), keepdim=True)
        self.means.detach_()
        self.vars.detach_()
        mom = self.mom
        self.means.lerp_(m, mom)
        self.vars.lerp_ (v, mom)

    def forward(self, x):
        if self.training: self.update_stats(x)
        means = self.means
        vars  = self.vars
        if self.steps<100:
            corr = 1-(1-self.mom)**self.steps
            means = means/corr
            vars  = vars /corr
        x = (x-means).div_((vars+self.eps).sqrt())
        return x.mul_(self.mults).add_(self.adds)
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_layer, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.416889140625, tensor(0.8799, device='cuda:0')]
    valid: [0.608126513671875, tensor(0.8132, device='cuda:0')]
    train: [0.152734150390625, tensor(0.9553, device='cuda:0')]
    valid: [0.110173779296875, tensor(0.9694, device='cuda:0')]
    train: [0.1070734375, tensor(0.9681, device='cuda:0')]
    valid: [0.1031563232421875, tensor(0.9700, device='cuda:0')]
    CPU times: user 3.55 s, sys: 445 ms, total: 3.99 s
    Wall time: 4 s

### jit

```
#export
from torch.jit import ScriptModule, script_method, script
from typing import *

class BatchNormJit(ScriptModule):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        self.mults = nn.Parameter(torch.ones (1,nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(1,nf,1,1))
        self.register_buffer('vars',  torch.zeros(1,nf,1,1))
        self.register_buffer('means', torch.zeros(1,nf,1,1))
        self.register_buffer('steps', tensor(0))

    @script_method
    def update_stats(self, x):
        self.steps += 1
        mom,eps = 0.1,1e-5
        m, invstd = torch.batch_norm_stats(x, eps)
        views = (1,-1,1,1)
        m = m.view(*views)
        v = (invstd**-2).view(*views)
        if bool(self.steps>0):
            m = self.means*(1-mom) + m*mom
            v = self.vars *(1-mom) + v*mom
        self.means.copy_(m.detach())
        self.vars .copy_(v.detach())

    @script_method
    def forward(self, x):
        if self.training: self.update_stats(x)
        # squeeze.unsqueeze is to create a barrier to trick the fuser
        factor = (self.mults/self.vars.sqrt()).squeeze(-1).unsqueeze(-1)
        offset = (self.adds-self.means).squeeze(-1).unsqueeze(-1)
        return factor * (x+offset)
```

```
def conv_jit(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(BatchNormJit(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_jit, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [nan, tensor(0.0984, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    train: [nan, tensor(0.0986, device='cuda:0')]
    valid: [nan, tensor(0.0991, device='cuda:0')]
    CPU times: user 4.47 s, sys: 452 ms, total: 4.92 s
    Wall time: 4.92 s

### Builtin batchnorm

```
#export
def conv_layer(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(nn.BatchNorm2d(nf, eps=1e-5))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_layer, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.4807440234375, tensor(0.8586, device='cuda:0')]
    valid: [0.12600108642578126, tensor(0.9641, device='cuda:0')]
    train: [0.104688974609375, tensor(0.9686, device='cuda:0')]
    valid: [0.08653324584960938, tensor(0.9739, device='cuda:0')]
    train: [0.072819384765625, tensor(0.9781, device='cuda:0')]
    valid: [0.1579060791015625, tensor(0.9513, device='cuda:0')]
    CPU times: user 2.7 s, sys: 278 ms, total: 2.98 s
    Wall time: 2.98 s

### With scheduler

```
sched = combine_scheds([0.5, 0.5], [sched_lin(0.2, 1.), sched_lin(1., 0.1)])
```

```
learn,run = get_learn_run(nfs, data, 0.9, conv_layer, cbs=cbfs
                          +[partial(ParamScheduler,'lr', sched)])
```

```
run.fit(8, learn)
```

    train: [1.04039875, tensor(0.7148, device='cuda:0')]
    valid: [0.240669921875, tensor(0.9373, device='cuda:0')]
    train: [0.1803751171875, tensor(0.9492, device='cuda:0')]
    valid: [0.1454875, tensor(0.9592, device='cuda:0')]
    train: [0.104181181640625, tensor(0.9698, device='cuda:0')]
    valid: [0.08910442504882812, tensor(0.9743, device='cuda:0')]
    train: [0.0783616455078125, tensor(0.9760, device='cuda:0')]
    valid: [0.13091724853515624, tensor(0.9612, device='cuda:0')]
    train: [0.0602369091796875, tensor(0.9817, device='cuda:0')]
    valid: [0.0673140869140625, tensor(0.9793, device='cuda:0')]
    train: [0.0450689013671875, tensor(0.9869, device='cuda:0')]
    valid: [0.05500914306640625, tensor(0.9841, device='cuda:0')]
    train: [0.0345672021484375, tensor(0.9903, device='cuda:0')]
    valid: [0.05089189147949219, tensor(0.9841, device='cuda:0')]
    train: [0.02767552734375, tensor(0.9925, device='cuda:0')]
    valid: [0.04703995971679688, tensor(0.9849, device='cuda:0')]

## More norms

### Layer norm

From [the paper](https://arxiv.org/abs/1607.06450): "*batch normalization cannot be applied to online learning tasks or to extremely large distributed models where the minibatches have to be small*".

General equation for a norm layer with learnable affine:

$$y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta$$

```
class LayerNorm(ScriptModule):
    __constants__ = ['eps']
    def __init__(self, eps=1e-5):
        super().__init__()
        self.eps = eps
        self.mult = nn.Parameter(tensor(1.))
        self.add  = nn.Parameter(tensor(0.))

    @script_method
    def forward(self, x):
        m = x.mean((1,2,3), keepdim=True)
        v = x.std ((1,2,3), keepdim=True)
        x = (x-m) / ((v+self.eps).sqrt())
        return x*self.mult + self.add
```

```
def conv_ln(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(LayerNorm())
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.8, conv_ln, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.484382421875, tensor(0.8470, device='cuda:0')]
    valid: [0.1195010498046875, tensor(0.9664, device='cuda:0')]
    train: [0.11501509765625, tensor(0.9643, device='cuda:0')]
    valid: [0.0873831298828125, tensor(0.9735, device='cuda:0')]
    train: [0.069044267578125, tensor(0.9777, device='cuda:0')]
    valid: [0.07121566162109375, tensor(0.9785, device='cuda:0')]
    CPU times: user 4.19 s, sys: 511 ms, total: 4.7 s
    Wall time: 4.7 s

*Thought experiment*: can this distinguish green frogs from red frogs (assuming you're using it before the first conv)?

### Instance norm

From [the paper](https://arxiv.org/abs/1607.08022):

The key difference between **contrast** and batch normalization is that the latter applies the normalization to a  whole batch of images instead for single ones:

\begin{equation}\label{eq:bnorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{i}}{\sqrt{\sigma_i^2 + \epsilon}},
    \quad
    \mu_i = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_i^2 = \frac{1}{HWT}\sum_{t=1}^T\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_i)^2.
\end{equation}

In order to combine the effects of instance-specific normalization and batch normalization, we propose to replace the latter by the *instance normalization* (also known as *contrast normalization*) layer:

\begin{equation}\label{eq:inorm}
    y_{tijk} =  \frac{x_{tijk} - \mu_{ti}}{\sqrt{\sigma_{ti}^2 + \epsilon}},
    \quad
    \mu_{ti} = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H x_{tilm},
    \quad
    \sigma_{ti}^2 = \frac{1}{HW}\sum_{l=1}^W \sum_{m=1}^H (x_{tilm} - mu_{ti})^2.
\end{equation}

```
class InstanceNorm(nn.Module):
    __constants__ = ['eps']
    def __init__(self, nf, eps=1e-0):
        super().__init__()
        self.eps = eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds  = nn.Parameter(torch.zeros(nf,1,1))

    def forward(self, x):
        m = x.mean((2,3), keepdim=True)
        v = x.std ((2,3), keepdim=True)
        res = (x-m) / ((v+self.eps).sqrt())
        return res*self.mults + self.adds
```

```
def conv_in(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(InstanceNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.1, conv_in, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [2.30295890625, tensor(0.1075, device='cuda:0')]
    valid: [2.3020640625, tensor(0.1064, device='cuda:0')]
    train: [2.30127734375, tensor(0.1136, device='cuda:0')]
    valid: [2.301862890625, tensor(0.1064, device='cuda:0')]
    train: [2.30114171875, tensor(0.1136, device='cuda:0')]
    valid: [2.3020126953125, tensor(0.1064, device='cuda:0')]
    CPU times: user 3.64 s, sys: 507 ms, total: 4.15 s
    Wall time: 4.15 s

*Question*: why can't this classify anything?

### Group norm

*From the PyTorch docs:*

`GroupNorm(num_groups, num_channels, eps=1e-5, affine=True)`

The input channels are separated into `num_groups` groups, each containing
``num_channels / num_groups`` channels. The mean and standard-deviation are calculated
separately over the each group. $\gamma$ and $\beta$ are learnable
per-channel affine transform parameter vectorss of size `num_channels` if
`affine` is ``True``.

This layer uses statistics computed from input data in both training and
evaluation modes.

Args:
-    num_groups (int): number of groups to separate the channels into
-    num_channels (int): number of channels expected in input
-    eps: a value added to the denominator for numerical stability. Default: 1e-5
-    affine: a boolean value that when set to ``True``, this module
        has learnable per-channel affine parameters initialized to ones (for weights)
        and zeros (for biases). Default: ``True``.

Shape:
- Input: :math:`(N, num\_channels, *)`
- Output: :math:`(N, num\_channels, *)` (same shape as input)

Examples::

    >>> input = torch.randn(20, 6, 10, 10)
    >>> # Separate 6 channels into 3 groups
    >>> m = nn.GroupNorm(3, 6)
    >>> # Separate 6 channels into 6 groups (equivalent with InstanceNorm)
    >>> m = nn.GroupNorm(6, 6)
    >>> # Put all 6 channels into a single group (equivalent with LayerNorm)
    >>> m = nn.GroupNorm(1, 6)
    >>> # Activating the module
    >>> output = m(input)

## Fix small batch sizes

### Exponentially Weighted Layer Norm

```
class EWLayerNorm(nn.Module):
    def __init__(self, nf, mom=0.1, eps=1e-5):
        super().__init__()
        self.mom,self.eps = mom,eps
        self.mult = nn.Parameter(tensor(1.))
        self.add  = nn.Parameter(tensor(0.))
        self.register_buffer('batch', tensor(0.))
        self.register_buffer('sum'  , tensor(0.))
        self.register_buffer('sqr'  , tensor(0.))
        self.register_buffer('count', tensor(0.))

    def update_stats(self, x):
        bs,nc,*_ = x.shape
        self.sum.detach_()
        self.sqr.detach_()
        mom = 1 if self.batch==0 else self.mom
        self.sum  .add_(mom, x.sum()     - self.sum  )
        self.sqr  .add_(mom, (x*x).sum() - self.sqr  )
        self.count.add_(mom, x.numel()   - self.count)
        self.batch += bs

    def forward(self, x):
        if self.training: self.update_stats(x)
        mean = self.sum/self.count
        var  = self.sqr/self.count - mean.pow(2)
        x = (x-mean)/((var+self.eps).sqrt())
        return x*self.mult + self.add
```

```
def conv_ewln(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=True),
              GeneralRelu(**kwargs)]
    if bn: layers.append(EWLayerNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 0.6, conv_ewln, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.4745987109375, tensor(0.8501, device='cuda:0')]
    valid: [0.15462315673828125, tensor(0.9546, device='cuda:0')]
    train: [0.1423449609375, tensor(0.9565, device='cuda:0')]
    valid: [0.11461904296875, tensor(0.9652, device='cuda:0')]
    train: [0.09990123046875, tensor(0.9691, device='cuda:0')]
    valid: [0.083691064453125, tensor(0.9747, device='cuda:0')]
    CPU times: user 4.38 s, sys: 591 ms, total: 4.97 s
    Wall time: 4.98 s

### Exponentially Weighted Batch Norm

```
class EWBatchNorm(nn.Module):
    def __init__(self, nf, mom=0.01, eps=1e-5):
        super().__init__()
        self.mom,self.eps = mom,eps
        self.mults = nn.Parameter(torch.ones (nf,1,1))
        self.adds = nn.Parameter(torch.zeros(nf,1,1))
        self.register_buffer('sums', torch.zeros(1,nf,1,1))
        self.register_buffer('sqrs', torch.zeros(1,nf,1,1))
        self.register_buffer('batch', tensor(0.))
        self.register_buffer('count', tensor(0.))
        self.register_buffer('step', tensor(0.))
        self.register_buffer('dbias', tensor(0.))

    def update_stats(self, x):
        bs,nc,*_ = x.shape
        self.sums.detach_()
        self.sqrs.detach_()
        dims = (0,2,3)
        s = x.sum(dims, keepdim=True)
        ss = (x*x).sum(dims, keepdim=True)
        c = self.count.new_tensor(x.numel()/nc)
        self.mom1 = self.dbias.new_tensor(1 - (1-self.mom)**x.shape[0])
        self.sums.lerp_(s, self.mom1)
        self.sqrs.lerp_(ss, self.mom1)
        self.count.lerp_(c, self.mom1)
        self.dbias.lerp_(self.mom1, self.mom1)
        self.batch += bs
        self.step += 1

    def forward(self, x):
        if self.training: self.update_stats(x)
        sums = self.sums
        sqrs = self.sqrs
        c = self.count
        if self.step<100:
            sums = sums / self.dbias
            sqrs = sqrs / self.dbias
            c    = c    / self.dbias
        means = sums/c
        vars = (sqrs/c).sub_(means*means)
        if bool(self.batch < 20): vars.clamp_min_(0.01)
        x = (x-means).div_((vars.add_(self.eps)).sqrt())
        return x.mul_(self.mults).add_(self.adds)
```

```
def conv_ewbn(ni, nf, ks=3, stride=2, bn=True, **kwargs):
    layers = [nn.Conv2d(ni, nf, ks, padding=ks//2, stride=stride, bias=not bn),
              GeneralRelu(**kwargs)]
    if bn: layers.append(EWBatchNorm(nf))
    return nn.Sequential(*layers)
```

```
learn,run = get_learn_run(nfs, data, 4, conv_ewbn, cbs=cbfs)
```

```
%time run.fit(3, learn)
```

    train: [0.3655568359375, tensor(0.8842, device='cuda:0')]
    valid: [0.1037153564453125, tensor(0.9706, device='cuda:0')]
    train: [0.0777454052734375, tensor(0.9763, device='cuda:0')]
    valid: [0.0712057861328125, tensor(0.9795, device='cuda:0')]
    train: [0.0537394970703125, tensor(0.9834, device='cuda:0')]
    valid: [0.06340899658203125, tensor(0.9814, device='cuda:0')]
    CPU times: user 4.24 s, sys: 537 ms, total: 4.78 s
    Wall time: 4.78 s

```
learn,run = get_learn_run(nfs, data, 0.9, conv_ewbn, cbs=cbfs
                          +[partial(ParamScheduler,'lr', sched)])
```

```
run.fit(8, learn)
```

    train: [0.623816875, tensor(0.8351, device='cuda:0')]
    valid: [0.2730313232421875, tensor(0.9192, device='cuda:0')]
    train: [0.13766603515625, tensor(0.9612, device='cuda:0')]
    valid: [0.14829573974609375, tensor(0.9548, device='cuda:0')]
    train: [0.086273017578125, tensor(0.9742, device='cuda:0')]
    valid: [0.09565546875, tensor(0.9729, device='cuda:0')]
    train: [0.06433189453125, tensor(0.9801, device='cuda:0')]
    valid: [0.09912955322265625, tensor(0.9703, device='cuda:0')]
    train: [0.046377255859375, tensor(0.9859, device='cuda:0')]
    valid: [0.05765380859375, tensor(0.9827, device='cuda:0')]
    train: [0.03398404052734375, tensor(0.9900, device='cuda:0')]
    valid: [0.05323286743164062, tensor(0.9842, device='cuda:0')]
    train: [0.025488779296875, tensor(0.9936, device='cuda:0')]
    valid: [0.048959304809570316, tensor(0.9853, device='cuda:0')]
    train: [0.02033477294921875, tensor(0.9955, device='cuda:0')]
    valid: [0.04754931945800781, tensor(0.9858, device='cuda:0')]

## Export

```
!python notebook2script.py 07_batchnorm.ipynb
```

    Converted 07_batchnorm.ipynb to nb_07.py

```

```


# NeuroDyn Python model

## Overview

This guide introduces the Python software environment for setting up and simulating a model of the NeuroDyn hardware equations. By the end of the guide you should get an understanding of the dynamics of a NeuroDyn neuron, be able to construct a NeuroDyn object, as well as understand how to fit the parameters of the model in order to emulate the biophysical conductance-based models.

## Setting up the environment

We first need to import the modules with the appropriate model definitions, as well as the Python packages that we will need:

- `matplotlib.pyplot`: Python plotting module
- `numpy`: Python library for numerical operations
- `cb_models`: model definitions for both standard conductance-based models as well as NeuroDyn

```python
import matplotlib.pyplot as plt
from cb_models import HHModel, NeuroDynModel

# **Ignore overflow warnings**
import numpy as np
np.seterr(all="ignore")

%matplotlib inline
%load_ext autoreload
%autoreload 2
```

## Hodgkin-Huxley model

We start with a short overview of the standard Hodgkin-Huxley model of squid giant axon and how we can define it using the environment.

The Hodgkin-Huxley model of the neural membrane dynamics is given by the following set of equations:

\begin{align}
C \dot{V} &= - \bar{g}_l (V - E_l) - \bar{g}_{Na} m^3 h (V - E_{Na}) - \bar{g}_K n^4 (V - E_K) + I_{app} \\
\dot{m} &= \alpha_m (V) (1 - m) - \beta_m (V) m \\
\dot{h} &= \alpha_h (V) (1 - h) - \beta_h (V) h \\
\dot{n} &= \alpha_n (V) (1 - n) - \beta_n (V) n
\end{align}

Let's start by defining a Hodgkin-Huxley model of a neuron that implements these equations - a NeuroDyn model will be done in very much the same way in the following sections.

```python
hh_neuron = HHModel()
```

By passing no arguments to the `HHModel` class, an instance of the neuron is created with default parameters taken from the original Hodgkin-Huxley equations. Note that the values of the constants are shifted so that the resting potential is at $0 mV$. If we instead wanted to initialize a model with, for example, different sodium current parameters, we could do that as:

`hh_neuron = HHModel(gna = 100, Ena = 50)`

which would set $g_{Na} = 100 mS$ and $E_{Na} = 50 mV$.

Let's try simulating the model for which we only need to define the $I_{app}(t)$ function for the applied current. We can try different values for the magnitude of the constant current $I_0$ to showcase both resting and spiking behaviors.

```python
def Iapp(t):
    I0 = 10 # applied current in uA
    return I0

T = 100 # length of the simulation in ms
trange = (0, T)

x0 = [0,0,0,0] # initial conditions [V0, m0, h0, n0]

sol = hh_neuron.simulate(trange, x0, Iapp)

plt.figure()
plt.xlabel("Time [$ms$]")
plt.ylabel("Voltage [$mV$]")
plt.title("Membrane voltage")
plt.plot(sol.t, sol.y[0])
plt.show()
```

### Additional options

##### Converting units to SI

There is a few things to note here. For convenience, all of the parameters and variables are in non-SI units, so that the voltages are expressed in $mV$, currents in $\mu A$, time in $m s $, etc. If we instead wanted to convert the model to SI units, we can initialize the neuron as:

`hh_neuron = HHModel(SI_units = True)`

which will convert all parameters of the model to SI units.

##### Voltage and time scaling

A few other options that will be useful once we start defining NeuroDyn models are the parameters for scaling the time and the voltage scales of the model, `scl_t` and `scl_v`. Let's see how this works by instead defining a neuron as:

```python
hh_neuron_scaled = HHModel(scl_v = 3, scl_t = 2)

def Iapp(t):
    I0 = 10 # applied current in uA
    return I0

T = 100 # length of the simulation in ms
trange = (0, T)

x0 = [0,0,0,0] # initial conditions [V0, m0, h0, n0]

sol = hh_neuron_scaled.simulate(trange, x0, Iapp)

plt.figure()
plt.xlabel("Time [$ms$]")
plt.ylabel("Voltage [$mV$]")
plt.title("Membrane voltage")
plt.plot(sol.t, sol.y[0])
plt.show()
```

We can try changing the values for `scl_t` and `scl_v` and see how it changes the behavior compared to the non-scaled simulation we have previously produced.

When we try fitting the conductance-based models to the NeuroDyn hardware we will see that it will be necessary to *scale* both voltage and time of the original model in order to fit them to the hardware constraints. This will change the timescale and the voltage range in which the NeuroDyn neuron operates, but importantly, will not change the underlying dynamical properties of the model.

## NeuroDyn model

### Initializing a NeuroDyn model

Creating a NeuroDyn object is done in exactly the same way as creating a Hodgkin-Huxley object by using the class `NeuroDynModel`. We can therefore create a NeuroDyn neuron using default parameters by calling:

`nd_neuron = NeuroDynModel()`

Note that NeuroDyn uses SI units for all of its parameters and variables, so that the voltage output is in $V$, time is in $s$, etc.

Before we create a NeuroDyn neuron and use it to simulate spiking behavior, let's first look at the NeuroDyn equations and how they connect to the Hodgkin-Huxley equations. The details of the section may be skipped and readers might go directly to the **Summary** that lists all of the controllable parameters and the mappings to the corresponding Hodgkin-Huxley parameters.

### NeuroDyn equations

Let's now take a look at the full list of parameters that are required to define a NeuroDyn model. NeuroDyn circuit replicates the equations of Hodgkin-Huxley in hardware, and we will now see how the different parameters map to the biophysical constants of the conductance-based models.

#### Membrane equation

Let's start with the membrane dynamics equation:

\begin{align}
C_{ND} \dot{V} &= -I_l - I_{Na} - I_K + I_{app}\\
 &= - 2 I_{g_l} \tanh \bigg( \frac{\kappa_{lin} (V - E_l)}{2 V_T} \bigg) - 2 I_{g_{Na}} \tanh \bigg( \frac{\kappa_{lin} (V - E_{Na})}{2 V_T} \bigg) - 2 I_{g_{K}} \tanh \bigg( \frac{\kappa_{lin} (V - E_K)}{2 V_T} \bigg)
\end{align}

Note that capacitance of NeuroDyn $C_{ND} = 4 pF$ is significantly smaller than the capacitance of Hodgkin-Huxley $C_{HH} = 1 \mu F$ which we will need to take into consideration when setting the other parameters of the circuit.

The nonlinear $\tanh$ relationship comes from the transconductance amplifier characteristic that implements the individual conductance relatinships. These amplifiers have additional circuitry to linearize this relationship which makes the constant $\kappa_{lin}$ small. In the simulation, we set $\kappa_{lin} = 0.2$. The constant $V_T$ is the temperature-dependent thermal voltage and is assumed to be $V_T \approx 26 mV$.

##### Maximal conductances

Assuming the conductance circuit is in the linear range so that $\tanh{x} \approx x$, we can define the relationship between the circuit parameters and the original Hodgkin-Huxley equation by comparing the two directly:

\begin{equation}
g_l = \frac{\kappa_{lin}}{V_T} I_{g_l}, \quad
g_{Na} = \frac{\kappa_{lin}}{V_T} I_{g_{Na}}, \quad
g_{K} = \frac{\kappa_{lin}}{V_T} I_{g_{K}}
\end{equation}

It is clear from these equations that the conductances $g_l$, $g_{Na}$ and $g_K$ are effectively represented by three currents in the circuit $I_{g_l}$, $I_{g_{Na}}$ and $I_{g_K}$. The current $I_{g_l}$ is constant, reflecting the static nature of the leak current, while currents $I_{g_{Na}}$ and $I_{g_K}$ have the dynamics replicating the ion channel opening and closing processes.

##### Reversal potentials

The reversal potentials $E_l$, $E_{Na}$ and $E_K$ are defined using digital values using the following equation:

\begin{equation}
E_{rev} = I_{voltage} \cdot \frac{dE_{rev}}{1024} \cdot sign \cdot R_{rev} + V_{ref}
\end{equation}

The current $I_{voltage}$ is controllable and is shared by all the variables in the circuit. Similarly, $V_{ref}$ is controllable and is shared by all the circuits - it effectively shifts all of the parameters in the voltage range. The resistor $R_{rev} = 1.63 M\Omega$ is fixed.

#### Ion currents

Let's now look at the dynamics for the currents $I_{g_{Na}}$ and $I_{g_K}$:

\begin{align}
I_{g_{Na}} &= I_{\bar{g}_{Na}} \bigg(\frac{I_m}{I_{ref}} \bigg)^3 \bigg(\frac{I_h}{I_{ref}} \bigg)\\
I_{g_K} &= I_{\bar{g}_{K}} \bigg(\frac{I_n}{I_{ref}} \bigg)^4
\end{align}

Comparing to the equations of Hodgkin-Huxley, it is clear that the relationship between gating variables and currents is simply:

\begin{equation}
m = \frac{I_m}{I_{ref}}, \quad
h = \frac{I_h}{I_{ref}}, \quad
n = \frac{I_n}{I_{ref}}
\end{equation}

The currents determining the maximal conductance parameters are set by digital values $dg$ that set the currents between $0$ and the current $I_{master}$:

\begin{equation}
I_{\bar{g}} = I_{master} \frac{dg}{1024}
\end{equation}

#### Gating variables

Let's now take a look at the gating variable equations. The dynamical equations for the gating currents are given by:

\begin{align}
\frac{d}{dt} \bigg( \frac{I_m}{I_{ref}} \bigg) &= \frac{I_{\alpha_m}}{C_g V_T} \bigg(1 - \frac{I_m}{I_{ref}} \bigg) - \frac{I_{\beta_m}}{C_g V_T} \bigg( \frac{I_m}{I_{ref}} \bigg) \\
\frac{d}{dt} \bigg( \frac{I_h}{I_{ref}} \bigg) &= \frac{I_{\alpha_h}}{C_g V_T} \bigg(1 - \frac{I_h}{I_{ref}} \bigg) - \frac{I_{\beta_h}}{C_g V_T} \bigg( \frac{I_h}{I_{ref}} \bigg) \\
\frac{d}{dt} \bigg( \frac{I_n}{I_{ref}} \bigg) &= \frac{I_{\alpha_n}}{C_g V_T} \bigg(1 - \frac{I_n}{I_{ref}} \bigg) - \frac{I_{\beta_n}}{C_g V_T} \bigg( \frac{I_n}{I_{ref}} \bigg)
\end{align}

Comparing again to the original Hodgkin-Huxley equations, we see that a simple substitution relates $I_{\alpha}$ and $I_{\beta}$ with $\alpha (V)$ and $\beta (V)$ for each gating variable $m$, $h$ and $n$:

\begin{equation}
\alpha (V) = \frac{I_{\alpha} (V)}{C_g V_T}, \quad
\beta(V) = \frac{I_{\beta} (V)}{C_g V_T}
\end{equation}

The constant $V_T$ is the same thermal voltage defined previously, while the capacitance used in the gating variable equations $C_g = 5 p F$ is different from the membrane capacitance $C_{ND}$.

So far, we have seen that there is a one-to-one correspondence between the physical parameters of NeuroDyn and the Hodgkin-Huxley parameters. Let's now see how we can shape the $\alpha(V)$ and $\beta(V)$ functions in NeuroDyn by designing the corresponding currents $I_{\alpha}$ and $I_{\beta}$.

#### Designing $\alpha (V)$ and $\beta (V)$ functions

Each current $I_{\alpha}$ and $I_{\beta}$ is built as a summation of **seven** sigmoidal IV currents, so that by changing the weights of these seven sigmoids we are able to fit the parameters to different $\alpha (V)$ and $\beta (V)$ functions. Therefore, the currents $I_{\alpha}$ and $I_{\beta}$ for each gating variable are represented by:

\begin{align}
I_{\alpha} &= \sum_{j=1}^7 \frac{I_{\alpha, j}}{1 + \exp{\big( sign \cdot \kappa \cdot (V_{b,j} - V)} \big)} \\
I_{\beta} &= \sum_{j=1}^7 \frac{I_{\beta, j}}{1 + \exp{\big( sign \cdot \kappa \cdot (V_{b,j} - V)} \big)}
\end{align}

The process-dependent constant $\kappa \approx 0.7$ is constant throughout the circuit. Note that this is different from $\kappa_{lin}$ that was encountered in the membrane equation, where the constant is decreased in order to increase the linear range of the $\tanh$ nonlinearity.

Importantly, note that all sigmoids in the summation have a $sign$ parameter that is either $+1$ or $-1$ . This is because $\alpha(V)$ and $\beta(V)$ are either monotonically increasing or decreasing functions - a monotonically increasing function will be built as a summation of monotonically increasing sigmoids, and vice-versa, a monotonically decreasing function will be built as a summation of monotonically decreasing sigmoids. So for example, the gating variable $m$ is an activating variable, so its $\alpha(V)$ function is monotonically increasing, while the gating variable $h$ is an inactivating variable, and so its $\alpha(V)$ function is instead monotonically decreasing.

The sigmoid placements $V_{b,j}$ are equally spaced and shared by all currents in the NeuroDyn chip, and can be controlled by the current $I_{voltage}$.

Each of the sigmoid weights $I_{\alpha, j}$ and $I_{\beta, j}$ are set by digital values in the same way as we have defined previously for maximal conductance currents:

\begin{align}
I_{\alpha, j} &= I_{master} \frac{dI_{\alpha,j}}{1024} \\
I_{\beta, j} &= I_{master} \frac{dI_{\beta,j}}{1024}
\end{align}

### Summary of the parameters

The parameters that can be passed to the NeuroDyn class are the following:

- `dg = [dgna, dgk, dgl]`: digital values for maximal conductances $\bar{g}_{Na}$, $\bar{g}_{K}$ and $\bar{g}_l$
- `dErev = [dEna, dEk, dEl]`: digital values for reversal potentials $E_{Na}$, $E_K$ and $E_l$
- `dIb = [dIb_m, dIb_h, dIb_n]`: digital values for the weights of the spline functions
    - each `dIb_m` `dIb_h` and `dIb_n` is an array consisting of `[dIb_alpha, dIb_beta]`
    - each `dIb_alpha` and `dIb_beta` is an array of 7 digital values for the spline coefficients
- `V_ref`: analog reference voltage (shifts all voltage values to be around $V_{ref}$)
- `I_voltage`: analog current determining the range of reversal potentials, as well as spline positions
- `I_master`: analog current determining the range of maximal conductances, as well as spline weights
- `I_ref`: reference current for gating variables
    - **Note**: the current has no effect on the model equations, but is a controllable parameter in the chip
- `capacitance_scaling`: additional scaling factor that multiplies the membrane capacitance of the model
    - **Note**: by default, the scaling factor is $1.0$, and the membrane capacitance is set to $4 pF$
    - Setting `capacitance_scaling = 0.1` for instance would set the membrane capacitance to $0.4 pF$

Note that the currents `I_voltage`, `I_master` and `I_ref` are directly passed in the simulation, while in the chip we control these currents with corresponding voltage values. All digital values are integers between $0$ and $1023$.

### Fitting the equations

Let's now see how we can fit the parameters of the NeuroDyn model so that it replicates the behavior of a conductance-based model of a neuron. For this we will use the `fitting_utilities` module.

```python
from fitting_utilities import FitND
```

The class `FitND` takes a single `HHModel` object at the initialization, defining the conductance-based neuron we would like to fit to.

We can also pass additional arguments that are otherwise set to their default values:
- `I_master`: the $I_{master}$ current that we have set for the NeuroDyn model. Default value is $150 nA$.
- `I_voltage`: the $I_{voltage}$ current that we have set for the NeuroDyn model. Default value is $200 nA$.
- `capacitance_scaling`: capacitance scaling for the NeuroDyn model we will use. Default values is $1$.
- `vrange`: numpy array of voltage points at which we evaluate the fitting. Default is `np.arange(Ek, Ena/2, 5e-4).T`.

Let's first try to pass a regular Hodgkin-Huxley neuron to the fitting class:

```python
hh_neuron = HHModel(SI_units = True)

fit_obj = FitND(hh_neuron)
```

Note that we have set `SI_units = True` - this is important as all of the NeuroDyn values are defined in SI units, so we would like to first convert our original model to SI units before we continue fitting.

If we now call the method `fitHH` of the class, the code will conduct a non-negative least squares fit in order to find the right sigmoid weights $I_{b,j}$ to fit the gating variables of the Hodgkin-Huxley model. The fitting first tries to find the optimal *analog* values for these currents, and then converts them to digital values based on the quantization discussed previously.

We can pass optinal keyword arguments to the `fitHH` function that are by default set to `False`:

- `plot_alpha_beta = True`: Plot the comparison of the original $\alpha(V)$ and $\beta(V)$ functions to the fitting results
- `plot_tau_inf = True`: Plot the comparison of the original $\tau(V)$ and $m_{\infty}(V)$ functions to the fitting results

Let's see how this works:

```python
weights = fit_obj.fitHH(plot_alpha_beta=True) # weights stores the sigmoid coefficients
```

We can see that the fits do not look very good initially.

Firstly, it seems that the slopes of the sigmoids are too gentle to capture the steeper slopes of the $\alpha(V)$ and $\beta(V)$ functions, which we observe by comparing the blue and the orange curves. As the slopes of the sigmoid functions are fixed by the physical implementations, we can instead change the slopes of the Hodgkin-Huxley model instead! By changing the `scl_v` parameter of the Hodgkin-Huxley model, we can stretch the voltage range so that our sigmoids fit better to the stretched $\alpha(V)$ and $\beta(V)$ functions. This will effectively render the spikes of the NeuroDyn model larger.

Secondly, we can see that there is a significant discrepancy between the orange line (continuous fit) and the green line (fit after quantization). This happens when the current coefficients are to small to be captured by our quantization range - remember that the smallest current that NeuroDyn can output here is

\begin{equation}
\Delta I_b = \frac{I_{master}}{1024}
\end{equation}

so that if the currents returned by the fitting procedure are smaller than this, they will be set to $0$.

#### Changing the voltage scaling

Let's now try instead to fit the NeuroDyn model to a *scaled* version of the Hodgkin-Huxley object.

```python
scl_v = 3 # voltage scaling

hh_neuron = HHModel(scl_v = scl_v, SI_units = True)

fit_obj = FitND(hh_neuron)

weights = fit_obj.fitHH(plot_alpha_beta = True)
```

The results now look better.

Still, we can see in the plot for $\beta_n(V)$ that the currents are too small to be captured by the quantization. In order to improve the quantized results, we can try manually increasing the coefficients corresponding to the function $\beta_n(V)$.

You can also try changing the fitting parameters to see how they affect the procedure. The parameters that have the most effect are `I_voltage` as it controls the spread of the sigmoids in the voltage range, as well as `scl_v` as it effectively changes the relative slope of the sigmoids.

#### Scaling the time

We need to determine what is the appropriate `scl_t` for the Hodgkin-Huxley model that will allow us to emulate it in hardware.

First, let's address the fact that the capacitance of $C_{ND}$ is different from the capacitance of Hodgkin-Huxley $C_{HH}$. Looking at the membrane voltage equation of Hodgkin-Huxley, we can see that we can keep the dynamics intact if we multiple both sides of the equation by the appropriate constant so that on the left we have the capacitance of the NeuroDyn circuit. This gives the corresponding NeuroDyn maximal conductances and the applied current:

\begin{align}
\bar{g}^{(ND)} &= \bar{g}^{(HH)} \cdot \frac{C_{ND}}{C_{HH}} \\
I_{app}^{(ND)} &= I_{app}^{(HH)} \cdot \frac{C_{ND}}{C_{HH}}
\end{align}

This effectively means that we need to decrease the conductances and the applied current of the NeuroDyn circuit to adjust for the difference in membrane capacitance.

We now need to ensure that the required parameter values are realizable within the hardware constraints. Remember that the $I_{master}$ current in the NeuroDyn chip effectively sets the range of possible values for the maximal conductance values $\bar{g}$ as well as the sigmoid weights $I_b$.

After we have scaled the maximal conductances appropriately, and have received the analog values for the bias currents $I_b$ from the fitting procedure, we need to make sure that all current values are within the range $(0, I_{master})$. If not, we need to scale all currents so that the maximal value fits into this range, effectively determining the constant `scl_t`.

This procedure is done automatically when we call the `fitHH` method!

#### Getting the parameters

The previous procedure of finding the right time scaling is done automatically by the quantization method in the fitting module. The quantization takes into account the maximum value of the maximal conductance parameter we would like to fit to, as well as the value of the maximal sigmoid weight, and scales the time scale so that the parameters fit within the NeuroDyn physical range.

Let's now obtain the digital parameters that we can then pass to our NeuroDyn class. For this, we can use the methods `get_digital_Ib(weights)`, `get_digital_g(g)` and `get_digital_E(E)` that will give us the digital values for the sigmoid weights, maximal conductances, and reversal potentials, respectively. The values will be automatically scaled, taking into account the `scl_v` parameter of the Hodgkin-Huxley model passed at the initialization, as well as the time scaling `scl_t` necessary for all the parameters to fit within the physical range.

```python
g0 = [120e-3,36e-3,0.3e-3] # maximal conductances
E0 = [120e-3,-12e-3,10.6e-3] # reversal potentials
dIb = fit_obj.get_digital_Ib(weights)
dg = fit_obj.get_digital_g(g0)
dE = fit_obj.get_digital_E(E0)

I_master = fit_obj.I_master
I_voltage = fit_obj.I_voltage

# Manually increase coefficients if too low after fitting
dIb[2][1] = dIb[2][1]*15

scl_t = fit_obj.get_scl_t()
print("The time scaling scl_t is", scl_t)

#%% Print the parameter values
print('\nImaster = ', I_master)
print('Ivoltage = ', I_voltage)

print('\nDigital values for maximal conductances:')
print('[gna, gk, gl] = ', dg, '\n')

print('Digital values for reversal potentials:')
print('[Ena, Ek, El] = ', dE, '\n')

print('Digital values for gating variable kinetics:')
print('alpha_m = ', dIb[0][0])
print('beta_m = ', dIb[0][1], '\n')
print('alpha_h = ', dIb[1][0])
print('beta_h = ', dIb[1][1], '\n')
print('alpha_n = ', dIb[2][0])
print('beta_n = ', dIb[2][1], '\n')
```

### Simulating the fitted model

Let's now try to use the parameters we have just calculated in order to define a new NeuroDyn object:

```python
I0 = 0e-6 # current in the original Hodgkin-Huxley model
Iapp = lambda t : fit_obj.convert_I(I0) # scale the current based on scl_t, scl_v and C_HH/C_ND

V_ref = 0.9

ND = NeuroDynModel(dg, dE, dIb, V_ref, I_voltage, I_master)

T = 0.02
trange = (0, T)

sol = ND.simulate(trange,[0.7,0,0,0],Iapp)

plt.figure()
plt.xlabel('t [s]')
plt.ylabel('V [V]')
plt.title('NeuroDyn simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

We can compare this to the scaled Hodgkin-Huxley model:

```python
hh_neuron = HHModel(scl_v = 3, scl_t = scl_t, SI_units = True)

I0 = 8e-6
Iapp = lambda t: I0

T = 0.02
trange = (0, T)

sol = hh_neuron.simulate(trange,[0,0,0,0],Iapp)

plt.figure()
plt.xlabel('t')
plt.ylabel('V')
plt.title('Hodgkin-Huxley simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

### Passing analog values to NeuroDyn

Instead of passing the digital parameter values `dg`, `dErev` and `dIb` we can also see how the NeuroDyn circuit would behave with the exact analog values obtained through the fitting process. This can allow us to get a better understanding of how the quantization error affects the circuit behavior.

This can be easily done by passing a keyword argument `digital_values = False` when creating a NeuroDyn object. The object in this case expects analog values to be passed to `dg`, `dErev` and `dIb` variables instead of the standard integers between $0$ and $1023$.

In order to get the analog values, we can call the corresponding `get_analog_Ib(weights)`, `get_analog_g(g)` and `get_analog_E(E)` instead of the digital methods that we have used previously.

Let's see how this works using the results from our previous fitting procedure:

```python
Ib = fit_obj.get_analog_Ib(weights)
g = fit_obj.get_analog_g(g0)
E = fit_obj.get_analog_E(E0)

ND_analog = NeuroDynModel(g, E, Ib, V_ref, I_voltage, I_master, digital_values = False)

I0 = 0e-6 # current in the original Hodgkin-Huxley model
Iapp = lambda t : fit_obj.convert_I(I0) # scale the current based on scl_t and C_HH/C_ND

T = 0.02
trange = (0, T)

sol = ND_analog.simulate(trange,[0.7,0,0,0],Iapp)

plt.figure()
plt.xlabel('t')
plt.ylabel('V')
plt.title('NeuroDyn simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

You may notice in this case that even by increasing the applied current to the neuron, the circuit does not start regularly spiking. The main source of discrepancy that is left here is the non-linearity of the conductance relationship involving the $\tanh$ input-output relationship. This non-linearity means that the same conductance values will lead to a somewhat different behavior for the same parameters, as observed by the lack of regular spiking here. Still, slight changes to the values may fix this - in this case, we can decrease the potassium conductance to *increase* the excitability of the neuron.

```python
g[1] = 0.6*g[1] # decrease potassium conductance

ND_analog = NeuroDynModel(g, E, Ib, V_ref, I_voltage, I_master, digital_values = False)

I0 = 7e-6 # current in the original Hodgkin-Huxley model
Iapp = lambda t : fit_obj.convert_I(I0) # scale the current based on scl_t and C_HH/C_ND

T = 0.02
trange = (0, T)

sol = ND_analog.simulate(trange,[0.7,0,0,0],Iapp)

plt.figure()
plt.xlabel('t')
plt.ylabel('V')
plt.title('NeuroDyn simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

## Modelling variability

The model we have introduced in the previous sections of course presents an idealized picture compared to the physical realization in hardware. The difference stems from the inherent noise and variability present in electrical circuits, which is especially true for subthreshold analog techniques that we are utilizing within NeuroDyn. Due to this effect, setting the same set of external parameters to different NeuroDyn chips will yield different behavior and appropriate compensation techniques will need to be applied to tackle the variability.

The basic source of variability is the *transistor mismatch*: applying the same gate voltage to different transistors within the circuit will generate different drain currents. We can model this variability in a simple way, by considering that these currents will obey a Gaussian distribution:

\begin{equation}
I_{out} = I_0 \cdot (1 + \mathcal{N}(0, \sigma^2))
\end{equation}

where $I_0$ is the nominal output current and $\mathcal{N}$ is a Gaussian distribution with the standard deviation $\sigma$. A rule of thumb that we use here is to set $\sigma \approx 0.15$, which is the default value in the model.

### Perturb function

We can apply the transistor current variability to our NeuroDyn model by calling the function `perturb(sigma)`. This function will apply the Gaussian distribution variability to the main parameters affected: maximal conductances, reversal potentials, sigmoid weights of the $\alpha(V)$ and $\beta(V)$ functions and the gating variable exponents (e.g. $4$ in $m^4$ for the sodium current).

Let's see how our previously defined model behaves after we add the variability (make sure you ran the previous code where we defined the initial digital NeuroDyn model):

```python
I0 = 0e-6
Iapp = lambda t : fit_obj.convert_I(I0)

ND.perturb() # leaving out the keyword argument sets default sigma = 0.15

sol = ND.simulate(trange,[0.7,0,0,0],Iapp)

plt.figure()
plt.xlabel('t')
plt.ylabel('V')
plt.title('Perturbed NeuroDyn simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

We can rerun this code multiple times and see how each instance behaves. You should notice that each time we generate an instance of the neuron the behavior changes - the spike frequency and height may change, and sometimes we do not get spiking at all!

You can try compensating for this by changing some parameters of the model to make the neuron excitable again. For example, maybe due to the perturbation the threshold voltage has increased and a simple increase in the applied current can get it back to spiking. Alternatively we may need to increase the maximal conductance of the sodium current relative to the maximal conductance of the potassium current in order to increase the excitability of the model. Sometimes though the perturbation might just be too large for these simple techniques to work!

Let's try this out. First, let's check the current values of these parameters:

```python
print("The current value of applied current is ", I0)
print("The current values for the maximal conductances are", dg)
```

Now let's see if our simple compensation technique can work:

```python
# First try changing the applied current
I0 = 0e-6
Iapp = lambda t : fit_obj.convert_I(I0)

# If changing applied current didn't help, try adjusting the maximal conductances
# dg = [1023, 307, 3]
dg_comp = np.array([1023, 307, 3])
ND.update_dg(dg_comp)

sol = ND.simulate(trange,[0.7,0,0,0],Iapp)

plt.figure()
plt.xlabel('t')
plt.ylabel('V')
plt.title('Compensated NeuroDyn simulation')
plt.plot(sol.t, sol.y[0])
plt.show()
```

You can try changing other parameters as well and get a feel for how hard it is to compensate for the internal mismatch -  this inherent variability is common to both biological neurons and neuromorphic circuits and is the main challenge to designing and controlling bioinspired circuits.

```python

```


# Pronóstico adaptativo

## Descripción del problema real

Los pronósticos de los precios de la electricidad en mercados liberalizados son un insumo fundamental para la toma de decisiones dentro de las organizaciones. Fundamentalmente, los pronosticos de corto plazo son utilizados en decisiones de carácter operativo. En el caso abordado, es necesario contar con los pronósticos para definir la politica de operación de una empresa del sector eléctrico.

## Descripción del problema en términos de los datos

La carpeta `datos/precios/` contiene los precios historicos horarios de la electricidad en la Bolsa de Energía del mercado eléctrico colombiano, publicados por el operador del mercado. Se desean obtener pronósticos para el precio promedio diario para los próximos siete (7) días a la fecha actual.

## Aproximaciones posibles

En este caso, se desea evaluar la capacidad de pronóstico de un ADALINE con aprendizaje en línea.

## Requerimientos

Usted debe:

* Procesar los datos históricos para conformar el conjunto de datos. Los archivos de Excel no pueden ser modificados y actualizados directamente por el operador del sistema. Su código debe leer los archivos y crear la serie de precios promedio diarios de la electricidad.

* Determinar si el modelo debe pronosticar los precios promedios sin ninguna transformación, o si la inclusión de una transformación (logaritmo natural, raíz cúbica, raíz cuadrada, etc) resulta en un incremento de la precisión del pronóstico.

* Generar el pronóstico para los precios de los próximos siete días.

* Preparar el código para que el modelo sea entrenado usando el gradiente y el momentum.

* Determinar cuál es el número óptimo de retardos (observaciones) que el modelo debe considerar hacia atrás para producir el pronóstico.

* Determinar los valores óptimos de la tasa de aprendizaje y el momentum.

```python
import numpy as np
np.random.seed(0)
```

## Cargar y Limpieza de Datos

Se hace una lectura de las bases propuestas para el ejercicio y se procede a unir todas las bases para trabajar con un mismo conjunto de datos, luego se promedia el precio de la energia por día y se crea un campo 'year_month' el cual nos servira mas adelante para hacer un join.

```python
from os import scandir,getcwd
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

DIR = 'datos/precios/'
archivos = scandir(DIR)
data = pd.DataFrame()
for archivo in archivos:
    print(archivo.name)
    auxdata = pd.read_excel(DIR + archivo.name,header=None,names=['fecha'] + [str(i)for i in range(1,25)]+['basura1']+['basura2'])
    data = pd.concat([data,auxdata])

data = data.drop(['basura1','basura2'],axis=1)
print('longitud:',len(data))
data = data.dropna()
print('longitud despues de eliminar na:',len(data))
data = data.drop(data[data['fecha']=='Fecha'].index)
print('Longitud despues de eliminar header fecha',len(data))
data = data.reset_index(drop=True)

#data.to_excel(DIR + 'precios.xlsx')
data['fecha']=data['fecha'].astype('datetime64[D]')

data['promedio'] = data.mean(axis=1)

data['year'] = data["fecha"].dt.strftime("%Y")
data['year'] = data['year'].astype('int')

data['month'] = data["fecha"].dt.strftime("%m")
data['month'] = data['month'].astype('int')

data['day'] = data["fecha"].dt.strftime("%d")
data['day'] = data['day'].astype('int')

data = data.sort_values(['year','month','day'],ascending = True)
data['year_month'] = data['year']*100  + data['month']
#data.head()

plt.figure(figsize=(22,6))
plt.plot(data['promedio'].values, color='black');
data.head()

```

## IPC

Para trabajar los precios sin tener en cuenta la inflación y poder reducirle complejidad al modelo, extraemos un historico de los IPC para colombia de la pagina .... luego hacemos un join con nuestra base de precios de energía por el campo 'year_month' que nos da la información del IPC mensual.

Luego se procede mediante una regla de 3 eliminar la inflación en los precios, tomando el IPC de 1995 como base.

$$Precio_i = Precio_i * \frac{IPC_{base}}{IPC_i}$$

```python
data_ipc = pd.read_excel('datos/ipc.xlsx',names=['fecha','indice','inflacion_anual','inflacion_mensual','inflacion_año_corrido'])
data_ipc = data_ipc.dropna()
data_ipc = data_ipc.drop(11)
data_ipc = data_ipc.sort_values(['fecha'],ascending = True)
data_ipc = data_ipc.reset_index(drop=True)
data_ipc['fecha'] = data_ipc['fecha'].astype('str')
data_ipc['year_month'] = data_ipc['fecha'].astype('int')
data_ipc = data_ipc.drop(['fecha'],axis=1)

data = pd.merge(data,data_ipc,on='year_month')
ipc_base = data['indice']
data['promedio'] = data['promedio']*ipc_base / data['indice']
data['promedio'] = data['promedio'].astype('int')

plt.figure(figsize=(22,6))
plt.plot(data['promedio'].values, color='black');
data.head()
```

```python
base_data = data
base_data.head()
```

      fecha
      1
      2
      3
      4
      5
      6
      7
      8
      9
      ...
      24
      promedio
      year
      month
      day
      year_month
      indice
      inflacion_anual
      inflacion_mensual
      inflacion_año_corrido

      0
      1995-07-21
      1.073
      1.000
      1.000
      1.00
      1.00
      1.000
      5.000
      6.000
      6.000
      ...
      1.000
      5
      1995
      7
      21
      199507
      20.93
      21.5
      0.77
      14.7

      1
      1995-07-22
      1.073
      1.073
      1.000
      1.00
      1.00
      1.073
      1.303
      1.303
      1.303
      ...
      1.073
      1
      1995
      7
      22
      199507
      20.93
      21.5
      0.77
      14.7

      2
      1995-07-23
      1.073
      1.000
      1.000
      1.00
      1.00
      1.000
      0.100
      1.000
      1.000
      ...
      1.000
      0
      1995
      7
      23
      199507
      20.93
      21.5
      0.77
      14.7

      3
      1995-07-24
      1
      1.000
      0.990
      1.00
      1.00
      1.073
      3.000
      3.000
      3.000
      ...
      1.000
      4
      1995
      7
      24
      199507
      20.93
      21.5
      0.77
      14.7

      4
      1995-07-25
      0.99
      0.990
      0.989
      0.99
      0.99
      1.073
      1.263
      1.263
      1.263
      ...
      0.990
      1
      1995
      7
      25
      199507
      20.93
      21.5
      0.77
      14.7

5 rows × 34 columns

## Implementación del Modelo

La implemetnación del modelo se hace con base en el codigo de clase sin ninguna modificación.

```python
import numpy as np
import math
class Adaline:
    def __init__(self,
                 learning_rate=0.001, # tasa de aprendizaje
                 max_epochs=100,      # número máximo de iteraciones sobre el set de datos
                 shuffle=False,       # mezcla patrones para aprendizaje online
                 random_state=None,   #
                 warm_start=False):   #

        self.learning_rate = learning_rate
        self.max_epochs = max_epochs
        self.shuffle = shuffle
        self.random_state = random_state
        self.warm_start = warm_start
        self.coef_ = None
        self.intercept_ = None

    def fit(self, X, y):

        if not isinstance(X, np.ndarray):
            X = np.array(X)
        if not isinstance(y, np.ndarray):
            d = np.array(y)
        else:
            d = y.copy()

        if self.random_state is not None:
            np.random.RandomState(self.random_state)

        if self.coef_ is None or self.warm_start is False:
            self.coef_ = np.random.uniform(-1, 1, X.shape[1])

        if self.intercept_ is None  or self.warm_start is False:
            self.intercept_ = np.random.uniform(-1, 1, 1)

        errors2 = []
        forecasts = []

        for epoch in range(self.max_epochs):

            if self.shuffle is True:
                n = list(range(X.shape[0]))
                np.random.shuffle(n)
                X = X[n,:]
                d = d[n]

            for i in range(X.shape[0]):
                u = np.dot(X[i,:], self.coef_) + self.intercept_
                e = (d[i] - u)[0]
                self.coef_ += 2 * self.learning_rate * e * X[i,:]
                self.intercept_ += 2 * self.learning_rate * e
                errors2.append(e**2)
                forecasts.append(u)

        return errors2, forecasts

    def predict(self, X):
        if not isinstance(X, np.ndarray):
            X = np.array(X)
        u = np.dot(X, self.coef_) + self.intercept_
        return u
```

## Procesamiento de Datos

```python
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline

data = base_data['promedio']
data = np.array(data.values)
print('Varianza :',np.var(data))

#Logaritmica
log = np.log(data)
print('Varianza Logaritmica:',np.var(log/log.max()))

#Raiz cuadrada
rai2 = np.sqrt(data)
print('Varianza raiz cuadrada:',np.var(rai2/rai2.max()))

#Raiz cubica
rai3 = np.power(data,1/3)
print('Varianza raiz cubica:',np.var(rai3/rai3.max()))

#Raiz cuatro
rai4 = np.power(data,1/4)
print('Varianza raiz cubica:',np.var(rai4/rai4.max()))

plt.figure(figsize=(22,6))
plt.title('Serie de Tiempo Precio de la Energía')
plt.ylabel('COP')
#plt.plot(data, color='black',label='Original')
plt.plot(log, color='red',label='Logaritmica')
plt.plot(rai2, color='blue',label='Raiz Cuadrada')
plt.plot(rai3, color='yellow',label='Raiz Cubica')
plt.plot(rai4, color='green',label='Raiz 4')
plt.legend()
plt.show()
```

Luego de analisadas las distintas trasnformación, se descarta de primera la función logaritmica, ya que esta genera valores en cero que traen como consecuencias perdidas de información; Luego se analisan las tranformaciones de las raices y se puede ver que su forma no cambia, solo cambian su escala de datos, a partir de esto se procede a trabajar con la raiz cubica y se le aplica una normalización.

```python
from sklearn.preprocessing import MinMaxScaler
#Normalizamos los datos
transformer = MinMaxScaler().fit(np.array(rai3).reshape(-1, 1))
d = transformer.transform(np.array(rai3).reshape(-1, 1))
plt.figure(figsize=(22,6))
plt.plot(d, color='yellow',label='Raiz Cuadrada');
d = list(d.reshape(1,-1)[0])
```

## Entrenamiento y Evaluación

Se procede luego a entrenar el modelo, haciendo una busqueda manual de los mejores parametros los cuales nos disminuyan las metricas tomadas, que en este caso fueron el MSE y el MAE, donde apartir de esto tenemos que los siguientes parametros fueron los que mejor se comportaron.

\begin{align}
L = 8 \\
learning rate = 0.0005 \\
max epochs = 1000 \\
\end{align}

```python
import matplotlib.pyplot as plt
from sklearn.preprocessing import normalize
%matplotlib inline

##
##  Para pronosticar el valor actual se toman los `L`
##  valores previos de la serie
##
L = 8

##
## Forma la matriz X
##
X = []
for t in range(L, len(d)):
    X.append(d[t-L:t].copy())

adaline = Adaline(
    learning_rate=0.0005,  # tasa de aprendizaje
    max_epochs=1000,        # número máximo de iteraciones sobre el set de datos
    shuffle=False,       # mezcla patrones para aprendizaje online
    random_state=123,    # corrida reproducible
    warm_start=False)    # inicia con pesos aleatorios

errors2, forecasts = adaline.fit(
    X,
    d[L:])

len_test = 7
test = d[len(d)-len_test:len(d)]
predict =forecasts[len(forecasts)-len_test:len(forecasts)]

#Transformada inversa
#predict = np.power(predict,2)
#Desnormalizamos
#predict =  [(i*rango)+min(data) for i in predict]

plt.figure(figsize=(14,3))
plt.plot(test, color='black');
plt.plot(predict, color = 'red');

from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error

mae = mean_absolute_error(test, predict)
mse = mean_squared_error(test, predict)

print('Error Medio Cuadratico:',mse)
print('Error Medio Absoluto:',mae)
```

Se realiza la trasnformada inversa al procesamiento que se le hizo a los datos para corrobar su comportamiento con respecto a la salida original (teniendo en cuenta la conversión con el IPC)

```python
len_test = 7
test = data[len(data)-len_test:len(data)]
predict =forecasts[len(forecasts)-len_test:len(forecasts)]

#Transformada Inversa
predict = transformer.inverse_transform(predict)
predict = np.power(predict,3)

plt.figure(figsize=(14,3))
plt.plot(test, color='black');
plt.plot(predict, color = 'red');

from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error

mae = mean_absolute_error(test, predict)
mse = mean_squared_error(test, predict)

print('Error Medio Cuadratico:',mse)
print('Error Medio Absoluto:',mae)
```

```python

```


# SI within and outside its limits (unlimited method)

```python
# %load ../../imports.py
"""
These is the standard setup for the notebooks.
"""

%matplotlib inline
%load_ext autoreload
%autoreload 2

#from jupyterthemes import jtplot
#jtplot.style(theme='onedork', context='notebook', ticks=True, grid=False)

import pandas as pd
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)
import numpy as np
import os
import matplotlib.pyplot as plt
#plt.style.use('paper')

#import data
import copy
from rolldecay.bis_system import BisSystem
from rolldecay import database
from mdldb.tables import Run

from sklearn.pipeline import Pipeline
from rolldecayestimators.transformers import CutTransformer, LowpassFilterDerivatorTransformer, ScaleFactorTransformer, OffsetTransformer
from rolldecayestimators.direct_estimator_cubic import EstimatorQuadraticB, EstimatorCubic
from rolldecayestimators.ikeda_estimator import IkedaQuadraticEstimator
import rolldecayestimators.equations as equations
import rolldecayestimators.lambdas as lambdas
from rolldecayestimators.substitute_dynamic_symbols import lambdify
import rolldecayestimators.symbols as symbols
import sympy as sp

from sklearn.metrics import r2_score
import rolldecay.paper_writing as paper_writing

```

```python
import rolldecayestimators.measure as measure
from rolldecay.froude_scaling import froude_scale
import seaborn as sns
import rolldecay
```

```python
df_rolldecay = database.load(rolldecay_table_name='rolldecay_quadratic_b', limit_score=0.95,
                             exclude_table_name='rolldecay_exclude')

df_ikeda = database.load(rolldecay_table_name='rolldecay_simplified_ikeda_unlimited', limit_score=-np.inf,
                             exclude_table_name='rolldecay_exclude')

```

```python
df_ikeda.describe()
```

```python
df_rolldecay.describe()
```

```python
df_ = pd.merge(left=df_rolldecay, right=df_ikeda, how='inner', left_index=True, right_index=True, suffixes=('','_ikeda'))
df_.describe()
```

```python
db = database.get_db()
```

```python
df_rolldecay['ship_speed']*=1.852/3.6
df_ikeda['ship_speed']*=1.852/3.6

description = pd.read_sql_table('description', con=db.engine, index_col='id')
description.loc['ship_speed','unit']='m/s'
description.loc['VDES','unit']='m/s'
description.loc['Disp'] = {'description':'Ship discplacement','unit':'m3'}

df_ikeda['Disp']=df_ikeda['Volume']
df_rolldecay['Disp']=df_rolldecay['Volume']

skip=['omega0']  # This one is already model scale...
df_ikeda = froude_scale(data=df_ikeda, description=description, skip=skip)
df_rolldecay = froude_scale(data=df_rolldecay, description=description, skip=skip)
```

```python
df_compare = measure.linearized_matrix(df_rolldecay=df_rolldecay, df_ikeda=df_ikeda)
g=9.81
df_compare['Fn']=df_compare['ship_speed']/np.sqrt(df_compare['lpp']*g)
mask = df_compare['A0'] > 0.95
df_compare=df_compare.loc[mask].copy()
```

```python
df_compare.describe()
```

```python
len(df_compare['id'].unique())
```

```python
df_compare.head()
```

```python
fig,ax=plt.subplots()
df_compare.plot(x='B_e_hat', y='B_e_hat_ikeda', ax=ax, style='.')

xlim = ax.get_xlim()
ylim = ax.get_ylim()
lim = np.max([xlim[1],ylim[1]])
ax.set_xlim(0,lim)
ax.set_ylim(0,lim)
ax.plot([0,lim],[0,lim],'r-')
ax.set_aspect('equal', 'box')
ax.legend()
ax.grid(True)
```

```python
r2_score(y_true=df_compare['B_e_hat'], y_pred=df_compare['B_e_hat_ikeda'])
```

```python
df_compare['error'] = (df_compare['B_e_hat_ikeda'] - df_compare['B_e_hat']).abs()
```

## Now remove point outside the limits

```python
from rolldecayestimators.simplified_ikeda import limits_kawahara
```

```python
limits_kawahara=pd.Series(limits_kawahara)
```

```python
df_compare['g']=9.81
df_compare['draught']=(df_compare['TA'] + df_compare['TF'])/2
df_compare['CB'] = df_compare['Volume']/(df_compare['lpp']*df_compare['beam']*df_compare['draught'])
df_compare[r'B/d']=df_compare['beam']/df_compare['draught']
df_compare['OG']=df_compare['draught']-df_compare['kg']
df_compare[r'OG/d']=df_compare['OG']/df_compare['draught']
df_compare['CMID']=df_compare['A0']
df_compare[r'bBk/B']=df_compare['BKB']/df_compare['beam']
df_compare[r'lBk/LPP']=df_compare['BKL']/df_compare['lpp']
df_compare['OMEGA_hat']=lambdas.omega_hat(beam=df_compare['beam'], g=df_compare['g'], omega0=df_compare['omega0'])
```

```python
limit_keys = list(limits_kawahara.keys())
corr = df_compare[limit_keys].corr().abs()

mask = np.zeros_like(corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

ax = sns.heatmap(corr, vmin=0, vmax=1, cmap='gist_yarg', annot=True, mask = mask)
```

```python
y_vars = list(limit_keys)+['Fn']
sns.pairplot(data=df_compare, x_vars=y_vars, y_vars=y_vars, markers='.', height=1.5)
```

```python
def compare_versus(x,y):
    fig,ax=plt.subplots()

    df_compare.plot(x=x,y=y, style='.', ax=ax)
    ax.set_ylabel(y)
    xlim=ax.get_xlim()
    ylim=ax.get_ylim()
    alpha=0.7

    y_=[np.max([ylim[0],limits_kawahara[y][0]]),
      np.min([ylim[1],limits_kawahara[y][1]]),
      ]
    ax.fill_between(x=xlim, y1=y_[0], y2=y_[1], color='g', alpha=alpha)

    x_=[np.max([xlim[0],limits_kawahara[x][0]]),
      np.min([xlim[1],limits_kawahara[x][1]]),
      ]
    ax.fill_between(x=x_, y1=ylim[0], y2=ylim[1], color='g', alpha=alpha);

    return ax
```

```python
compare_versus(x='CMID', y='B/d');
```

```python
compare_versus(x='B/d', y='bBk/B');
```

```python
compare_versus(x='CMID', y='bBk/B');
```

```python

g = sns.jointplot(x=r'B/d', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'B/d', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'CMID', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'OMEGA_hat', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'OG/d', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'phi_a', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

g = sns.jointplot(x=r'Fn', y='error', data=df_compare,
                  kind="reg", truncate=False,
                  color="m", height=7)

```

```python
fig,ax=plt.subplots()
for phi_a, group in df_compare.groupby(by='phi_a'):
    group.sort_values(by='B/d', inplace=True)
    group.plot(x='B/d', y='error', style='-', ax=ax)

```

```python
fig,ax=plt.subplots()
df_compare.plot(x='B/d', y=['B_e_hat_ikeda','B_e_hat'],ax=ax, style='o', alpha=0.5)
```

```python
bad_ship = df_compare.sort_values(by='B_e_hat_ikeda', ascending=False).iloc[0]
bad_ship['B_e_hat_ikeda']
```

```python
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot(df_compare['B/d'], df_compare['OG/d'], df_compare['B_e_hat_ikeda'], '.')
ax.plot(df_compare['B/d'], df_compare['OG/d'], df_compare['B_e_hat'], '.')
ax.plot(bad_ship['B/d'], bad_ship['OG/d'], bad_ship['B_e_hat_ikeda'],'ro')
#ax.set_zlim(0,0.15)

```

```python
bad_ship['OG/d']
```

```python
np.rad2deg(bad_ship['phi_a'])
```

```python
from rolldecayestimators.simplified_ikeda_class import SimplifiedIkeda, SimplifiedIkedaBK2
bad_ship['V']=bad_ship['ship_speed']
bad_ship['w']=bad_ship['omega0']
bad_ship['fi_a']=bad_ship['phi_a']
bad_ship['volume']=bad_ship['Volume']
bad_ship['lBK']=bad_ship['BKL']
bad_ship['bBK']=bad_ship['BKB']

si = SimplifiedIkeda(**bad_ship)
si.calculate_B44()

```

```python
si2 = SimplifiedIkedaBK2(**bad_ship)
si2._R=0.02*bad_ship['beam']
si2.calculate_B44()
```

```python
bad_ship2 = bad_ship.copy()
N=100

b_ds=np.linspace(4.5,6,N)
bad_ship2['B/d']=np.tile(b_ds,[N,1])
bad_ship2['draught'] = bad_ship2['beam']/bad_ship2['B/d']

OG_ds =  np.linspace(-0.85,-0.5,N)
bad_ship2['OG/d'] = np.tile(OG_ds,[N,1]).T
bad_ship2['OG'] = bad_ship2['OG/d']*bad_ship2['draught']
bad_ship2['kg'] = bad_ship2['draught'] - bad_ship2['OG']

si2 = SimplifiedIkeda(**bad_ship2)

B_44_hat = si2.calculate_B44()

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot_surface(bad_ship2['B/d'], bad_ship2['OG/d'], B_44_hat)

ax.plot(bad_ship['B/d'], bad_ship['OG/d'], si.calculate_B44()[0],'ro')
```

```python
fig,ax=plt.subplots()
cs = ax.contour(bad_ship2['B/d'], bad_ship2['OG/d'], B_44_hat, levels=6)
ax.clabel(cs, inline=1, fontsize=10)
ax.set_xlabel('B/d')
ax.set_ylabel('OG/d')

df_compare.plot(x='B/d', y='OG/d', style='b.', ax=ax)
ax.plot(bad_ship['B/d'], bad_ship['OG/d'],'ro')
```

<a href id='mountain'></a>

```python
good_ship = df_compare.sort_values(by='error', ascending=True).iloc[0]
good_ship['V']=good_ship['ship_speed']
good_ship['w']=good_ship['omega0']
good_ship['fi_a']=good_ship['phi_a']
good_ship['volume']=good_ship['Volume']
good_ship['lBK']=good_ship['BKL']
good_ship['bBK']=good_ship['BKB']

good_ship2 = good_ship.copy()
N=30

b_ds=np.linspace(df_compare['B/d'].min(),df_compare['B/d'].max(),N)
good_ship2['B/d']=np.tile(b_ds,[N,1])
good_ship2['draught'] = good_ship2['beam']/good_ship2['B/d']

OG_ds =  np.linspace(df_compare['OG/d'].min(),df_compare['OG/d'].max(),N)
good_ship2['OG/d'] = np.tile(OG_ds,[N,1]).T
good_ship2['OG'] = good_ship2['OG/d']*good_ship2['draught']
good_ship2['kg'] = good_ship2['draught'] - good_ship2['OG']

si = SimplifiedIkeda(**good_ship)
si2 = SimplifiedIkeda(**good_ship2)

B_44_hat = si2.calculate_B44()

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot_surface(good_ship2['B/d'], good_ship2['OG/d'], B_44_hat)

ax.plot(good_ship['B/d'], good_ship['OG/d'], si.calculate_B44()[0],'ro')
```

```python
fig,ax=plt.subplots()
cs = ax.contour(good_ship2['B/d'], good_ship2['OG/d'], B_44_hat, levels=3)
ax.clabel(cs, inline=1, fontsize=10)
ax.set_xlabel('B/d')
ax.set_ylabel('OG/d')

df_compare.plot(x='B/d', y='OG/d', style='b.', ax=ax)
ax.plot(good_ship['B/d'], good_ship['OG/d'],'ro')
```

```python
from matplotlib import cm

fig,ax=plt.subplots()
cs = ax.contourf(good_ship2['B/d'], good_ship2['OG/d'], B_44_hat, cmap='gray_r')
ax.set_xlabel('B/d')
ax.set_ylabel('OG/d')

df_compare.plot(x='B/d', y='OG/d', style='b.', ax=ax)
ax.plot(good_ship['B/d'], good_ship['OG/d'],'ro')
cbar = fig.colorbar(cs)
```

```python
good_ship['B/d']
```

```python
good_ship['OG/d']
```

```python
corr = df_compare[limit_keys+['error']].corr().abs()

fig,ax=plt.subplots()
corr.plot(y='error', kind='bar', ax=ax);
ax.set_title('Absolut correlation coefficient for the error');
```

```python
fig,axes=plt.subplots(nrows=len(limits_kawahara))
fig.set_size_inches(15,12)

for (item, limits),ax in zip(limits_kawahara.items(),axes):

    df_compare.hist(item, ax=ax)
    ax.set_xlabel(item)
    ax.set_title('')

    ylims = ax.get_ylim()
    ax.fill_between(limits, [ylims[1],ylims[1]], y2=0, color='green', alpha=0.5, label='valid')

plt.tight_layout()

```

```python
def cut(df, limits, item):
    df=df.copy()
    mask = ((df[item] >= limits[0]) & (df[item] <= limits[1])) | (df[item]==0)
    df=df.loc[mask].copy()
    return df

def plot_select(df_compare, limits_used):

    df_compare_within = df_compare.copy()

    fig,axes=plt.subplots(nrows=len(limits_used))
    fig.set_size_inches(15,12)

    for (item, limits),ax in zip(limits_used.items(),axes):

        n_bins = 20
        bins=np.linspace(df_compare[item].min(), df_compare[item].max(),n_bins)
        df_compare.hist(item, ax=ax, bins=bins, color='gray')

        df_compare_within = cut(df=df_compare_within, limits=limits, item=item)
        df_compare_within.hist(item, ax=ax, bins=bins, color='blue')
        ax.set_xlabel(item)
        ax.set_title('')

        ylims = ax.get_ylim()
        ax.fill_between(limits, [ylims[1],ylims[1]], y2=0, color='green', alpha=0.5, label='valid')

    plt.tight_layout()

    fig,ax=plt.subplots()
    df_compare_within.plot(x='B_e_hat', y='B_e_hat_ikeda', ax=ax, style='.')

    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    lim = np.max([xlim[1],ylim[1]])
    ax.set_xlim(0,lim)
    ax.set_ylim(0,lim)
    ax.plot([0,lim],[0,lim],'r-')
    ax.set_aspect('equal', 'box')
    ax.legend()
    ax.grid(True)

    r2 = r2_score(y_true=df_compare_within['B_e_hat'], y_pred=df_compare_within['B_e_hat_ikeda'])
    print('r2:%f' % r2)

    return df_compare_within

```

```python
limits_kawahara
```

If we put **bBk/B** and **CMID** first, the other limits becomes redundant:

```python
keys_ordered=[
'bBk/B',
'CMID',
'CB',
'B/d',
'OG/d',
'lBk/LPP',
'OMEGA_hat',
]

limits_used = limits_kawahara[keys_ordered]
df_compare_within = plot_select(df_compare=df_compare, limits_used=limits_used)
```

```python
limits_used = limits_kawahara.copy()
limits_used.pop('CMID')
df_compare_within = plot_select(df_compare=df_compare, limits_used=limits_used)

```

```python
def select(df_compare, limits_used):

    df_compare_within = df_compare.copy()

    for item, limits in limits_used.items():

        df_compare_within = cut(df=df_compare_within, limits=limits, item=item)

    return df_compare_within
```

```python
limits_used = limits_kawahara.copy()
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
len(df_['id'].unique())
```

```python
limits_used = limits_kawahara.copy()
limits_used.pop(r'CMID')

df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = {}
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['B/d']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['B/d','bBk/B']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['B/d','bBk/B','CMID']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['CMID']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['CMID','bBk/B']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
len(df_['id'].unique())
```

```python
limits_used = limits_kawahara[['CMID','B/d']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])
```

```python
limits_used = limits_kawahara[['CMID','bBk/B']]
df_ = select(df_compare=df_compare, limits_used=limits_used)
r2_score(y_true=df_['B_e_hat'], y_pred=df_['B_e_hat_ikeda'])

df_compare_within = plot_select(df_compare=df_compare, limits_used=limits_used)
```

```python
size=3.0
with plt.style.context('paper'):

    fig,ax=plt.subplots()
    fig.set_size_inches(size,size)

    df_unlimited = select(df_compare=df_compare, limits_used={})  # Unlimited
    df_unlimited.plot(x='B_e_hat', y='B_e_hat_ikeda', ax=ax, style=r'X', label='no limits')

    limits_used = limits_kawahara[['CMID','bBk/B']]
    df_limited = select(df_compare=df_compare, limits_used=limits_used)
    df_limited.plot(x='B_e_hat', y='B_e_hat_ikeda', ax=ax, style='.', label='within limits')

    ax.set_xlabel(r'$\hat{B}_{e}^{Model}$')
    ax.set_ylabel(r'$\hat{B}_{e}^{SI}$')

    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    lim = np.max([xlim[1],ylim[1]])
    ax.set_xlim(0,lim)
    ax.set_ylim(0,lim)
    ax.plot([0,lim],[0,lim],'r-')
    ax.set_aspect('equal', 'box')
    ax.legend()
    ax.grid(True)

    paper_writing.save_fig(fig=fig, name='si_model_within')

```

```python
r2_score(y_true=df_unlimited['B_e_hat'], y_pred=df_unlimited['B_e_hat_ikeda'])
```

```python
r2_score(y_true=df_limited['B_e_hat'], y_pred=df_limited['B_e_hat_ikeda'])
```

## Save to table

```python
mean_='$R^2$'
number_of_points = 'Number of points'

validation = pd.DataFrame(columns=[mean_, number_of_points])

s = pd.Series(name='SI no limits')
s[mean_] = r2_score(y_true=df_unlimited['B_e_hat'], y_pred=df_unlimited['B_e_hat_ikeda'])
s[number_of_points] = int(len(df_unlimited))
validation=validation.append(s)

s = pd.Series(name='SI within limits')
s[mean_] = r2_score(y_true=df_limited['B_e_hat'], y_pred=df_limited['B_e_hat_ikeda'])
s[number_of_points] = int(len(df_limited))
validation=validation.append(s)

validation[number_of_points]=validation[number_of_points].astype(int)
```

```python
validation
```

```python
latex = validation.to_latex(float_format='%0.2f', na_rep='')
name='si_validation'
file_path = os.path.join(rolldecay.equations_path,name)

# (Uncomment this one if you want to regenerate this table)
paper_writing.save_table(file_path=file_path, tabular_tex=latex, label='tab:si_validation',
           caption='Validation of SI within and outside limits')
```

```python
prefixes = [
    'B_W_e_hat',
    'B_F_e_hat',
    'B_E_e_hat',
    'B_BK_e_hat',
    'B_L_e_hat',]

labels=[
    r'$\hat{B}_{W}$',
    r'$\hat{B}_{F}$',
    r'$\hat{B}_{E}$',
    r'$\hat{B}_{BK}$',
    r'$\hat{B}_{L}$',
    ]

with plt.style.context('paper'):

    fig,ax=plt.subplots()
    fig.set_size_inches(size,size)

    df_unlimited['B_e_hat_residual_si_model_abs'] = (df_unlimited['B_e_hat_ikeda'] - df_unlimited['B_e_hat']).abs()
    df_unlimited.plot(x='B_e_hat_residual_si_model_abs', y=prefixes, style='.', label=labels, ax=ax)
    ax.set_xlabel(r'$|\hat{B}_{e}^{SI}-\hat{B}_{e}^{Model}|$')

    ax.legend()
    ax.grid(True)
    ax.set_ylabel(r'$\hat{B}$ [-]')
    ax.set_aspect('equal', 'box')

    paper_writing.save_fig(fig, name='component_residual')
```

```python
df_unlimited.head()
```

```python

```


### Experiments pipeline description

#### Stage 1. Data mining

The data is split into two populations: `positive` and `negative` samples, natures of mining of which are different.

* positive samples are obtained by going through a list of repositories (20 most starred Java apache repos), and running RefactoringMiner tool to get instances of ExtractMethodRefactorings from authors' commits. For each instance a range of metrics is then computed (78) and written to csv file

* negative samples are obtained by going through a list of repositories (20 most starred Java apache repos), each Java file is then opened, and each method is considered that is then split into statements with PsiMiner. From the list of statements, then all combinations of consequent statements of lengths from 1 to n, where n - number of statements within the method. Then, each combination is considered as a fragment of code, for which the same list of metrics is computed and its Haas score is measured

Haas score, is a scoring function inspired by Haas et al work [cite something] and can be written as
$$Score(F) = LengthScore(F) + AreaScore(F) + DepthScore(F)$$
And each component is defined as
$$
\begin{align}
&LengthScore(F) = min(0.1 \cdot min(Length(F), Length(F^C)), 3) \\
&AreaScore(F) = \frac{2 \cdot Depth(M)}{Area(M)} \cdot min(Area(M) - Area(F), Area(M) - Area(F^C)) \\
&DepthScore(F) = min(Depth(M) - Depth(F), Depth(M) - Depth(F^C)) \\
\end{align}
$$

$M$ - method, $F$ - fragment of code, $F^C$ - fragment's complement w/r to method, or *remainder*

$Length(F)$ - line-count of fragment of code, i.e. number of rows.

$Depth(F)$ - nesting depth of code, i.e. row-wise maximal nesting level within fragment.

$Area(F)$ - nesting area of code, i.e. row-wise sum of nesting levels within fragment.

#### Stage 2. Data preparation

Once the scripts were finished, overall about 16 thousand `positive` and 4 million `negative` samples were obtained. In order to reduce the stress on soon to-be-trained models it was decided to make a smaller subsample of negative observations that accounted for 1 million elements.

After that, 80\% of both sets were left for training and validation stages, and 20% were left for the testing stage.

(80\% from positive set, and 80\% from negative set)

#### Stage 3. Training procedure

Before each models training, two preprocessing tasks were always performed on the dataset.

1. Negative samples Haas scores' of which were too high (higher than 50th percentile, or 95th, depending on the experiment) were dropped from the training set. Haas scores were also dropped from the dataset (not a feature for a model to know)
2. Training data X scaled by MinMax strategy feature-wise, i.e. $X = \frac{X - min(X)}{max(X) - min(X)}$, to ensure distribution of features from 0 to 1. Implemented via `sklearn.preprocessing.MinMaxScaler` and incorporated into the model through sklearn Pipelines

After that, each model was trained with two thresholds for Haas-Score drop (50th, and 95th).

Overall training procedure was conducted with use of `sklearn.model_selection.GridSearchCV` to find optimal hyperparameters from the given *grid* and get CrossValidation scores at the same time. As metric to optimize, the Precision-Recall Area Under Curve was chosen (PR AUC).

#### Stage 4. Testing procedure

Each trained model was tested on the same dataset, that was built during stage 2, and 3 *metrics* were under consideration during the tests.

1. Precision Recall Curve as a whole
2. Area under the curve
3. Recall at points Precision=0.9, and Precision=0.8

Overall, this results are yet to be computed)

### Overall information

#### Repository list

1.  https://github.com/apache/dubbo
2.  https://github.com/apache/kafka
3.  https://github.com/apache/skywalking
4.  https://github.com/apache/flink
5.  https://github.com/apache/rocketmq
6.  https://github.com/apache/shardingsphere
7.  https://github.com/apache/hadoop
8.  https://github.com/apache/druid
9.  https://github.com/apache/zookeeper
10.  https://github.com/apache/pulsar
11.  https://github.com/apache/shardingsphere-elasticjob
12.  https://github.com/apache/cassandra
13.  https://github.com/apache/storm
14.  https://github.com/apache/tomcat
15.  https://github.com/apache/jmeter
16.  https://github.com/apache/zeppelin
17.  https://github.com/apache/incubator-shenyu
18.  https://github.com/apache/beam
19.  https://github.com/apache/groovy
20.  https://github.com/apache/hbase

#### Models used

1. `sklearn.ensemble.RandomForestClassifier` - RandomForest
2. `sklearn.ensemble.BaggingClassifier` of `sklearn.svm.SVC` - Ensemble of support vector machines
3. `sklearn.naive_bayes.GaussianNB` - Gaussian Naive Bayes
4. `sklearn.neural_network.MLPClassifier` - MultiLayer Perceptron
5. `sklearn.ensemble.GradientBoostingClassifier` - Ensemble of Decision Trees with Gradient Boosting

#### Models' Inference

Models were trained with sklearn, Python's module and saved in PMML format via [sklearn2pmml](https://github.com/jpmml/sklearn2pmml), then used in Java through [pmml4s library](https://github.com/autodeployai/pmml4s)

```python

```


```python
import pycalphad
from pycalphad.tests.datasets import ALFE_TDB
from pycalphad import Database, Model
import pycalphad.variables as v
from sympy import Piecewise, Function

dbf = Database(ALFE_TDB)
mod = Model(dbf, ['AL','FE', 'VA'], 'B2_BCC')
t = mod.ast.diff(v.Y('B2_BCC', 1, 'AL'), v.Y('B2_BCC', 0, 'FE'))
#print(t)
def func(x):
    f = Function('f')
    a = []
    for t in x.args:
        a.append(t[0])
    #print(a)
    return f(*a)

t = t.replace(lambda expr: isinstance(expr, Piecewise), func)
for _ in range(5):
    t = t + t**(t+3)

from timeit import default_timer as clock
from symengine import sympify
t1 = clock()
p = str(t)
t2 = clock()

sympy_time = t2-t1
print(t2-t1)

t1 = clock()
t = sympify(t)
p = str(t)
t2 = clock()

print(t2-t1)
symengine_time = t2-t1
print(sympy_time / symengine_time)
```

```python
import pycalphad
from pycalphad.tests.datasets import ALFE_TDB
from pycalphad import Database, Model
import pycalphad.variables as v
from sympy import Piecewise, Function

dbf = Database(ALFE_TDB)
mod = Model(dbf, ['AL','FE', 'VA'], 'B2_BCC')
t = mod.ast

def func(x):
    f = Function('f')
    a = []
    for t in x.args:
        a.append(t[0])
    return f(*a)

t = t.replace(lambda expr: isinstance(expr, Piecewise), func)

from timeit import default_timer as clock
from symengine import sympify, diff
t1 = clock()
p = str(t.diff(v.Y('B2_BCC', 1, 'AL'), v.Y('B2_BCC', 0, 'FE')))
t2 = clock()

print(t2-t1)
sympy_time = t2-t1

t1 = clock()
t = sympify(t)
p = str(t.diff(v.Y('B2_BCC', 1, 'AL')).diff(v.Y('B2_BCC', 0, 'FE')))
t2 = clock()

print(t2-t1)

symengine_time = t2-t1

print(sympy_time/symengine_time)
```

    2.1349420790002114
    0.11995693399967422
    17.797571243409816

```python

```


*This notebook is part of  course materials for CS 345: Machine Learning Foundations and Practice at Colorado State University.
Original versions were created by Asa Ben-Hur.
The content is availabe [on GitHub](https://github.com/asabenhur/CS345).*

*The text is released under the [CC BY-SA license](https://creativecommons.org/licenses/by-sa/4.0/), and code is released under the [MIT license](https://opensource.org/licenses/MIT).*

<a href="https://colab.research.google.com/github//asabenhur/CS345/blob/master/notebooks/module03_04_linear_regression_gradient_descent.ipynb">

</a>

```python
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
%matplotlib inline
%autosave 0
```

    Autosave disabled

# Solving linear regression using gradient descent

In this notebook we finally derive an algorithm for solving linear regression, i.e. finding parameters $\mathbf{w}$ that minimize

$$
J(\mathbf{w}) = \frac{1}{N}\sum_{i=1}^N(y_i-\hat{y}_i)^2 = \frac{1}{N}\sum_{i=1}^N(y_i - \mathbf{w}^\top \mathbf{x}_i)^2 .
$$

For simplicity, we are ignoring the bias term $b$.

Rather than finding the minimum of a function of a single variable our task is to find the minimum of a function of multiple variables.  A good way to think about the problem is in terms of a topographic map whose contours represent the values of the function we wish to optimize.  Here we show two views of the island of Hawaii:

Before getting to the algorithm of gradient descent, we need another concept, namely that of the gradient.

### The gradient

For a function of two variables, $f(x_1,x_2)$, the gradient is a two-dimensional vector consisting of the derivatives of the function with respect to its variables $x_1$ and $x_2$:

$$\nabla f(x_1,x_2) =
\begin{pmatrix}
\frac{\partial f(x_1,x_2)}{\partial x_1} \\
\frac{\partial f(x_1,x_2)}{\partial x_2}.
\end{pmatrix}$$

For a general multivariate function $f(x_1,\ldots,x_d)$ it is simply the vector of partial derivatives.

$$\nabla f(x_1,\ldots, x_d) =
\begin{pmatrix}
\frac{\partial f(x_1,\ldots,x_d)}{\partial x_1} \\
\vdots \\
\frac{\partial f(x_1,\ldots,x_d)}{\partial x_d}.
\end{pmatrix}$$

An alternative notation to $\nabla f$ is $\frac{\partial f}{\partial \mathbf{x}}$.

**Example**

For the function $f(x_1, x_2) = x_1^2 + x_2^2$ we have

$$\nabla f(x_1,x_2) =
\begin{pmatrix}
\frac{\partial f(x_1,x_2)}{\partial x_1} \\
\frac{\partial f(x_1,x_2)}{\partial x_1}
\end{pmatrix} =
\begin{pmatrix}2x_1 \\ 2x_2 \end{pmatrix} $$

The $\nabla$ symbol, is a shorthad for the vector of partial derivatives, which is also called the **gradient**.

The following figure shows contours of the function $f$ we just looked at, along with the direction of the gradient.  It should give us the intuition that the gradient points in the direction of steepest increase of the function.  Notice that the gradient is perpendicular to the contours of constant function value.
Also notice that the gradient tends to zero as we approach the minimum of the function at the origin.

```python
def f(x, y) :
    return x**2 + y**2

x = np.arange(-10, 10, 1)
y = np.arange(-10, 10, 1)

x_dense = np.linspace(-10, 10, 500)
y_dense = np.linspace(-10, 10, 500)

X, Y = np.meshgrid(x_dense, y_dense)
U, V = np.meshgrid(2*x, 2*y)

Z = f(X, Y)

plt.figure(figsize=(5,5))
C = plt.contour(x_dense, y_dense, Z, 10, colors='black')

q = plt.quiver(x, y, U, V)

im = plt.imshow(Z, origin='lower',
                cmap=plt.cm.Purples, extent=(-10,10,-10,10) )

```

### Practice with gradients

Let's practice with taking gradients of multivariate functions.
We'll start with the dot product.  We will take the gradient with respect to the parameters $\mathbf{w}$.

$$
f(\mathbf{w}) = \mathbf{w}^\top \mathbf{x} = \sum_{i=1}^d w_i x_i.
$$

We are interested in the dependence of this function on $\mathbf{w}$, because these are the parameters we are seeking to optimize.  The $\mathbf{x}$'s are constant in this case.  This is an easy case:

$$
\frac{\partial f}{\partial w_i} = x_i .
$$

We can thus conclude that

$$
\nabla_{\mathbf{w}} f(\mathbf{w}) = \mathbf{x} .
$$

Here we used the notation $\nabla_{\mathbf{w}}$ to denote that the gradient is with respect to $\mathbf{w}$.
In our alternative notation, we can write this as
$
\frac{\partial f}{\partial \mathbf{w}} = \mathbf{x} .
$

### The gradient for linear regression

The mean-squared-error is our cost function for linear regression:

$$
J(\mathbf{w}) = \frac{1}{N}\sum_{i=1}^N(y_i-\hat{y}_i)^2 = \frac{1}{N}\sum_{i=1}^N(y_i - \mathbf{w}^\top \mathbf{x}_i)^2 .
$$

For simplicity, we are ignoring the bias term $b$.  Before taking the gradient, we'll express it as:

$$
J(\mathbf{w}) = \frac{1}{N}\sum_{i=1}^N \left( y_i^2 - 2y_i \mathbf{w}^\top \mathbf{x}_i + (\mathbf{w}^\top \mathbf{x}_i)^2 \right) .
$$

Now for the gradient:

\begin{equation}
\begin{split}
\frac{\partial J(\mathbf{w})}{\partial \mathbf{w}} & =
\frac{1}{N}\sum_{i=1}^N \left( - 2y_i \mathbf{x}_i + 2(\mathbf{w}^\top \mathbf{x}_i)\mathbf{x}_i \right) \\
 & = -\frac{1}{N}\sum_{i=1}^N 2 \left( y_i - \mathbf{w}^\top \mathbf{x}_i\right) \mathbf{x}_i  .
\end{split}
\end{equation}

### Gradient descent

In the same way we were able to find the minimum of a univariate function using derivatives, we can find the minimum of a multi-dimensional surface by following the gradient.  This process is called **gradient descent**, and is described next.

Given a function $J(\mathbf{w})$, the gradient is the direction of steepest ascent.
Therefore to minimize $J(\mathbf{w})$, we iteratively take small steps in the direction of the negative of the gradient.

Gradient descent is an iterative algorithm as follows:

* Initialize $\mathbf{w}(0),\eta$
* for $t = 0,1,\ldots$
  * $\mathbf{w}(t+1) = \mathbf{w}(t) - \eta \frac{\partial J}{\partial \mathbf{w}}$
  * if converged, break

The halting condition for the algorithm can be specified in terms of the number of epochs like we did for the perceptron, or in terms of convergence of the loss function (the mean-squared-error).  In other words, if there is little change of the loss function across epochs.

### Questions:

* How would you need to modify the algorithm to find a local maximum?

* Can you guarantee the algorithm would find a global minimum/maximum?

* Find the gradient when we have a bias $b$.

### Linear regression using gradient descent

We are now ready to implement linear regression using gradient descent, and apply it to the advertising data.

```python
data = pd.read_csv('http://faculty.marshall.usc.edu/gareth-james/ISL/Advertising.csv', index_col=0)
X = data[['TV', 'radio']].values
y = data['sales'].values
X.shape, y.shape
```

    ((200, 2), (200,))

```python
class linear_regression :
    def __init__(self, learning_rate=0.00001,
                 bias_learning_rate=0.01, epochs=1000) :
        self.learning_rate = learning_rate
        self.bias_learning_rate = bias_learning_rate
        self.epochs = epochs

    def compute_loss(self, X, y):
        squared_errors = np.power((self.predict(X) - y), 2)
        return np.sum(squared_errors) / len(X)

    def predict(self, X) :
        return X @ self.w + self.b

    def fit(self, X, y):
        self.w = np.zeros( (X.shape[1]), np.float)
        self.b = 0.0
        # self.loss stores a history of the loss function values
        self.loss = []
        for i in range(self.epochs):
            y_pred = self.predict(X)
            # compute individual gradients and then average them:
            gradients_w = np.array([-2*(y[i]-y_pred[i])*X[i]
                                    for i in range(len(X))])
            gradient_w = np.mean(gradients_w, axis=0)
            # updating w with the gradient
            #print(gradient_w.shape)
            self.w = self.w - self.learning_rate * gradient_w
            gradient_b = - 2 * np.mean( (y - y_pred) )
            #gradients_b = [-2*(y[i]-y_pred[i]) for i in range(len(X))]
            #gradient_b = np.sum(gradients_b) / len(X)
            # updating b with the gradient:
            self.b = self.b - self.bias_learning_rate * gradient_b

            self.loss.append(self.compute_loss(X, y))

```

```python
linreg = linear_regression()
linreg.fit(X, y)
```

```python
plt.figure(figsize=(5,3))
plt.plot(linreg.loss[:50], 'b')
plt.xlabel('epochs')
plt.ylabel('loss');
```

Let's compare the learned parameters with the solution using the scikit-learn implementation:

```python
from sklearn.linear_model import LinearRegression

# instantiate and fit
linreg_sklearn = LinearRegression()
linreg_sklearn.fit(X, y)

# print the coefficients
print (linreg_sklearn.coef_, linreg_sklearn.intercept_)
print (linreg.w, linreg.b)
```

    [0.04575482 0.18799423] 2.9210999124051362
    [0.04552492 0.18527279] 3.030795322196597

```python

```