# Tutorial 2: Regularization techniques part 2
**Week 1, Day 5: Regularization**

**By Neuromatch Academy**

__Content creators:__ Ravi Teja Konkimalla, Mohitrajhu Lingan Kumaraian, Kevin Machado Gamboa, Kelson Shilling-Scrivo, Lyle Ungar

__Content reviewers:__ Piyush Chauhan, Siwei Bai, Kelson Shilling-Scrivo

__Content editors:__ Roberto Guidotti, Spiros Chavlis

__Production editors:__ Saeed Salehi, Spiros Chavlis

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives

1.   Regularization as shrinkage of overparameterized models: L1, L2
2.   Regularization by Dropout
3.   Regularization by Data Augmentation
4.   Perils of Hyper-Parameter Tuning
5.   Rethinking generalization

```python

# @title Tutorial slides

# @markdown These are the slides for the videos in this tutorial
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/7um6p/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

---
# Setup
Note that some of the code for today can take up to an hour to run. We have therefore "hidden" that code and shown the resulting outputs.

```python
# @title Install dependencies
#!apt-get install -y ffmpeg --quiet
!pip install imageio-ffmpeg --quiet

!pip install git+https://github.com/NeuromatchAcademy/evaltools --quiet
from evaltools.airtable import AirtableForm

atform = AirtableForm('appn7VdPRseSoMXEG','W1D5_T2', 'https://portal.neuromatchacademy.org/api/redirect/to/a76f99c1-9005-4566-8bcd-bed4e53d21f1')
```

```python
# Imports
from __future__ import print_function

import copy
import torch
import random
import pathlib

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

from torchvision import datasets, transforms
from torchvision.datasets import ImageFolder
from torch.optim.lr_scheduler import StepLR

from tqdm.auto import tqdm
from IPython.display import HTML, display
```

```python
# @title Figure Settings
import ipywidgets as widgets
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/content-creation/main/nma.mplstyle")
```

```python
# @title Loading Animal Faces data
import requests, os
from zipfile import ZipFile

print("Start downloading and unzipping `AnimalFaces` dataset...")
name = 'afhq'
fname = f"{name}.zip"
url = f"https://osf.io/kgfvj/download"

if not os.path.exists(fname):
  r = requests.get(url, allow_redirects=True)
  with open(fname, 'wb') as fh:
    fh.write(r.content)

  if os.path.exists(fname):
    with ZipFile(fname, 'r') as zfile:
      zfile.extractall(f".")
      os.remove(fname)

print("Download completed.")
```

    Start downloading and unzipping `AnimalFaces` dataset...
    Download completed.

```python
# @title Loading Animal Faces Randomized data
from IPython.display import clear_output

print("Start downloading and unzipping `Randomized AnimalFaces` dataset...")

names = ['afhq_random_32x32', 'afhq_10_32x32']
urls = ["https://osf.io/9sj7p/download",
        "https://osf.io/wvgkq/download"]

for i, name in enumerate(names):
  url = urls[i]
  fname = f"{name}.zip"

  if not os.path.exists(fname):
    r = requests.get(url, allow_redirects=True)
    with open(fname, 'wb') as fh:
      fh.write(r.content)

    if os.path.exists(fname):
      with ZipFile(fname, 'r') as zfile:
        zfile.extractall(f".")
        os.remove(fname)

print("Download completed.")
```

    Start downloading and unzipping `Randomized AnimalFaces` dataset...
    Download completed.

```python
# @title Plotting functions

def imshow(img):
  img = img / 2 + 0.5  # unnormalize
  npimg = img.numpy()
  plt.imshow(np.transpose(npimg, (1, 2, 0)))
  plt.axis(False)
  plt.show()

def plot_weights(norm, labels, ws, title='Weight Size Measurement'):
  plt.figure(figsize=[8, 6])
  plt.title(title)
  plt.ylabel('Frobenius Norm Value')
  plt.xlabel('Model Layers')
  plt.bar(labels, ws)
  plt.axhline(y=norm,
              linewidth=1,
              color='r',
              ls='--',
              label='Total Model F-Norm')
  plt.legend()
  plt.show()

def visualize_data(dataloader):

  for idx, (data,label) in enumerate(dataloader):
    plt.figure(idx)
    # Choose the datapoint you would like to visualize
    index = 22

    # choose that datapoint using index and permute the dimensions
    # and bring the pixel values between [0,1]
    data = data[index].permute(1, 2, 0) * \
           torch.tensor([0.5, 0.5, 0.5]) + \
           torch.tensor([0.5, 0.5, 0.5])

    # Convert the torch tensor into numpy
    data = data.numpy()

    plt.imshow(data)
    plt.axis(False)
    image_class = classes[label[index].item()]
    print(f'The image belongs to : {image_class}')

  plt.show()
```

```python
# @title Helper functions

## Network Class - Animal Faces
class AnimalNet(nn.Module):
  def __init__(self):
    super(AnimalNet, self).__init__()
    self.fc1 = nn.Linear(3 * 32 * 32, 128)
    self.fc2 = nn.Linear(128, 32)
    self.fc3 = nn.Linear(32, 3)

  def forward(self, x):
    x = x.view(x.shape[0], -1)
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    output = F.log_softmax(x, dim=1)
    return output

# Simple Net
class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()

    self.fc1 = nn.Linear(1, 300)
    self.fc2 = nn.Linear(300, 500)
    self.fc3 = nn.Linear(500, 1)

  def forward(self, x):
    x = F.leaky_relu(self.fc1(x))
    x = F.leaky_relu(self.fc2(x))
    output = self.fc3(x)
    return output

# Network Class - Animal Faces
class BigAnimalNet(nn.Module):
  def __init__(self):
    super(BigAnimalNet, self).__init__()
    self.fc1 = nn.Linear(3*32*32, 124)
    self.fc2 = nn.Linear(124, 64)
    self.fc3 = nn.Linear(64, 3)

  def forward(self, x):
    x = x.view(x.shape[0],-1)
    x = F.leaky_relu(self.fc1(x))
    x = F.leaky_relu(self.fc2(x))
    x = self.fc3(x)
    output = F.log_softmax(x, dim=1)
    return output

def train(args, model, train_loader, optimizer, epoch,
          reg_function1=None, reg_function2=None, criterion=F.nll_loss):
  """
  Trains the current inpur model using the data
  from Train_loader and Updates parameters for a single pass
  """

  device = args['device']
  model.train()
  for batch_idx, (data, target) in enumerate(train_loader):
    data, target = data.to(device), target.to(device)
    optimizer.zero_grad()
    output = model(data)
    # L1 regularization
    if reg_function2 is None and reg_function1 is not None:
      loss = criterion(output, target) + args['lambda1']*reg_function1(model)
    # L2 regularization
    elif reg_function1 is None and reg_function2 is not None:
      loss = criterion(output, target) + args['lambda2']*reg_function2(model)
    # No regularization
    elif reg_function1 is None and reg_function2 is None:
      loss = criterion(output, target)
    # Both L1 and L2 regularizations
    else:
      loss = criterion(output, target) + args['lambda1']*reg_function1(model) + args['lambda2']*reg_function2(model)
    loss.backward()
    optimizer.step()

  return model

def test(model, test_loader, loader='Test', criterion=F.nll_loss,
         device='cpu'):
  """
  Tests the current Model
  """
  model.eval()
  test_loss = 0
  correct = 0
  with torch.no_grad():
    for data, target in test_loader:
      data, target = data.to(device), target.to(device)
      output = model(data)
      test_loss += criterion(output, target, reduction='sum').item()  # sum up batch loss
      pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
      correct += pred.eq(target.view_as(pred)).sum().item()

  test_loss /= len(test_loader.dataset)
  def __init__(self):
    super(BigAnimalNet, self).__init__()
    self.fc1 = nn.Linear(3*32*32, 124)
    self.fc2 = nn.Linear(124, 64)
    self.fc3 = nn.Linear(64, 3)

  def forward(self, x):
    x = x.view(x.shape[0],-1)
    x = F.leaky_relu(self.fc1(x))
    x = F.leaky_relu(self.fc2(x))
    x = self.fc3(x)
    output = F.log_softmax(x, dim=1)
    return output
  return 100. * correct / len(test_loader.dataset)

def main(args, model, train_loader, val_loader, test_data,
         reg_function1=None, reg_function2=None, criterion=F.nll_loss):
  """
  Trains the model with train_loader and tests the learned model using val_loader
  """

  device = args['device']

  model = model.to(device)
  optimizer = optim.SGD(model.parameters(), lr=args['lr'], momentum=args['momentum'])

  val_acc_list, train_acc_list,param_norm_list = [], [], []
  for epoch in tqdm(range(args['epochs'])):
    trained_model = train(args, model, train_loader, optimizer, epoch,
                          reg_function1=reg_function1,
                          reg_function2=reg_function2)
    train_acc = test(trained_model, train_loader, loader='Train', device=device)
    val_acc = test(trained_model, val_loader, loader='Val', device=device)
    param_norm = calculate_frobenius_norm(trained_model)
    train_acc_list.append(train_acc)
    val_acc_list.append(val_acc)
    param_norm_list.append(param_norm)

  return val_acc_list, train_acc_list, param_norm_list, model

def calculate_frobenius_norm(model):
    norm = 0.0
    # Sum the square of all parameters
    for name,param in model.named_parameters():
        norm += torch.norm(param).data**2
    # Return a square root of the sum of squares of all the parameters
    return norm**0.5

def early_stopping_main(args, model, train_loader, val_loader, test_data):

  device = args['device']

  model = model.to(device)
  optimizer = optim.SGD(model.parameters(), lr=args['lr'], momentum=args['momentum'])

  best_acc  = 0.0
  best_epoch = 0

  # Number of successive epochs that you want to wait before stopping training process
  patience = 20

  # Keps track of number of epochs during which the val_acc was less than best_acc
  wait = 0

  val_acc_list, train_acc_list = [], []
  for epoch in tqdm(range(args['epochs'])):
    trained_model = train(args, model, device, train_loader, optimizer, epoch)
    train_acc = test(trained_model, train_loader, loader='Train', device=device)
    val_acc = test(trained_model, val_loader, loader='Val', device=device)
    if (val_acc > best_acc):
      best_acc = val_acc
      best_epoch = epoch
      best_model = copy.deepcopy(trained_model)
      wait = 0
    else:
      wait += 1
    if (wait > patience):
      print(f'early stopped on epoch: {epoch}')
      break
    train_acc_list.append(train_acc)
    val_acc_list.append(val_acc)

  return val_acc_list, train_acc_list, best_model, best_epoch
```

```python
# @title Set random seed
# @markdown Executing `set_seed(seed=seed)` you are setting the seed

# for DL its critical to set the random seed so that students can have a
# baseline to compare their results to expected results.
# Read more here: https://pytorch.org/docs/stable/notes/randomness.html

# Call `set_seed` function in the exercises to ensure reproducibility.
import random
import torch

def set_seed(seed=None, seed_torch=True):
  if seed is None:
    seed = np.random.choice(2 ** 32)
  random.seed(seed)
  np.random.seed(seed)
  if seed_torch:
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

  print(f'Random seed {seed} has been set.')

# In case that `DataLoader` is used
def seed_worker(worker_id):
  worker_seed = torch.initial_seed() % 2**32
  np.random.seed(worker_seed)
  random.seed(worker_seed)
```

```python
# @title Set device (GPU or CPU). Execute `set_device()`
# especially if torch modules used.

# inform the user if the notebook uses GPU or CPU.

def set_device():
  device = "cuda" if torch.cuda.is_available() else "cpu"
  if device != "cuda":
    print("WARNING: For this notebook to perform best, "
        "if possible, in the menu under `Runtime` -> "
        "`Change runtime type.`  select `GPU` ")
  else:
    print("GPU is enabled in this notebook.")

  return device
```

```python
SEED = 2021
set_seed(seed=SEED)
DEVICE = set_device()
```

    Random seed 2021 has been set.
    GPU is enabled in this notebook.

```python
# @title Dataloaders for the Dataset
## Dataloaders for the Dataset
batch_size = 128
classes = ('cat', 'dog', 'wild')

train_transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
     ])
data_path = pathlib.Path('.')/'afhq' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)

####################################################
g_seed = torch.Generator()
g_seed.manual_seed(SEED)

##Dataloaders for the  Original Dataset
img_train_data, img_val_data,_ = torch.utils.data.random_split(img_dataset,
                                                               [100, 100, 14430])

#Creating train_loader and Val_loader
train_loader = torch.utils.data.DataLoader(img_train_data,
                                           batch_size=batch_size,
                                           worker_init_fn=seed_worker,
                                           num_workers=2,
                                           generator=g_seed)
val_loader = torch.utils.data.DataLoader(img_val_data,
                                         batch_size=1000,
                                         num_workers=2,
                                         worker_init_fn=seed_worker,
                                         generator=g_seed)

#creating test dataset
test_transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
     ])
img_test_dataset = ImageFolder(data_path/'val', transform=test_transform)

####################################################

## Dataloaders for the  Random Dataset

# splitting randomized data into training and validation data
data_path = pathlib.Path('.')/'afhq_random_32x32/afhq_random' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)
random_img_train_data, random_img_val_data,_ = torch.utils.data.random_split(img_dataset, [100,100,14430])

#Randomized train and validation dataloader
rand_train_loader = torch.utils.data.DataLoader(random_img_train_data,
                                                batch_size=batch_size,
                                                num_workers=2,
                                                worker_init_fn=seed_worker,
                                                generator=g_seed)
rand_val_loader = torch.utils.data.DataLoader(random_img_val_data,
                                              batch_size=1000,
                                              num_workers=2,
                                              worker_init_fn=seed_worker,
                                              generator=g_seed)

####################################################

## Dataloaders for the Partially Random Dataset

# Splitting data between training and validation dataset for partially randomized data
data_path = pathlib.Path('.')/'afhq_10_32x32/afhq_10' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)
partially_random_train_data, partially_random_val_data, _ = torch.utils.data.random_split(img_dataset, [100,100,14430])

#Training and Validation loader for partially randomized data
partial_rand_train_loader = torch.utils.data.DataLoader(partially_random_train_data,
                                                        batch_size=batch_size,
                                                        num_workers=2,
                                                        worker_init_fn=seed_worker,
                                                        generator=g_seed)
partial_rand_val_loader = torch.utils.data.DataLoader(partially_random_val_data,
                                                      batch_size=1000,
                                                      num_workers=2,
                                                      worker_init_fn=seed_worker,
                                                      generator=g_seed)
```

---
# Section 1: L1 and L2 Regularization

*Time estimate: ~30 mins*

```python
# @title Video 1: L1 and L2 regression
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV19h41167H7", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"oQNdloKdysM", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 1: L1 and L2 regression')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

Some of you might have already come across L1 and L2 regularization before in other courses. L1 and L2 are the most common types of regularization. These update the general cost function by adding another term known as the regularization term.

***Cost function = Loss (say, binary cross entropy) + Regularization term***

This regularization term makes the parameters smaller, giving simpler models that will overfit less.

Discuss among your teammates whether the above assumption is good or bad?

## Section 1.1: Unregularized Model

```python
# @markdown #### Dataloaders for Regularization
data_path = pathlib.Path('.')/'afhq' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)

# Splitting dataset
reg_train_data, reg_val_data,_ = torch.utils.data.random_split(img_dataset,
                                                               [30, 100, 14500])
g_seed = torch.Generator()
g_seed.manual_seed(SEED)

# Creating train_loader and Val_loader
reg_train_loader = torch.utils.data.DataLoader(reg_train_data,
                                               batch_size=batch_size,
                                               worker_init_fn=seed_worker,
                                               num_workers=2,
                                               generator=g_seed)
reg_val_loader = torch.utils.data.DataLoader(reg_val_data,
                                             batch_size=1000,
                                             worker_init_fn=seed_worker,
                                             num_workers=2,
                                             generator=g_seed)
```

Now let's train a model without any regularization and keep it aside as our benchmark for this section.

```python
# Set the arguments
args = {
    'epochs': 150,
    'lr': 5e-3,
    'momentum': 0.99,
    'device': DEVICE,
}

# intialize the model
set_seed(seed=SEED)
model = AnimalNet()

# Train the model
val_acc_unreg, train_acc_unreg, param_norm_unreg, _ = main(args,
                                                           model,
                                                           reg_train_loader,
                                                           reg_val_loader,
                                                           img_test_dataset)

# Train and Test accuracy plot
plt.figure()
plt.plot(val_acc_unreg, label='Val Accuracy', c='red', ls='dashed')
plt.plot(train_acc_unreg, label='Train Accuracy', c='red', ls='solid')
plt.axhline(y=max(val_acc_unreg), c='green', ls='dashed')
plt.title('Unregularized Model')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
print(f"maximum Validation Accuracy reached: {max(val_acc_unreg)}")
```

## Section 1.2: L1 Regularization

L1 (or "LASSO") Regularization uses a penalty which is the sum of the absolute value of all the weights in the DLN, resulting in the following loss function ($L$ is the usual Cross Entropy loss):

\begin{equation}
L_R=L + \lambda \sum \left| w^{(r)}_{ij} \right|
\end{equation}

where $r$ denotes the layer, and $ij$ the specific weight in that layer.

At a high level, L1 Regularization is similar to L2 Regularization since it leads to smaller weights (you will see the analogy in the next subsection). It results in the following weight update equation when using Stochastic Gradient Descent:

\begin{equation}
w^{(r)}_{ij}←w^{(r)}_{ij}−\eta \lambda \text{sgn}\left(w^{(r)}_{ij}\right)−\eta \frac{\partial L}{\partial w_{ij}^{(r)}}
\end{equation}

where  $\text{sgn}(\cdot)$ is the sign function, such that

\begin{equation}
\text{sgn}(w) =
\left\{
  \begin{array}{ll}
  +1 & \mbox{if  } w > 0 \\
  -1 & \mbox{if  } w < 0 \\
  0 & \mbox{if  } w = 0
  \end{array}
\right.
\end{equation}

### Coding Exercise 1.1: L1 Regularization

Write a function which calculates the L1 norm of all the tensors of a Pytorch model.

```python
def l1_reg(model):
  """
    Inputs: Pytorch model
    This function calculates the l1 norm of the all the tensors in the model
  """
  l1 = 0.0
  ####################################################################
  # Fill in all missing code below (...),
  # then remove or comment the line below to test your function
  #raise NotImplementedError("Complete the l1_reg function")
  ####################################################################
  for param in model.parameters():
    l1 += torch.sum(torch.abs(param.data))

  return l1

# add event to airtable
atform.add_event('Coding Exercise 1.1: L1 Regularization')

set_seed(seed=SEED)
## uncomment to test
net = nn.Linear(20, 20)
print(f"L1 norm of the model: {l1_reg(net)}")
```

    Random seed 2021 has been set.
    L1 norm of the model: 48.445133209228516

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content-dl/tree/main//tutorials/W1D5_Regularization/solutions/W1D5_Tutorial2_Solution_f9f318de.py)

```
Random seed 2021 has been set.
L1 norm of the model: 48.445133209228516
```

Now, let's train a classifier which uses L1 regularization. Tune the hyperparameter `lambda1` such that the validation accuracy is higher than that of the unregularized model.

```python
# Set the arguments
args1 = {
    'test_batch_size': 1000,
    'epochs': 150,
    'lr': 5e-3,
    'momentum': 0.99,
    'device': DEVICE,
    'lambda1': 0.001  # <<<<<<<< Tune the hyperparameter lambda
}

# intialize the model
set_seed(seed=SEED)
model = AnimalNet()

# Train the model
val_acc_l1reg, train_acc_l1reg, param_norm_l1reg, _ = main(args1,
                                                           model,
                                                           reg_train_loader,
                                                           reg_val_loader,
                                                           img_test_dataset,
                                                           reg_function1=l1_reg)

# Train and Test accuracy plot
plt.figure()
plt.plot(val_acc_l1reg, label='Val Accuracy L1 Regularized',
         c='red', ls='dashed')
plt.plot(train_acc_l1reg, label='Train Accuracy L1 regularized',
         c='red', ls='solid')
plt.axhline(y=max(val_acc_l1reg), c='green', ls='dashed')
plt.title('L1 regularized model')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
print(f"maximum Validation Accuracy reached: {max(val_acc_l1reg)}")
```

What value of `lambda1` hyperparameter, worked for L1 Regularization? (Note that the $\lambda$ in the equations is the `lambda1` in the code for clarity)

## Section 1.3: L2 / Ridge Regularization

L2 Regularization, sometimes referred to as “Weight Decay”, is widely used. It works by adding a quadratic penalty term to the Cross Entropy Loss Function $L$, which results in a new Loss Function $L_R$  given by:

\begin{equation}
L_R = L + \lambda \sum \left( w^{(r)}_{ij} \right)^2
\end{equation}

where, again, $r$ denotes the layer, and $ij$ the specific weight in that layer.

In order to get further insight into L2 Regularization, we investigate its effect on the Gradient Descent based update equations for the weight and bias parameters. Taking the derivative on both sides of the above equation, we obtain

\begin{equation}
\frac{\partial L_R}{\partial w^{(r)}_{ij}}=\frac{\partial L}{\partial w^{(r)}_{ij}} + 2\lambda w^{(r)}_{ij}
\end{equation}

Thus the weight update rule becomes:

\begin{equation}
w^{(r)}_{ij}←w^{(r)}_{ij}−η\frac{\partial L}{\partial w^{(r)}_{ij}}−2 \eta \lambda w^{(r)}_{ij}=(1−2 \eta \lambda)w^{(r)}_{ij} − \eta \frac{\partial L}{\partial w^{(r)}_{ij}}
\end{equation}

where, $\eta$ is learning rate.

### Coding Exercise 1.2: L2 Regularization

Write a function which calculates the L2 norm of all the tensors of a Pytorch model. (What did we call this before?)

```python
def l2_reg(model):

  """
    Inputs: Pytorch model
    This function calculates the l2 norm of the all the tensors in the model
  """

  l2 = 0.0
  ####################################################################
  # Fill in all missing code below (...),
  # then remove or comment the line below to test your function
  #raise NotImplementedError("Complete the l2_reg function")
  ####################################################################
  for param in model.parameters():
        l2 += torch.sum(torch.abs(param.data)**2)

  return l2

# add event to airtable
atform.add_event('Coding Exercise 1.2: L2 Regularization')

set_seed(SEED)
## uncomment to test
net = nn.Linear(20, 20)
print(f"L2 norm of the model: {l2_reg(net)}")
```

    Random seed 2021 has been set.
    L2 norm of the model: 7.328375816345215

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content-dl/tree/main//tutorials/W1D5_Regularization/solutions/W1D5_Tutorial2_Solution_8505e24d.py)

```
Random seed 2021 has been set.
L2 norm of the model: 7.328375816345215
```

Now we'll train a classifier which uses L2 regularization. Tune the hyperparameter `lambda` such that the val accuracy is higher than that of the unregularized model.

```python
# Set the arguments
args2 = {
    'test_batch_size': 1000,
    'epochs': 150,
    'lr': 5e-3,
    'momentum': 0.99,
    'device': DEVICE,
    'lambda2': 0.001  # <<<<<<<< Tune the hyperparameter lambda
}

# intialize the model
set_seed(seed=SEED)
model = AnimalNet()

# Train the model
val_acc_l2reg, train_acc_l2reg, param_norm_l2reg, model = main(args2,
                                                               model,
                                                               train_loader,
                                                               val_loader,
                                                               img_test_dataset,
                                                               reg_function2=l2_reg)

## Train and Test accuracy plot
plt.figure()
plt.plot(val_acc_l2reg, label='Val Accuracy L2 regularized',
         c='red', ls='dashed')
plt.plot(train_acc_l2reg, label='Train Accuracy L2 regularized',
         c='red', ls='solid')
plt.axhline(y=max(val_acc_l2reg), c='green', ls='dashed')
plt.title('L2 Regularized Model')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
print(f"maximum Validation Accuracy reached: {max(val_acc_l2reg)}")
```

What value `lambda2` worked for L2 Regularization? (Note that the $\lambda$ in the equations is the `lambda2` in the code for clarity)

Now, let's run a model with both L1 and L2 regularization terms.

```python
# @markdown Visualize all of them together (Run Me!)

# @markdown `lambda1=0.001` and `lambda2=0.001`

args3 = {
    'test_batch_size': 1000,
    'epochs': 150,
    'lr': 5e-3,
    'momentum': 0.99,
    'device': DEVICE,
    'lambda1': 0.001,
    'lambda2': 0.001
}

# Intialize the model
set_seed(seed=SEED)
model = AnimalNet()
val_acc_l1l2reg, train_acc_l1l2reg, param_norm_l1l2reg, _ = main(args3,
                                                                 model,
                                                                 train_loader,
                                                                 val_loader,
                                                                 img_test_dataset,
                                                                 reg_function1=l1_reg,
                                                                 reg_function2=l2_reg)

plt.figure()

plt.plot(val_acc_l2reg, c='red', ls='dashed')
plt.plot(train_acc_l2reg,
         label=f"L2 regularized, $\lambda_2$={args2['lambda2']}",
         c='red', ls='solid')
plt.axhline(y=max(val_acc_l2reg), c='red', ls='dashed')

plt.plot(val_acc_l1reg, c='green', ls = 'dashed')
plt.plot(train_acc_l1reg,
         label=f"L1 regularized, $\lambda_1$={args1['lambda1']}",
         c='green', ls='solid')
plt.axhline(y=max(val_acc_l1reg), c='green', ls='dashed')

plt.plot(val_acc_unreg, c='blue', ls = 'dashed')
plt.plot(train_acc_unreg,
         label='Unregularized', c='blue', ls='solid')
plt.axhline(y=max(val_acc_unreg), c='blue', ls='dashed')

plt.plot(val_acc_l1l2reg, c='orange', ls='dashed')
plt.plot(train_acc_l1l2reg,
         label=f"L1+L2 regularized, $\lambda_1$={args3['lambda1']}, $\lambda_2$={args3['lambda2']}",
         c='orange', ls='solid')
plt.axhline(y=max(val_acc_l1l2reg), c='orange', ls = 'dashed')

plt.xlabel('epoch')
plt.ylabel('Accuracy (%)')
plt.legend()
plt.show()
```

Now, let's visualize what these different regularization does to the parameters of the model. We observe the effect by computing the size (technically, the Frobenius norm) of the model parameters

```python
# @markdown #### Visualize Norm of the Models (Train Me!)
plt.figure()
plt.plot(param_norm_unreg, label='Unregularized', c='blue')
plt.plot(param_norm_l1reg, label='L1 Regularized', c='green')
plt.plot(param_norm_l2reg, label='L2 Regularized', c='red')
plt.plot(param_norm_l1l2reg, label='L1+L2 Regularized', c='orange')
plt.xlabel('epoch')
plt.ylabel('Parameter Norms')
plt.legend()
plt.show()
```

In the above plots, you should have seen that even after the model achieves 100% train accuracy the val accuracies are fluctuating. This suggests that the model is still trying to learn something. Why would this be the case?

---
# Section 2: Dropout

*Time estimate: ~25 mins*

```python
# @title Video 2: Dropout
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1gU4y1G7V2", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"UZfUzawej3A", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 2: Dropout')

display(out)
```

In dropout, we literally drop out (zero out) some neurons during training. Throughout training, on each iteration, standard dropout zeros out some fraction (usually 1/2) of the nodes in each layer before calculating the subsequent layer. Randomly selecting different subsets to dropout introduces noise into the process and reduces overfitting.

Now let's revisit the toy dataset that we generated above to visualize how the dropout stabilizes training on a noisy dataset. We will slightly modify the architecture we used above to add dropout layers.

```python
# Network Class - 2D
class NetDropout(nn.Module):
  def __init__(self):
    super(NetDropout, self).__init__()

    self.fc1 = nn.Linear(1, 300)
    self.fc2 = nn.Linear(300, 500)
    self.fc3 = nn.Linear(500, 1)
    # We add two dropout layers
    self.dropout1 = nn.Dropout(0.4)
    self.dropout2 = nn.Dropout(0.2)

  def forward(self, x):
    x = F.leaky_relu(self.dropout1(self.fc1(x)))
    x = F.leaky_relu(self.dropout2(self.fc2(x)))
    output = self.fc3(x)
    return output
```

```python
# @markdown #### Run to train the default network
set_seed(seed=SEED)
# creating train data
X = torch.rand((10, 1))
X.sort(dim = 0)
Y = 2*X + 2*torch.empty((X.shape[0], 1)).normal_(mean=0, std=1)  # adding small error in the data

X = X.unsqueeze_(1)
Y = Y.unsqueeze_(1)

# creating test dataset
X_test = torch.linspace(0, 1, 40)
X_test = X_test.reshape((40, 1, 1))

# train the network on toy dataset
model = Net()
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=1e-4)
max_epochs = 10000
iters = 0

running_predictions = np.empty((40, (int)(max_epochs/500 + 1)))

train_loss = []
test_loss = []
model_norm = []

for epoch in tqdm(range(max_epochs)):

  #training
  model_norm.append(calculate_frobenius_norm(model))
  model.train()
  optimizer.zero_grad()
  predictions = model(X)
  loss = criterion(predictions,Y)
  loss.backward()
  optimizer.step()

  train_loss.append(loss.data)
  model.eval()
  Y_test = model(X_test)
  loss = criterion(Y_test, 2*X_test)
  test_loss.append(loss.data)

  if (epoch % 500 == 0 or epoch == max_epochs - 1):
    running_predictions[:, iters] = Y_test[:, 0, 0].detach().numpy()
    iters += 1
```

```python
# train the network on toy dataset

# Intialize the model
set_seed(seed=SEED)
model = NetDropout()
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=1e-4)
max_epochs = 10000
iters = 0

running_predictions_dp = np.empty((40, (int)(max_epochs / 500)))

train_loss_dp = []
test_loss_dp = []
model_norm_dp = []

for epoch in tqdm(range(max_epochs)):

  # training
  model_norm_dp.append(calculate_frobenius_norm(model))
  model.train()
  optimizer.zero_grad()
  predictions = model(X)
  loss = criterion(predictions, Y)
  loss.backward()
  optimizer.step()

  train_loss_dp.append(loss.data)
  model.eval()
  Y_test = model(X_test)
  loss = criterion(Y_test, 2*X_test)
  test_loss_dp.append(loss.data)

  if (epoch % 500 == 0 or epoch == max_epochs):
    running_predictions_dp[:, iters] = Y_test[:, 0, 0].detach().numpy()
    iters += 1
```

Now that we have finished training, let's see how the model has evolved over the training process.

```python
# @markdown Animation! (Run Me!)
set_seed(seed=SEED)

fig = plt.figure(figsize=(8, 6))
ax = plt.axes()
def frame(i):
  ax.clear()
  ax.scatter(X[:, 0, :].numpy(), Y[:, 0, :].numpy())
  plot = ax.plot(X_test[:, 0, :].detach().numpy(),
                 running_predictions_dp[:, i])
  title = f"Epoch: {i*500}"
  plt.title(title)
  ax.set_xlabel("X axis")
  ax.set_ylabel("Y axis")
  return plot

anim = animation.FuncAnimation(fig, frame, frames=range(20),
                               blit=False, repeat=False,
                               repeat_delay=10000)
html_anim = HTML(anim.to_html5_video());
plt.close()
display(html_anim)
```

```python
# @markdown Plot the train and test losses
plt.figure()
plt.plot(test_loss_dp, label='test loss dropout', c='blue', ls='dashed')
plt.plot(test_loss, label='test loss', c='red', ls='dashed')
plt.ylabel('loss')
plt.xlabel('epochs')
plt.title('dropout vs without dropout')
plt.legend()
plt.show()
```

```python
# @markdown Plot the train and test losses
plt.figure()
plt.plot(train_loss_dp, label='train loss dropout', c='blue', ls='dashed')
plt.plot(train_loss, label='train loss', c='red', ls='dashed')
plt.ylabel('loss')
plt.xlabel('epochs')
plt.title('dropout vs without dropout')
plt.legend()
plt.show()
```

```python
# @markdown Plot model weights with epoch
plt.figure()
plt.plot(model_norm_dp, label='dropout')
plt.plot(model_norm, label='no dropout')
plt.ylabel('norm of the model')
plt.xlabel('epochs')
plt.legend()
plt.title('Size of the model vs Epochs')
plt.show()
```

Do you think this performed better than the initial model?

## Section 2.1: Dropout Implementation Caveats

*  Dropout is used only during training, during testing the complete model weights are used and hence it is important to use model.eval() before testing the model.

* Dropout reduces the capacity of the model during training and hence as a general practice wider networks are used when using dropout. If you are using a dropout with a random probability of 0.5 then you might want to double the number of hidden neurons in that layer.

Now, let's see how dropout fares on the Animal Faces Dataset. We first modify the existing model to include dropout and then train the model.

```python
# Network Class - Animal Faces
class AnimalNetDropout(nn.Module):
  def __init__(self):
    super(AnimalNetDropout, self).__init__()
    self.fc1 = nn.Linear(3*32*32, 248)
    self.fc2 = nn.Linear(248, 210)
    self.fc3 = nn.Linear(210, 3)
    self.dropout1 = nn.Dropout(p=0.5)
    self.dropout2 = nn.Dropout(p=0.3)

  def forward(self, x):
    x = x.view(x.shape[0], -1)
    x = F.leaky_relu(self.dropout1(self.fc1(x)))
    x = F.leaky_relu(self.dropout2(self.fc2(x)))
    x = self.fc3(x)
    output = F.log_softmax(x, dim=1)
    return output
```

```python
# Set the arguments
args = {
    'test_batch_size': 1000,
    'epochs': 200,
    'lr': 5e-3,
    'batch_size': 32,
    'momentum': 0.9,
    'device': DEVICE,
    'log_interval': 100
}

# intialize the model
set_seed(seed=SEED)
model = AnimalNetDropout()

# Train the model with Dropout
val_acc_dropout, train_acc_dropout, _, model_dp = main(args,
                                                       model,
                                                       train_loader,
                                                       val_loader,
                                                       img_test_dataset)

# intialize the BigAnimalNet model
set_seed(seed=SEED)
model = BigAnimalNet()

# Train the model
val_acc_big, train_acc_big, _, model_big = main(args,
                                                model,
                                                train_loader,
                                                val_loader,
                                                img_test_dataset)

# Train and Test accuracy plot
plt.figure()
plt.plot(val_acc_big, label='Val - Big', c='blue', ls='dashed')
plt.plot(train_acc_big, label='Train - Big', c='blue', ls='solid')
plt.plot(val_acc_dropout, label='Val - DP', c='magenta', ls='dashed')
plt.plot(train_acc_dropout, label='Train - DP', c='magenta', ls='solid')
plt.title('Dropout')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
```

When do you think dropouts can perform bad and do you think their placement within a model matters?

---
# Section 3: Data Augmentation

*Time estimate: ~15 mins*

```python
# @title Video 3: Data Augmentation
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1Xw411d7Pz", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"nm44FhjL3xc", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 3: Data Augmentation')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

Data augmentation is often used to increase the number of training samples. Now we will explore the effects of data augmentation on regularization. Here regularization is achieved by adding noise into training data after every epoch.

Pytorch's torchvision module provides a few built-in data augmentation techniques, which we can use on image datasets. Some of the techniques we most frequently use are:

*   Random Crop
*   Random Rotate
*   Vertical Flip
*   Horizontal Flip

```python
# @markdown ####  Data Loader without Data Augmentation

# For reproducibility
g_seed = torch.Generator()
g_seed.manual_seed(SEED)

train_transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
     ])
data_path = pathlib.Path('.')/'afhq' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)

# Splitting dataset
img_train_data, img_val_data,_ = torch.utils.data.random_split(img_dataset, [250,100,14280])

# Creating train_loader and Val_loader
train_loader = torch.utils.data.DataLoader(img_train_data,
                                           batch_size=batch_size,
                                           num_workers=2,
                                           worker_init_fn=seed_worker,
                                           generator=g_seed)
val_loader = torch.utils.data.DataLoader(img_val_data,
                                         batch_size=1000,
                                         num_workers=2,
                                         worker_init_fn=seed_worker,
                                         generator=g_seed)
```

Define a DataLoader using [torchvision.transforms](https://pytorch.org/docs/stable/torchvision/transforms.html) which randomly augments the data for us.

```python
# Data Augmentation using transforms
new_transforms = transforms.Compose([
                                     transforms.RandomHorizontalFlip(p=0.1),
                                     transforms.RandomVerticalFlip(p=0.1),
                                     transforms.ToTensor(),
                                     transforms.Normalize((0.5, 0.5, 0.5),
                                                          (0.5, 0.5, 0.5))
                                     ])

data_path = pathlib.Path('.')/'afhq'  # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=new_transforms)
# Splitting dataset
new_train_data, _,_ = torch.utils.data.random_split(img_dataset,
                                                    [250, 100, 14280])

# For reproducibility
g_seed = torch.Generator()
g_seed.manual_seed(SEED)

# Creating train_loader and Val_loader
new_train_loader = torch.utils.data.DataLoader(new_train_data,
                                               batch_size=batch_size,
                                               worker_init_fn=seed_worker,
                                               generator=g_seed)
```

```python
# Set the arguments
args = {
    'epochs': 250,
    'lr': 1e-3,
    'momentum': 0.99,
    'device': DEVICE,
}

# Intialize the model
set_seed(seed=SEED)
model_aug = AnimalNet()

# train the model
val_acc_dataaug, train_acc_dataaug, param_norm_datadug, _ = main(args,
                                                                 model_aug,
                                                                 new_train_loader,
                                                                 val_loader,
                                                                 img_test_dataset)
# Intialize the model
set_seed(seed=SEED)
model_pure = AnimalNet()

val_acc_pure, train_acc_pure, param_norm_pure, _, = main(args,
                                                         model_pure,
                                                         train_loader,
                                                         val_loader,
                                                         img_test_dataset)

# Train and Test accuracy plot
plt.figure()
plt.plot(val_acc_pure, label='Val Accuracy Pure',
         c='red', ls='dashed')
plt.plot(train_acc_pure, label='Train Accuracy Pure',
         c='red', ls='solid')
plt.plot(val_acc_dataaug, label='Val Accuracy data augment',
         c='blue', ls='dashed')
plt.plot(train_acc_dataaug, label='Train Accuracy data augment',
         c='blue', ls='solid')
plt.axhline(y=max(val_acc_pure), c='red', ls='dashed')
plt.axhline(y=max(val_acc_dataaug), c='blue', ls='dashed')
plt.title('Data Augmentation')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
```

```python
# Plot together: without and with augmenetation
plt.figure()
plt.plot(param_norm_pure, c='red', label='Without Augmentation')
plt.plot(param_norm_datadug, c='blue', label='With Augmentation')
plt.title('Norm of parameters as a function of training epoch')
plt.xlabel('epoch')
plt.ylabel('Norm of model parameters')
plt.legend()
plt.show()
```

Can you think of more ways of augmenting training data? (Think of other problems beyond object recogition.)

### Think! 3.1: Thought Question
Why does it work better to regularize an overparameterized ANN than to start with a smaller one? Think about  the regularization  methods you know.
Each group has a 10 min discussion.

```python
# @title Student Response
from ipywidgets import widgets

text=widgets.Textarea(
   value='Type your answer here and click on `Submit!`',
   placeholder='Type something',
   description='',
   disabled=False
)

button = widgets.Button(description="Submit!")

display(text,button)

def on_button_clicked(b):
   atform.add_answer('q1', text.value)
   print("Submission successful!")

button.on_click(on_button_clicked)
```

    Textarea(value='Type your answer here and click on `Submit!`', placeholder='Type something')

    Button(description='Submit!', style=ButtonStyle())

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content-dl/tree/main//tutorials/W1D5_Regularization/solutions/W1D5_Tutorial2_Solution_519e352b.py)

---
# Section 4: Stochastic Gradient Descent

*Time estimate: ~20 mins*

```python
# @title Video 4: SGD
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1nM4y1K7wP", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"rjzlFvJhNqE", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 4: SGD')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

## Section 4.1: Learning Rate
In this section, we will see how learning rate can act as regularizer while training a neural network. In summary:

*   Smaller learning rates regularize less. They slowly converge to deep minima.
*   Larger learning rates regularizes more by missing local minima and converging to broader, flatter minima, which often generalize better.

But beware, a very large learning rate may result in overshooting or finding a really bad local minimum.

In the block below, we will train the Animal Net model with different learning rates and see how that affects the regularization.

```python
# @markdown #### Generating Data Loaders

# For reproducibility
g_seed = torch.Generator()
g_seed.manual_seed(SEED)

batch_size = 128
train_transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
     ])

data_path = pathlib.Path('.')/'afhq' # using pathlib to be compatible with all OS's
img_dataset = ImageFolder(data_path/'train', transform=train_transform)
img_train_data, img_val_data, = torch.utils.data.random_split(img_dataset, [11700,2930])

full_train_loader = torch.utils.data.DataLoader(img_train_data,
                                                batch_size=batch_size,
                                                num_workers=2,
                                                worker_init_fn=seed_worker,
                                                generator=g_seed)
full_val_loader = torch.utils.data.DataLoader(img_val_data,
                                              batch_size=1000,
                                              num_workers=2,
                                              worker_init_fn=seed_worker,
                                              generator=g_seed)

test_transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))    # [TO-DO]
     ])
img_test_dataset = ImageFolder(data_path/'val', transform=test_transform)
# img_test_loader = DataLoader(img_test_dataset, batch_size=batch_size,shuffle=False, num_workers=1)
classes = ('cat', 'dog', 'wild')
```

```python
# Set the arguments
args = {
    'test_batch_size': 1000,
    'epochs': 350,
    'batch_size': 32,
    'momentum': 0.99,
    'device': DEVICE
}

learning_rates = [5e-4, 1e-3, 5e-3]
acc_dict = {}

for i, lr in enumerate(learning_rates):
  # Initialize the model
  set_seed(seed=SEED)
  model = AnimalNet()
  # Learning rate
  args['lr'] = lr
  # Train the model
  val_acc, train_acc, param_norm, _ = main(args,
                                           model,
                                           train_loader,
                                           val_loader,
                                           img_test_dataset)
  # store the outputs
  acc_dict[f'val_{i}'] = val_acc
  acc_dict[f'train_{i}'] = train_acc
  acc_dict[f'param_norm_{i}'] = param_norm
```

```python
# @markdown Plot Train and Validation accuracy (Run me)

plt.figure()
for i, lr in enumerate(learning_rates):
  plt.plot(acc_dict[f'val_{i}'], linestyle='dashed',
          label=f'lr={lr:0.1e} - validation')
  plt.plot(acc_dict[f'train_{i}'], label=f'{lr:0.1e} - train')

  print(f"Maximum Test Accuracy obtained with lr={lr:0.1e}: {max(acc_dict[f'val_{i}'])}")

plt.title('Optimal Learning Rate')
plt.ylabel('Accuracy (%)')
plt.xlabel('Epoch')
plt.legend()
plt.show()
```

```python
# @markdown Plot parametric norms (Run me)
plt.figure()

for i, lr in enumerate(learning_rates):
  plt.plot(acc_dict[f'param_norm_{i}'],label=f'lr={lr:0.2e}')

plt.legend()
plt.xlabel('epoch')
plt.ylabel('parameter norms')
plt.show()
```

In the model above, we observe something different from what we expected. Why do you think this is happening?

---
# Section 5: Hyperparameter Tuning

*Time estimate: ~5 mins*

```python
# @title Video 5: Hyperparameter tuning
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1E44y127Sn", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"HgkiKRYc-3A", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 5: Hyperparameter tuning')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

Hyper-Parameter tuning is often difficult and time consuming.  It is a key part of training any Deep Learning model to give good generalization. There are a few techniques that we can use to guide us during the search.

*   Grid Search: Try all possible combinations of hyperparameters
*   Random Search: Randomly try different combinations of hyperparameters
*   Coordinate-wise Gradient Descent: Start at one set of hyperparameters and try changing one at a time, accept any changes that reduce your validation error
*   Bayesian Optimization/ Auto ML:  Start from a set of hyperparameters that have worked well on a similar problem, and then do some sort of local exploration (e.g., gradient descent) from there.

There are lots of choices, like what range to explore over, which parameter to optimize first, etc. Some hyperparameters don’t matter much (people use a dropout of either 0.5 or 0, but not much else).  Others can matter a lot more (e.g., size and depth of the neural net). The key is to see what worked on similar problems.

One can automate the process of tuning the network Architecture using "Neural Architecture Search", which designs new architectures using a few building blocks (Linear, Convolutional, Convolution Layers, etc.) and optimizes the design based on performance using a wide range of techniques such as Grid Search, Reinforcement Learning, GD, Evolutionary Algorithms, etc. This obviously requires very high computer power. Read this [article](https://lilianweng.github.io/lil-log/2020/08/06/neural-architecture-search.html) to learn more about NAS.

## Think! 5: Overview of regularization techniques

Which regularization technique today do you think had the biggest effect on the network? Why might do you think so? Can you apply all of the regularization methods on the same network?

```python
# @title Student Response
from ipywidgets import widgets

text=widgets.Textarea(
   value='Type your answer here and click on `Submit!`',
   placeholder='Type something',
   description='',
   disabled=False
)

button = widgets.Button(description="Submit!")

display(text,button)

def on_button_clicked(b):
   atform.add_answer('q2', text.value)
   print("Submission successful!")

button.on_click(on_button_clicked)
```

    Textarea(value='Type your answer here and click on `Submit!`', placeholder='Type something')

    Button(description='Submit!', style=ButtonStyle())

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content-dl/tree/main//tutorials/W1D5_Regularization/solutions/W1D5_Tutorial2_Solution_a308b739.py)

---
# Summary

Congratulations! The first week of NMA-DL has ended! In this tutorial, you learned more techniques of regulariation, i.e., L1 and L2 regularization, Dropout, and Data Augmenetation. Finally, you have seen the learning rate of SGD can act as a reularizer. An iteresting paper can be found [here](https://arxiv.org/abs/1611.03530).

If you have time left, see the bonus material on *Adversarial Attacks*!

```python
# @title Airtable Submission Link
from IPython import display as IPydisplay
IPydisplay.HTML(
   f"""
 <div>
   <a href= "{atform.url()}" target="_blank">
   </a>
   </div>""" )
```

  <a href= "https://portal.neuromatchacademy.org/api/redirect/to/a76f99c1-9005-4566-8bcd-bed4e53d21f1?data=eyJmb3JtX2lkIjogImFwcG43VmRQUnNlU29NWEVHIiwgInRhYmxlX25hbWUiOiAiVzFENV9UMiIsICJhbnN3ZXJzIjogeyJxMSI6ICJXZSBpbmNyZWFzZSB0aGUgbnVtYmVyIG9mIHBvc3NpYmxlIHNvbHV0aW9ucywgd2hpY2ggYXJlIG1vcmUgbGlrZWx5IHRvIGJlIGZvdW5kIGR1cmluZyB0cmFpbmluZywgYW5kIHdlIHRhaWxvciB0aGFuIHRvIGNvbnZlcmdlIHRvIHNtb290aGVyIHNvbHV0aW9ucy4iLCAicTIiOiAiRGVwZW5kcyBvbiB0aGUgZGF0YXNldCwgYnV0IHJpZGdlIHJlZ3Jlc3Npb24gYW5kIGRyb3BvdXQgc2VlbXMgdG8gYmUgZ2VuZXJhbGx5IGFwcGxpY2FibGUuIn0sICJldmVudHMiOiBbeyJldmVudCI6ICJpbml0IiwgInRzIjogMTYyODI4MjQ2Ni40NjczMjM1fSwgeyJldmVudCI6ICJWaWRlbyAxOiBMMSBhbmQgTDIgcmVncmVzc2lvbiIsICJ0cyI6IDE2MjgyODI0NzUuNDcxMzQ0N30sIHsiZXZlbnQiOiAiQ29kaW5nIEV4ZXJjaXNlIDEuMTogTDEgUmVndWxhcml6YXRpb24iLCAidHMiOiAxNjI4MjgyNTQ5LjY4MTMxMn0sIHsiZXZlbnQiOiAiQ29kaW5nIEV4ZXJjaXNlIDEuMjogTDIgUmVndWxhcml6YXRpb24iLCAidHMiOiAxNjI4MjgyNjIxLjEyODMzMjZ9LCB7ImV2ZW50IjogIkNvZGluZyBFeGVyY2lzZSAxLjI6IEwyIFJlZ3VsYXJpemF0aW9uIiwgInRzIjogMTYyODI4Mjg1My44OTgyMDQzfSwgeyJldmVudCI6ICJWaWRlbyAxOiBMMSBhbmQgTDIgcmVncmVzc2lvbiIsICJ0cyI6IDE2MjgyODMwMTIuOTYzOTExNX0sIHsiZXZlbnQiOiAiQ29kaW5nIEV4ZXJjaXNlIDEuMTogTDEgUmVndWxhcml6YXRpb24iLCAidHMiOiAxNjI4MjgzMzEzLjM4NjE2fSwgeyJldmVudCI6ICJDb2RpbmcgRXhlcmNpc2UgMS4yOiBMMiBSZWd1bGFyaXphdGlvbiIsICJ0cyI6IDE2MjgyODMzODkuMzYwNTEwOH0sIHsiZXZlbnQiOiAiVmlkZW8gMzogRGF0YSBBdWdtZW50YXRpb24iLCAidHMiOiAxNjI4MjgzNzM5LjkyMjEwMDN9LCB7ImV2ZW50IjogIkNvZGluZyBFeGVyY2lzZSAxLjI6IEwyIFJlZ3VsYXJpemF0aW9uIiwgInRzIjogMTYyODI4NDQ0Ny45MjQ0NTE0fSwgeyJldmVudCI6ICJWaWRlbyA0OiBTR0QiLCAidHMiOiAxNjI4Mjg1NDY4Ljk3NzI4NH0sIHsiZXZlbnQiOiAiYW5zd2VyZWQgcTEiLCAidHMiOiAxNjI4Mjg1NjE2LjgwNjE1Njl9LCB7ImV2ZW50IjogIlZpZGVvIDU6IEh5cGVycGFyYW1ldGVyIHR1bmluZyIsICJ0cyI6IDE2MjgyODYwMTUuNjczNjYxMn0sIHsiZXZlbnQiOiAiYW5zd2VyZWQgcTIiLCAidHMiOiAxNjI4Mjg2MjAwLjkwMDk0OH0sIHsiZXZlbnQiOiAiYW5zd2VyZWQgcTIiLCAidHMiOiAxNjI4Mjg2MjA1LjI5MTY5MTh9LCB7ImV2ZW50IjogInVybCBnZW5lcmF0ZWQiLCAidHMiOiAxNjI4Mjg2MjM1LjE3MDIxODd9XX0%3D" target="_blank">

---
# Bonus: Adversarial  Attacks

*Time estimate: ~15 mins*

```python
# @title Video 6: Adversarial Attacks
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV19o4y1X74u", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"LzPPoiKi5jE", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

# add event to airtable
atform.add_event('Video 6: Adversarial Attacks')

display(out)
```

Designing perturbations to the input data to trick a machine learning model is called an "adversarial attack". These attacks are an inevitable consequence of learning in high dimensional space with complex decision boundaries. Depending on the application, these attacks can be very dangerous.

Hence, it is important for us to build models which can defend against such attacks. One possible way to do it is by regularizing the networks, which smooths the decision boundaries. A few ways of building models robust to such attacks are:

*   [Defensive Distillation](https://deepai.org/machine-learning-glossary-and-terms/defensive-distillation) : Models trained via distillation are less prone to such attacks as they are trained on soft labels as there is an element of randomness in the training process.
*   [Feature Squeezing](https://evademl.org/squeezing/): Identifies adversarial attacks for on-line classifiers whose model is being used by comparing model's perdiction before and after squeezing the input.
* [SGD](https://arxiv.org/abs/1706.06083) You can also pick weight to minimize what the adversary is trying to maximize via SGD.

Read more about adversarial attacks [here](https://openai.com/blog/adversarial-example-research/)


# DataRecord Tutorial

For more Landlab tutorials, click here: https://landlab.readthedocs.io/en/latest/user_guide/tutorials.html

This tutorial illustrates how to record variables of a Landlab model using DataRecord.

## What is DataRecord?
DataRecord is a data structure that can hold data variables relating to a Landlab model or to items living on the [Landlab grid](../grid_object_demo/grid_object_demo.ipynb).

DataRecord is built on [xarray](http://xarray.pydata.org/en/stable/index.html)'s Dataset structure: a multi-dimensional, in memory, array database. Dataset implements the mapping interface with keys given by variable names and values given by DataArray objects for each variable name. DataRecord inherits all the methods and attributes from xarray.Dataset.

A DataRecord can have one or both (or none) of the following dimensions:
- `time`: The simulated time in the model.
- `item_id`: An identifier of a generic item in the model.

Coordinates are one dimensional arrays used for label-based indexing.

The examples below illustrate different use cases for DataRecord.

We start by importing the necessary libraries:

```python
import numpy as np
from landlab import RasterModelGrid
from landlab.data_record import DataRecord

from landlab import imshow_grid
import matplotlib.pyplot as plt
from matplotlib.pyplot import plot, subplot, xlabel, ylabel, title, legend, figure
%matplotlib inline
```

## Case 1. DataRecord with 1 dimension: time
Let's start with an example where we set DataRecord to have only `time` as a dimension.
An example variable that varies over time and relates to the Landlab grid could be the mean elevation of the topographic surface. We will store this example variable in DataRecord.

We create a Raster grid, create a field (at nodes) called `topographic__elevation` and populate it with random values.

```python
grid_1 = RasterModelGrid((10, 10), (1., 1.))
z = np.random.rand(100)
_ = grid_1.add_field('topographic__elevation', z, at='node')
```

Print the current mean elevation.

```python
current_mean = np.mean(grid_1.at_node['topographic__elevation'])
print(current_mean)
```

Now we will create a DataRecord that will hold the data variable `mean_elevation` relating to `grid_1`. The first value, at time=0 is the current mean elevation on the grid.

```python
dr_1 = DataRecord(grid_1,
                  time=[0.],
                  items=None,
                  data_vars={'mean_elevation': (['time'], ([current_mean]))},
                  attrs={'mean_elevation': 'y'})
```

The input arguments passed in this case are: the grid, time (as a 1-element list), a data variable dictionary and an attributes dictionary. Note that `items` is not filled, we will see its use in other cases below.

Note the format of the `data_vars` dictionary:
```python
    {'variable_name_1' : (['dimensions'], variable_data_1),
     'variable_name_2' : (['dimensions'], variable_data_2),
     ...}
```

The attributes dictionary `attrs` can be used to store metadata about the variables: in this example, we use it to store the variable units.

So far, our DataRecord `dr_1` holds one variable `mean_elevation` with one record at time=0.

```python
dr_1
```

We can visualise this data structure as a [pandas dataframe](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html):

```python
dr_1.dataset.to_dataframe()
```

Now we will run a simple model where the grid surface is uplifted several times and the mean elevation is recorded at every time step. We use the method `add_record` to put the new value in the DataRecord `dr_1`:

```python
total_time = 100
dt = 20

uplift_rate = 0.01  # m/y

for t in range(20, total_time, dt):
    grid_1.at_node['topographic__elevation'] += uplift_rate * dt
    dr_1.add_record(time=[t],
                    new_record={
                        'mean_elevation':
                        (['time'],
                         ([np.mean(grid_1.at_node['topographic__elevation'])]))
                    })
```

Let's see what was recorded:

```python
dr_1.dataset['mean_elevation'].values
```

The corresponding time coordinates are:

```python
dr_1.dataset.time.values
```

Notice the different syntax used here:
- `time` is a **dimension** and can be called by `dr_1.time` (or `dr_1['time']`)
- whereas `mean_elevation` is a **variable** and must be called by `dr_1['mean_elevation']`

DataRecord also has the handy property `time_coordinates` that returns these values as a list:

```python
dr_1.time_coordinates
```

You can use the methods `get_data` and `set_data` to access and change the data:

```python
dr_1.get_data(time=[20.], data_variable='mean_elevation')
```

```python
dr_1.set_data(time=[80.], data_variable='mean_elevation', new_value=1.5)

dr_1.dataset['mean_elevation']
```

## Case 2. DataRecord with 1 dimension: item_id
An important feature of DataRecord is that it allows to create **items** that live on grid elements, and variables describing them. For instance, we can create *boulders* and store information about their *size* and *lithology*.

To create items, we need to instantiate a DataRecord and pass it a dictionary describing where each item lives on the Landlab grid. The format of this dictionary is:
```python
    {'grid_element' : [grid_element],
     'element_id' : [element_id]}
```

where:
- `grid_element` is a str or number-of-items-long array containing strings of the grid element(s) on which the items live (e.g.: node, link). Valid locations depend on the grid type (`my_grid.groups` gives the valid locations for your grid). If `grid_element` is provided as a string, it is assumed that all items live on the same type of grid element.
- `element_id` is an array of integers identifying the grid element IDs on which each item resides. For each item, `element_id` must be less than the number of this item's `grid_element` that exist on the grid. For example, if the grid has 10 links, no item can live at link 10 or link -3 because only links 0 to 9 exist in this example.

```python
grid_2 = RasterModelGrid((5, 5), (2, 2))

boulders = {
    'grid_element': 'node',
    'element_id': np.array([6, 11, 12, 17, 12])
}

initial_boulder_sizes = np.array([1, 1.5, 3, 1, 2])
boulder_lithologies = np.array(
    ['sandstone', 'granite', 'sandstone', 'sandstone', 'limestone'])

dr_2 = DataRecord(grid_2,
                  time=None,
                  items=boulders,
                  data_vars={
                      'boulder_size': (['item_id'], initial_boulder_sizes),
                      'boulder_litho': (['item_id'], boulder_lithologies)
                  },
                  attrs={'boulder_size': 'm'})
dr_2.dataset.to_dataframe()
```

Each *item* (in this case, each boulder) is designated by an `item_id`, its position on the grid is described by a `grid_element` and an `element_id`.

We can use the method `add_item` to add new boulders to the record:

```python
dr_2.add_item(
    new_item={
        'grid_element': np.array(['link', 'node']),
        'element_id': np.array([24, 8])
    },
    new_item_spec={'boulder_size': (['item_id'], np.array([1.2, 2.]))})

dr_2.dataset.to_dataframe()
```

Notice that we did not specify the lithologies of the new boulders, their recorded values are thus set as `NaN`. We can use the `set_data` method to report the boulder lithologies:

```python
dr_2.set_data(data_variable='boulder_litho',
              item_id=[5, 6],
              new_value=['sandstone', 'granite'])
dr_2.dataset.to_dataframe()
```

We can use the method `calc_aggregate_value` to apply a function to a variable aggregated at grid elements. For example, we can calculate the mean size of boulders on each node:

```python
mean_size = dr_2.calc_aggregate_value(func=np.mean,
                                      data_variable='boulder_size')
mean_size
```

Notice that boulder #5 is on a link so it is not taken into account in this calculation.

```python
# replace nans with 0:
mean_size[np.isnan(mean_size)] = 0

# show unfiltered mean sizes on the grid:
imshow_grid(grid_2, mean_size)
```

Before doing this calculation we could filter by lithology and only use the 'sandstone' boulders in the calculation:

```python
# define a filter array:
filter_litho = (dr_2.dataset['boulder_litho'] == 'sandstone')

# aggregate by node and apply function numpy.mean on boulder_size
filtered_mean = dr_2.calc_aggregate_value(func=np.mean,
                                          data_variable='boulder_size',
                                          at='node',
                                          filter_array=filter_litho)

filtered_mean
```

## Case 3. DataRecord with 2 dimensions: item_id and time

We may want to record variables that have both dimensions `time` *and* `item_id`.

In the previous example, some variables that characterize the items (boulders) may not vary with time, such as `boulder_lithology`. Although it can be interesting to keep track of the change in size through time. We will redefine the DataRecord such that the variable `boulder_size` varies among the items/boulders (identified by `item_id`) and through `time`. The variable `boulder_litho` varies only among the items/boulders and this lithogy variable does not vary through time.

```python
grid_3 = RasterModelGrid((5, 5), (2, 2))

initial_boulder_sizes_3 = np.array([[10], [4], [8], [3], [5]])
# boulder_lithologies = np.array(['sandstone', 'granite', 'sandstone', 'sandstone', 'limestone']) #same as above, already run

boulders_3 = {
    'grid_element': 'node',
    'element_id': np.array([[6], [11], [12], [17], [12]])
}

dr_3 = DataRecord(grid_3,
                  time=[0.],
                  items=boulders_3,
                  data_vars={
                      'boulder_size': (['item_id',
                                        'time'], initial_boulder_sizes_3),
                      'boulder_litho': (['item_id'], boulder_lithologies)
                  },
                  attrs={'boulder_size': 'm'})
dr_3
```

Note that the syntax to define the `initial_boulder_sizes_3` (as well as `element_id`) has changed: they are  number-of-items-by-1 arrays because they vary along both `time` and `item_id` (compared to `boulder_lithologies` which is just number-of-items long as it only varies along `item_id`).

```python
boulder_lithologies.shape, initial_boulder_sizes.shape, initial_boulder_sizes_3.shape
```

Let's define a very simple erosion law for the boulders:

$$
\begin{equation}
\frac{dD}{dt} = -k_{b} . D
\end{equation}
$$

where $D$ is the boulder diameter $[L]$ (this value represents the `boulder_size` variable), $t$ is time, and $k_{b}$ is the block erodibility $[L.T^{-1}]$.

We will now model boulder erosion and use DataRecord to store their size through time.

```python
dt = 100
total_time = 100000

time_index = 1

for t in range(dt, total_time, dt):

    # create a new time coordinate:
    dr_3.add_record(time=np.array([t]))

    # this propagates grid_element and element_id values forward in time (instead of the 'nan' default filling):
    dr_3.ffill_grid_element_and_id()

    for i in range(0, dr_3.number_of_items):
        # value of block erodibility:
        if dr_3.dataset['boulder_litho'].values[i] == 'limestone':
            k_b = 10**-5
        elif dr_3.dataset['boulder_litho'].values[i] == 'sandstone':
            k_b = 3 * 10**-6
        elif dr_3.dataset['boulder_litho'].values[i] == 'granite':
            k_b = 3 * 10**-7
        else:
            print('Unknown boulder lithology')

        dr_3.dataset['boulder_size'].values[i, time_index] = dr_3.dataset[
            'boulder_size'].values[i, time_index - 1] - k_b * dr_3.dataset[
                'boulder_size'].values[i, time_index - 1] * dt

    time_index += 1

print('Done')
```

```python
figure(figsize=(15, 8))

time = range(0, total_time, dt)
boulder_size = dr_3.dataset['boulder_size'].values

subplot(121)
plot(time, boulder_size[1], label='granite')
plot(time, boulder_size[3], label='sandstone')
plot(time, boulder_size[-1], label='limestone')
xlabel('Time (yr)')
ylabel('Boulder size (m)')
legend(loc='lower left')
title('Boulder erosion by lithology')

# normalized plots
subplot(122)
plot(time, boulder_size[1] / boulder_size[1, 0], label='granite')
plot(time, boulder_size[2] / boulder_size[2, 0], label='sandstone')
plot(time, boulder_size[-1] / boulder_size[-1, 0], label='limestone')
xlabel('Time (yr)')
ylabel('Boulder size normalized to size at t=0 (m)')
legend(loc='lower left')
title('Normalized boulder erosion by lithology')
plt.show()
```

## Other properties provided by DataRecord

```python
dr_3.variable_names
```

```python
dr_3.number_of_items
```

```python
dr_3.item_coordinates
```

```python
dr_3.number_of_timesteps
```

```python
dr_1.time_coordinates
```

```python
dr_1.earliest_time
```

```python
dr_1.latest_time
```

```python
dr_1.prior_time
```

# More on DataRecord

DataRecord is the data structure on which the following Landlab components are based:
- ClastTracker (coming soon)
- SpeciesEvolver (coming soon)

```python

```