# Tutorial 1: Introduction to CNNs
**Week 2, Day 1: Convnets And Recurrent Neural Networks**

**By Neuromatch Academy**

__Content creators:__ Dawn Estes McKnight, Richard Gerum, Cassidy Pirlot, Rohan Saha, Liam Peet-Pare, Saeed Najafi, Alona Fyshe

__Content reviewers:__ Saeed Salehi, Lily Cheng, Yu-Fang Yang, Polina Turishcheva, Bettina Hein

__Content editors:__ Nina Kudryashova, Anmol Gupta, Spiros Chavlis

__Production editors:__ Alex Tran-Van-Minh, Spiros Chavlis

*Based on material from:* Konrad Kording, Hmrishav Bandyopadhyay, Rahul Shekhar, Tejas Srivastava

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives
At the end of this tutorial, we will be able to:
- Define what convolution is
- Implement convolution as an operation

```
# @title Tutorial slides

# @markdown These are the slides for the videos in this tutorial
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/s8xz5/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

---
# Setup

```
# @title Install dependencies
!pip install livelossplot --quiet
!pip install Pillow --quiet
!pip install opencv-python --quiet
!pip install scipy --quiet
```

```
# Imports
import os
import cv2
import sys
import time
import torch
import pathlib
import scipy.signal
import numpy as np
import matplotlib.pyplot as plt

import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from torchvision.utils import make_grid
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader, TensorDataset

from IPython.display import HTML, display

from tqdm.auto import tqdm
from tqdm.notebook import tqdm, trange
from time import sleep
from PIL import Image
```

```
# @title Figure Settings
import ipywidgets as widgets       # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/content-creation/main/nma.mplstyle")

plt.rcParams["mpl_toolkits.legacy_colorbar"] = False
import warnings
warnings.filterwarnings("ignore", category=UserWarning, module="matplotlib")
```

```
# @title Helper functions
from scipy.signal import convolve2d
from scipy.signal import correlate2d

def check_shape_function(func, image_shape, kernel_shape):
  correct_shape = correlate2d(np.random.rand(*image_shape), np.random.rand(*kernel_shape), "valid").shape
  user_shape = func(image_shape, kernel_shape)
  if correct_shape != user_shape:
    print(f"❌ Your calculated output shape is not correct.")
  else:
    print(f"✅ Output for image_shape: {image_shape} and kernel_shape: {kernel_shape}, output_shape: {user_shape}, is correct.")

def check_conv_function(func, image, kernel):
  solution_user = func(image, kernel)
  solution_scipy = correlate2d(image, kernel, "valid")
  result_right = (solution_user == solution_scipy).all()
  if result_right:
    print("✅ The function calculated the convolution correctly.")
  else:
    print("❌ The function did not produce the right output.")
    print("For the input matrix:")
    print(image)
    print("and the kernel:")
    print(kernel)
    print("the function returned:")
    print(solution_user)
    print("the correct output would be:")
    print(solution_scipy)

def check_pooling_net(net, device='cpu'):
  x_img = emnist_train[x_img_idx][0].unsqueeze(dim=0).to(device)
  output_x = net(x_img)
  output_x = output_x.squeeze(dim=0).detach().cpu().numpy()

  right_output = [
                  [1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726,
                   1.2726, 1.2726,1.2726, 1.2726, 1.2726, 1.2726],
                  [1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726,
                   1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726],
                  [2.0491, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726,
                   1.5527, 2.0236, 1.2726, 1.2726, 1.2726, 1.2726],
                  [2.0109, 0.0000, 0.0000, 0.0000, 0.5471, 1.2726, 2.2145,
                   2.4945, 1.0181, 0.0000, 0.0000, 1.2217, 1.2726],
                  [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.0053,
                   0.0000, 0.0000, 0.0000, 0.0000, 1.9854, 1.2726],
                  [1.2726, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
                   0.0000, 0.0000, 0.0000, 2.5455, 1.5399, 1.2726],
                  [1.2726, 1.2854, 2.2527, 1.1835, 0.0000, 0.0000, 0.0000,
                   0.0000, 0.0000, 0.0000, 0.0000, 1.2217, 1.2726],
                  [1.3108, 2.1763, 2.5836, 0.0000, 0.0000, 1.8200, 0.1144,
                   0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1780],
                  [2.1891, 2.2145, 0.0000, 0.0000, 2.5836, 2.3800, 1.2726,
                   1.2726, 0.1271, 0.0000, 0.0000, 0.0000, 1.7563],
                  [1.2599, 0.0000, 1.3872, 2.0618, 1.9981, 1.2854, 1.2726,
                   1.2726, 1.2726, 1.2726, 0.9799, 1.2472, 2.5073],
                  [1.2726, 1.2599, 2.7746, 1.3745, 1.2726, 1.2726, 1.2726,
                   1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726],
                  [1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726,
                   1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726],
                  [1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726,
                   1.2726, 1.2726, 1.2726, 1.2726, 1.2726, 1.2726],
                  ]

  right_shape = (3, 13, 13)

  if output_x.shape != right_shape:
    print(f"❌ Your output does not have the right dimensions. Your output is {output_x.shape} the expected output is {right_shape}")
  elif (output_x[0] != right_output).all():
    print("❌ Your output is not right.")
  else:
    print("✅ Your network produced the correct output.")
```

```
# @title Plotting Functions

def display_image_from_greyscale_array(matrix, title):
  _matrix = matrix.astype(np.uint8)
  _img = Image.fromarray(_matrix, 'L')
  plt.figure(figsize=(3, 3))
  plt.imshow(_img, cmap='gray', vmin=0, vmax=255) # using 220 instead of 255 so the examples show up better
  plt.title(title)
  plt.axis('off')

def make_plots(original, actual_convolution, solution):
  display_image_from_greyscale_array(original, "Original Image")
  display_image_from_greyscale_array(actual_convolution, "Convolution result")
  display_image_from_greyscale_array(solution, "Your solution")
```

```
# @title Set random seed

# @markdown Executing `set_seed(seed=seed)` you are setting the seed

# for DL its critical to set the random seed so that students can have a
# baseline to compare their results to expected results.
# Read more here: https://pytorch.org/docs/stable/notes/randomness.html

# Call `set_seed` function in the exercises to ensure reproducibility.
import random
import torch

def set_seed(seed=None, seed_torch=True):
  if seed is None:
    seed = np.random.choice(2 ** 32)
  random.seed(seed)
  np.random.seed(seed)
  if seed_torch:
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

  print(f'Random seed {seed} has been set.')

# In case that `DataLoader` is used
def seed_worker(worker_id):
  worker_seed = torch.initial_seed() % 2**32
  np.random.seed(worker_seed)
  random.seed(worker_seed)
```

```
# @title Set device (GPU or CPU). Execute `set_device()`
# especially if torch modules used.

# inform the user if the notebook uses GPU or CPU.

def set_device():
  device = "cuda" if torch.cuda.is_available() else "cpu"
  if device != "cuda":
    print("WARNING: For this notebook to perform best, "
        "if possible, in the menu under `Runtime` -> "
        "`Change runtime type.`  select `GPU` ")
  else:
    print("GPU is enabled in this notebook.")

  return device
```

```
SEED = 2021
set_seed(seed=SEED)
DEVICE = set_device()
```

---
# Section 0: Recap the Experience from Last Week

Last week you learned a lot!  Recall that overparametrized ANNs are efficient universal approximators, but also that ANNs can memorize our data.  However, regularization can help ANNs to better generalize. You were introduced to several regularization techniques such as *L1*, *L2*, *Data Augmentation*, and *Dropout*.

Today we'll be talking about other ways to simplify ANNs, by making smart changes to their architecture.

```
# @title Video 1: Introduction to CNNs and RNNs
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1cL411p7rz", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"5598K-hS89A", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

## Think! 0: Regularization & effective number of params
Let's think back to last week, when you learned about regularization.  Recall that regularization comes in several forms.  For example, L1 regularization adds a term to the loss function that penalizes based on the sum of the _absolute_ magnitude of the weights. Below are the results from training a simple multilayer perceptron with one hidden layer (b) on a simple toy dataset (a).

Below that are two graphics that show the effect of regularization on both the number of non-zero weights (d), and on the network's accuracy (c).
What do you notice?

**Note**: Dense layers are the same as fully-connected layers.  And pytorch calls them linear layers.  Confusing, but now you know!

```
# to_remove explanation

"""
When you increase the regularization strength you can see that the test accuracy
improves (orange line in (c)),while the effective number of parameters goes down
(d).At a certain point, however, the regularization is too strong, and the
accuracy declines (1e-1).
"""
```

**Coming Up**

The rest of these lectures focus on another way to reduce parameters: weight-sharing. Weight-sharing is based on the idea that some sets of weights can be used at multiple points in a network. We will focus mostly on CNNs today, where the weight-sharing is across the 2D space of an image. At the end we will touch briefly on Recurrent Neural Networks (RNNs), which share parameters across time. Both of these weight-sharing techniques (across space and time) can reduce the number of parameters and increase a network's ability to generalize.

---
# Section 1: Neuroscience motivation, General CNN structure

```
# @title Video 2: Representations & Visual processing in the brain
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1c64y1x7mJ", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"AXO-iflKa58", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

## Think! 1: What makes a representation good?
Representations have a long and storied history, having been studied by the likes of Aristotle back in 300 BC! Representations are not a new idea, and they certainly don't exist just in neural networks.

Take a moment with your pod to discuss what would make a good representation, and how that might differ depending on the task you train your CNN to do.

If there's time, you can also consider how the brain's representations might differ from a *learned* representation inside a NN.

```
# to_remove explanation

"""
A good representation captures the important aspects of an entity - Using a good
representation, we can understand the nuances of the data - for example, a good
representation of an image showing a dog would accurately capture the eyes,
ears etc and a representation of, for example, a building, would capture the
edges.
"""
```

---
# Section 2: Convolutions and Edge Detection

Fundamental to CNNs are convolutions. After all, that _is_ what the C in CNN stands for! In this section, we will define what a convolution is, practice performing a convolution, and implement it in code.

```
# @title Video 3: Details about Convolution
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1Q64y1z77p", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"pmc40WCnF-w", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

Before jumping into coding exercises, take a moment to look at this animation that steps through the process of convolution.

Recall from the video that convolution involves sliding the kernel across the image, taking the element-wise product, and adding those products together.

Source A. Zhang, Z. C. Lipton, M. Li and A. J. Smola,  _[Dive into Deep Learning](http://d2l.ai/chapter_convolutional-neural-networks/conv-layer.html)_.

**Note:** You need to run the cell to activate the sliders, and again to run once changing the sliders.

**Tip:** In this animation, and all the ones that follow, you can hover over the parts of the code underlined in red to change them.

**Tip:** Below, the function is called `Conv2d` because the convolutional filter is a matrix with two dimensions (2D).  There are also 1D and 3D convolutions, but we won't talk about them today.

### Interactive Demo 2: Visualization of Convolution

```
# @markdown *Run this cell to enable the widget!*
%%html
<style>
    svg {
        #border: 1px solid black;
    }
.matrix {
    font-family: sans-serif;
    transition: all 700ms ease-in-out;
}
.cell rect {
    fill:white;stroke-width:1;stroke:rgb(0,0,0)
}
.padding rect {
    stroke: rgba(0, 0, 0, 0.25);
}
.padding text {
    fill: lightgray;
}
.highlight1 {
    fill:none;stroke-width:4;stroke: rgb(236, 58, 58);stroke-dasharray:10,5;
}
.highlight2 {
    fill:rgba(229, 132, 66, 0.25);stroke-width:5;stroke: rgb(229, 132, 66);
}
.highlight3 {
    fill:rgba(236, 58, 58, 0.25);stroke-width:2;stroke: rgb(236, 58, 58);;
}
.title {
    text-anchor: middle;
}
.button_play {
    display: inline-block;
    background: none;
    border: none;
    position: relative;
    top: -3px;
}
.button_play path {
    fill: darkgray;
}
.button_play:hover path {
    fill: rgb(236, 58, 58);
}
.display_vis_input input:not(:hover)::-webkit-outer-spin-button,
.display_vis_input input:not(:hover)::-webkit-inner-spin-button {
    /* display: none; <- Crashes Chrome on hover */
    -webkit-appearance: none;
    margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
}

.display_vis_input input:not(:hover)[type=number] {
    -moz-appearance:textfield; /* Firefox */
    width: 1ch;
    margin-right: 0px;
    z-index: 0;
}
.display_vis_input input[type=number] {
    width: 4ch;
    border: 0px;
    margin-right: -3ch;
    z-index: 6;
    display: inline-block;
    position: relative;
    padding: 0;
    border-bottom: 2px solid red;
    background: white;
    color: black
}
.display_vis_input .pair {
    display: inline-block;
    white-space:nowrap;
        position: relative;
}
.display_vis_input .pair .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
    display: inline-block;
    overflow: hidden;
    position: relative;
    top: 5px;
}
.pair:not(:hover) .pair_hide {
    max-width: 0;
}
.pairX .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
}

/* Dropdown Button */
.dropbtn {
  border-bottom: 2px solid red;
}

/* The container <div> - needed to position the dropdown content */
.dropdown {
  position: relative;
  display: inline-block;
}

/* Dropdown Content (Hidden by Default) */
.dropdown-content {
  display: none;
  position: absolute;
  background-color: #f1f1f1;
  min-width: 160px;
  box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
  z-index: 1;
}

/* Links inside the dropdown */
.dropdown-content a {
  color: black;
  padding: 5px 2px;
  text-decoration: none;
  display: block;
}

/* Change color of dropdown links on hover */
.dropdown-content a:hover {background-color: #ddd;}

/* Show the dropdown menu on hover */
.dropdown:hover .dropdown-content {display: block;}

</style>

<div id="animation_conv_filters" style="background: white">
    <div class="display_vis_input language-python" style="font-family: monospace; color: black; padding: 10px;">
        <!-- default -->
        import torch<br><br>
        input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixy" type="number" min="3" max="5" value="3">, <input class="input_matrixx" type="number" min="3" max="5" value="4">)<br>
        conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filtery" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filterx" type="number" min="2" max="4" value="3">)</span></span>
        <span class="pair" style="display: none"><span class="pair_hide">(</span><input class="input_stridex" type="hidden" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="hidden" min="1" max="2" value="1">)</span></span>
        <span class="pair" style="display: none"><span class="pair_hide">(</span><input class="input_paddingx" type="hidden" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="hidden" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)

        <!-- padding
        import torch<br><br>
        input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">))<br>
        conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="3">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="1"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="1">)</span></span>)<br>
        result = conv(input)
-->
    <!-- filters -- >
        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="number" min="1" max="3" value="3">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">)<br>
        conv = torch.nn.Conv2d(in_channels=<input class="input_matrixzB" type="number" min="1" max="3" value="3">, out_channels=<input class="input_filterz" type="number" min="1" max="3" value="2">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)
-->
 <!--
        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="hidden" min="1" max="3" value="1">1, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">))<br>
        conv = torch.nn.<div class="dropdown">
  <div class="dropbtn">MaxPool2d</div>
  <div class="dropdown-content">
    <a class="select_maxpool" href="#">MaxPool2d</a>
    <a class="select_avgpool" href="#">AvgPool2d</a>
  </div>
</div>(<input class="input_matrixzB" type="hidden" min="1" max="3" value="1"><input class="input_filterz" type="hidden" min="1" max="3" value="1">kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="2"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="2">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)
-->
    </div>
        <button class="button_play play"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 1.5,0 9.5,5 1.5,10 z"/></svg></button>
    <button class="button_play pause" style="display: none"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 0,0 4,0 4,10, 0,10 z"/><path d="M 6,0 10,0 10,10, 6,10 z"/></svg></button>
    <input type="range" min="1" max="100" value="50" class="slider" style="width: 300px; display: inline-block">
    <button class="button_play left"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,5 4,0 4,10 z"/></svg></button>
    <button class="button_play right"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,0 4,5 0,10 z"/></svg></button>
    <input type="checkbox" class="play_fast">fast play mode
    <br/>
    <svg height="0" width="0">
        <defs>
    <marker id="arrowhead" markerWidth="10" markerHeight="7"
    refX="0" refY="1.5" orient="auto" fill="rgb(236, 58, 58)">
      <polygon points="0 0, 4 1.5, 0 3" />
    </marker>
  </defs>
    </svg>
    <svg class="image" height="460" width="600">

    </svg>
</div>

```

#### Definitional Note

If you have a background in signal processing or math, you may have already heard of convolution. However, the definitions in other domains and the one we use here are slightly different. The more common definition involves flipping the kernel horizontally and vertically before sliding.

**For our purposes, no flipping is needed. If you are familiar with conventions involving flipping, just assume the kernel is pre-flipped.**

In more general usage, the no-flip operation that we call convolution is known as _cross-correlation_ (hence the usage of `scipy.signal.correlate2d` in the next exercise). Early papers used the more common definition of convolution, but not using a flip is easier to visualize, and in fact the lack of flip does not impact a CNN's ability to learn.

## Coding Exercise 2.1: Convolution of a Simple Kernel
At its core, convolution is just repeatedly multiplying a matrix, known as a _kernel_ or _filter_, with some other, larger matrix (in our case the pixels of an image). Consider the below image and kernel:

\begin{align}
\textbf{Image} &=
\begin{bmatrix}0 & 200 & 200 \\0 & 0 & 200 \\ 0 & 0 & 0
\end{bmatrix} \\ \\
\textbf{Kernel} &=
\begin{bmatrix} \frac{1}{4} &\frac{1}{4} \\\frac{1}{4} & \frac{1}{4}
\end{bmatrix}
\end{align}

Perform (by hand) the operations needed to convolve the kernel and image above. Afterwards enter your results in the "solution" section in the code below. Think about what this specific kernel is doing to the original image.

```
def conv_check():
  ####################################################################
  # Fill in missing code below (the elements of the matrix),
  # then remove or comment the line below to test your function
  raise NotImplementedError("Fill in the solution matrix, then delete this")
  ####################################################################
  # Write the solution array and call the function to verify it!
  solution = ...

  original = np.array([
                       [0, 200, 200],
                       [0,   0, 200],
                       [0,   0,   0]
                       ])

  kernel = np.array([
                     [0.25, 0.25],
                     [0.25, 0.25]
                     ])

  actual_convolution = scipy.signal.correlate2d(original, kernel, mode="valid")

  if (solution == actual_convolution).all():
    print("✅ Your solution is correct!\n")
  else:
    print("❌ Your solution is incorrect.\n")

  return original, kernel, actual_convolution, solution

## Uncomment to test your solution!
# original, kernel, actual_convolution, solution = conv_check()
# make_plots(original, actual_convolution, solution)
```

```
# to_remove solution
def conv_check():
  # Write the solution array and call the function to verify it!
  solution = np.array([
                       [50, 150],
                       [0,   50]
                       ])

  original = np.array([
                       [0, 200, 200],
                       [0,   0, 200],
                       [0,   0,   0]
                       ])

  kernel = np.array([
                     [0.25, 0.25],
                     [0.25, 0.25]
                     ])

  actual_convolution = scipy.signal.correlate2d(original, kernel, mode="valid")

  if (solution == actual_convolution).all():
    print("✅ Your solution is correct!\n")
  else:
    print("❌ Your solution is incorrect.\n")

  return original, kernel, actual_convolution, solution

## Uncomment to test your solution!
original, kernel, actual_convolution, solution = conv_check()
with plt.xkcd():
  make_plots(original, actual_convolution, solution)
```

## Coding Exercise 2.2: Convolution Output Size

Now, you have manually calculated a convolution. How did this change the shape of the output? When you know the shapes of the input matrix and kernel, what is the shape of the output?

*Hint: If you have problems figuring out what the output shape should look like, go back to the visualisation and see how the output shape changes as you modify the image and kernel size*.

```
def calculate_output_shape(image_shape, kernel_shape):
  image_height, image_width = image_shape
  kernel_height, kernel_width = kernel_shape
  ####################################################################
  # Fill in missing code below, then remove or comment the line below to test your function
  raise NotImplementedError("Fill in the lines below, then delete this")
  ####################################################################
  output_height = ...
  output_width = ...
  return output_height, output_width

# here we check if your function works correcly by applying it to different image
# and kernel shapes
# check_shape_function(calculate_output_shape, image_shape=(3, 3), kernel_shape=(2, 2))
# check_shape_function(calculate_output_shape, image_shape=(3, 4), kernel_shape=(2, 3))
# check_shape_function(calculate_output_shape, image_shape=(5, 5), kernel_shape=(5, 5))
# check_shape_function(calculate_output_shape, image_shape=(10, 20), kernel_shape=(3, 2))
# check_shape_function(calculate_output_shape, image_shape=(100, 200), kernel_shape=(40, 30))
```

```
# to_remove solution
def calculate_output_shape(image_shape, kernel_shape):
  image_height, image_width = image_shape
  kernel_height, kernel_width = kernel_shape
  output_height = image_height - kernel_height + 1
  output_width = image_width - kernel_width + 1
  return output_height, output_width

# here we check if your function works correcly by applying it to different image
# and kernel shapes
check_shape_function(calculate_output_shape, image_shape=(3, 3), kernel_shape=(2, 2))
check_shape_function(calculate_output_shape, image_shape=(3, 4), kernel_shape=(2, 3))
check_shape_function(calculate_output_shape, image_shape=(5, 5), kernel_shape=(5, 5))
check_shape_function(calculate_output_shape, image_shape=(10, 20), kernel_shape=(3, 2))
check_shape_function(calculate_output_shape, image_shape=(100, 200), kernel_shape=(40, 30))
```

```
✅ Output for image_shape: (3, 3) and kernel_shape: (2, 2), output_shape: (2, 2), is correct.
✅ Output for image_shape: (3, 4) and kernel_shape: (2, 3), output_shape: (2, 2), is correct.
✅ Output for image_shape: (5, 5) and kernel_shape: (5, 5), output_shape: (1, 1), is correct.
✅ Output for image_shape: (10, 20) and kernel_shape: (3, 2), output_shape: (8, 19), is correct.
✅ Output for image_shape: (100, 200) and kernel_shape: (40, 30), output_shape: (61, 171), is correct.
```

## Coding Exercise 2.3: Coding a Convolution
Here, we have the skeleton of a function that performs convolution using the provided image and kernel matrices.

*Exercise:* Fill in the missing lines of code. You can test your function by uncommenting the sections beneath it.

Note: in more general situations, once you understand convolutions, you can use functions already available in `pytorch`/`numpy` to perform convolution (such as `scipy.signal.correlate2d` or `scipy.signal.convolve2d`).

```
def convolution2d(image, kernel):
  """Convolves a 2D image matrix with a kernel matrix. Both are numpy arrays."""

  # get the height/width of the image, kernel, and output
  im_h, im_w = image.shape
  ker_h, ker_w = kernel.shape
  out_h = im_h - ker_h + 1
  out_w = im_w - ker_w + 1

  # create an empty matrix in which to store the output
  output = np.zeros((out_h, out_w))

  # iterate over the different positions at which to apply the kernel,
  #   storing the results in the output matrix
  for out_row in range(out_h):
    for out_col in range(out_w):
      # overlay the kernel on part of the image
      #   (multiply each element of the kernel with some element of the image, then sum)
      #   to determine the output of the matrix at a point
      current_product = 0
      for i in range(ker_h):
        for j in range(ker_w):
          ####################################################################
          # Fill in missing code below (...),
          # then remove or comment the line below to test your function
          raise NotImplementedError("Implement the convolution function")
          ####################################################################
          current_product += ...

      output[out_row, out_col] = current_product

  return output

## Tests
# first, we test the parameters we used before in the manual-calculation example
image = np.array([[0,  200, 200], [0,   0,  200], [0,   0,   0]])
kernel = np.array([[0.25, 0.25], [0.25, 0.25]])
# check_conv_function(convolution2d, image, kernel)

# next, we test with a different input and kernel (the numbers 1-9 and 1-4)
image = np.arange(9).reshape(3, 3)
kernel = np.arange(4).reshape(2, 2)
# check_conv_function(convolution2d, image, kernel)
```

```
# to_remove_solution
def convolution2d(image, kernel):
  """Convolves a 2D image matrix with a kernel matrix. Both are numpy arrays."""

  # get the height/width of the image, kernel, and output
  im_h, im_w = image.shape
  ker_h, ker_w = kernel.shape
  out_h = im_h - ker_h + 1
  out_w = im_w - ker_w + 1

  # create an empty matrix in which to store the output
  output = np.zeros((out_h, out_w))

  # iterate over the different positions at which to apply the kernel,
  #   storing the results in the output matrix
  for out_row in range(out_h):
    for out_col in range(out_w):
      # overlay the kernel on part of the image
      #   (multiply each element of the kernel with some element of the image, then sum)
      #   to determine the output of the matrix at a point
      current_product = 0
      for i in range(ker_h):
        for j in range(ker_w):
          current_product += image[out_row + i, out_col + j] * kernel[i, j]

      output[out_row, out_col] = current_product

  return output

## Tests
# first, we test the parameters we used before in the manual-calculation example
image = np.array([[0,  200, 200], [0,   0,  200], [0,   0,   0]])
kernel = np.array([[0.25, 0.25], [0.25, 0.25]])
check_conv_function(convolution2d, image, kernel)

# next, we test with a different input and kernel (the numbers 1-9 and 1-4)
image = np.arange(9).reshape(3, 3)
kernel = np.arange(4).reshape(2, 2)
check_conv_function(convolution2d, image, kernel)
```

```
✅ The function calculated the convolution correctly.
✅ The function calculated the convolution correctly.
```

### Convolution on the Chicago Skyline

After you have finished programming the above convolution function, run the below coding cell, which applies two different kernels to a greyscale picture of Chicago and takes the geometric average of the results.

**Make sure you remove all print statements from your convolution2d implementation, or this will run for a _very_ long time.** It should take somewhere between 10 seconds and 1 minute.

```
# @markdown ### Load images (run me)
!mkdir images
!wget "https://raw.githubusercontent.com/dem1995/generally_available_files/main/chicago_skyline_shrunk_v2.bmp"
!mv *.bmp images
```

```
# Visualize the output of your function
with open("images/chicago_skyline_shrunk_v2.bmp", 'rb') as skyline_image_file:
  img_skyline_orig = Image.open(skyline_image_file)
  img_skyline_mat = np.asarray(img_skyline_orig)
  kernel_ver = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]])
  kernel_hor = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]).T
  img_processed_mat_ver = convolution2d(img_skyline_mat, kernel_ver)
  img_processed_mat_hor = convolution2d(img_skyline_mat, kernel_hor)
  img_processed_mat = np.sqrt(np.multiply(img_processed_mat_ver,
                                          img_processed_mat_ver) + \
                              np.multiply(img_processed_mat_hor,
                                          img_processed_mat_hor))

  img_processed_mat *= 255.0/img_processed_mat.max()
  img_processed_mat = img_processed_mat.astype(np.uint8)
  img_processed = Image.fromarray(img_processed_mat, 'L')
  display(img_skyline_orig)
  display(img_processed)
```

Pretty cool, right? We will go into more detail on what's happening in the next section.

## Section 2.1: Demonstration of a CNN in PyTorch
At this point, you should have a fair idea of how to perform a convolution on an image given a kernel. In the following cell, we provide a code snippet that demonstrates setting up a convolutional network using PyTorch.

We look at the `nn` module in PyTorch. The `nn` module contains a plethora of functions that will make implementing a neural network easier. In particular we will look at the `nn.Conv2d()` function, which creates a convolutional layer that is applied to whatever image that you feed the resulting network.

Look at the code below. In it, we define a `Net` class that you can instantiate with a kernel to create a Neural Network object. When you apply the network object to an image (or anything in the form of a matrix), it convolves the kernel over that image.

```
class Net(nn.Module):
  """
  A convolutional neural network class.
  When an instance of it is constructed with a kernel, you can apply that instance
    to a matrix and it will convolve the kernel over that image.
  i.e. Net(kernel)(image)
  """
  def __init__(self, kernel=None, padding=0):
    super(Net, self).__init__()
    # Summary of the nn.conv2d parameters (you can also get this by hovering
    # over the method):
    # in_channels (int): Number of channels in the input image
    # out_channels (int): Number of channels produced by the convolution
    # kernel_size (int or tuple): Size of the convolving kernel
    # padding (int or tuple, optional): Zero-padding added to both sides of
    #     the input. Default: 0
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=1, kernel_size=2, \
                           padding=padding)

    # set up a default kernel if a default one isn't provided
    if kernel is not None:
      dim1, dim2 = kernel.shape[0], kernel.shape[1]
      kernel = kernel.reshape(1, 1, dim1, dim2)

      self.conv1.weight = torch.nn.Parameter(kernel)
      self.conv1.bias = torch.nn.Parameter(torch.zeros_like(self.conv1.bias))

  def forward(self, x):
    x = self.conv1(x)
    return x
```

```
# Format a default 2x2 kernel of numbers from 0 through 3
kernel = torch.Tensor(np.arange(4).reshape(2, 2))
# Prepare the network with that default kernel
net = Net(kernel=kernel, padding=0).to(DEVICE)

# set up a 3x3 image matrix of numbers from 0 through 8
image = torch.Tensor(np.arange(9).reshape(3, 3))
image = image.reshape(1, 1, 3, 3).to(DEVICE) # BatchSizeXChannelsXHeightXWidth

print("Image:\n" + str(image))
print("Kernel:\n" + str(kernel))
output = net(image) # Apply the convolution
print("Output:\n" + str(output))
```

As a quick aside, notice the difference in the input and output size. The input had a size of 3×3, and the output is of size 2×2. This is because of the fact that the kernel can't produce values for the edges of the image - when it slides to an end of the image and is centered on a border pixel, it overlaps space outside of the image that is undefined. If we don't want to lose that information, we will have to pad the image with some defaults (such as 0s) on the border. This process is, somewhat predictably, called *padding*. We will talk more about padding in the next section.

```
print("Image (before padding):\n" + str(image))
print("Kernel:\n" + str(kernel))
# Prepare the network with the aforementioned default kernel, but this
# time with padding
net = Net(kernel=kernel, padding=1).to(DEVICE)
output = net(image) # Apply the convolution onto the padded image
print("Output:\n" + str(output))
```

## Section 2.2: Padding and Edge Detection

Before we start in on the exercises, here's a visualization to help you think about padding.

### Interactive Demo 2.2: Visualization of Convolution with Padding and Stride
Recall that
* Padding adds rows and columns of zeros to the outside edge of an image
* Stride length adjusts the distance by which a filter is shifted after each convolution.

Change the padding and stride and see how this affects the shape of the output. How does the padding need to be configured to maintain the shape of the input?

```
%%html
# @markdown *Run this cell to enable the widget!*

<style>
  svg {
      #border: 1px solid black;
  }
  .matrix {
      font-family: sans-serif;
      transition: all 700ms ease-in-out;
  }
  .cell rect {
      fill:white;stroke-width:1;stroke:rgb(0,0,0)
  }
  .padding rect {
      stroke: rgba(0, 0, 0, 0.25);
  }
  .padding text {
      fill: lightgray;
  }
  .highlight1 {
      fill:none;stroke-width:4;stroke: rgb(236, 58, 58);stroke-dasharray:10,5;
  }
  .highlight2 {
      fill:rgba(229, 132, 66, 0.25);stroke-width:5;stroke: rgb(229, 132, 66);
  }
  .highlight3 {
      fill:rgba(236, 58, 58, 0.25);stroke-width:2;stroke: rgb(236, 58, 58);;
  }
  .title {
      text-anchor: middle;
  }
  .button_play {
      display: inline-block;
      background: none;
      border: none;
      position: relative;
      top: -3px;
  }
  .button_play path {
      fill: darkgray;
  }
  .button_play:hover path {
      fill: rgb(236, 58, 58);
  }
  .display_vis_input input:not(:hover)::-webkit-outer-spin-button,
  .display_vis_input input:not(:hover)::-webkit-inner-spin-button {
      /* display: none; <- Crashes Chrome on hover */
      -webkit-appearance: none;
      margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
  }

  .display_vis_input input:not(:hover)[type=number] {
      -moz-appearance:textfield; /* Firefox */
      width: 1ch;
      margin-right: 0px;
      z-index: 0;
  }
  .display_vis_input input[type=number] {
      width: 4ch;
      border: 0px;
      margin-right: -3ch;
      z-index: 6;
      display: inline-block;
      position: relative;
      padding: 0;
      border-bottom: 2px solid red;
      background: white;
      color: black
  }
  .display_vis_input .pair {
      display: inline-block;
      white-space:nowrap;
          position: relative;
  }
  .display_vis_input .pair .pair_hide {
      max-width: 4em;
      transition: max-width 1s ease-in;
      display: inline-block;
      overflow: hidden;
      position: relative;
      top: 5px;
  }
  .pair:not(:hover) .pair_hide {
      max-width: 0;
  }
  .pairX .pair_hide {
      max-width: 4em;
      transition: max-width 1s ease-in;
  }

  /* Dropdown Button */
  .dropbtn {
    border-bottom: 2px solid red;
  }

  /* The container <div> - needed to position the dropdown content */
  .dropdown {
    position: relative;
    display: inline-block;
  }

  /* Dropdown Content (Hidden by Default) */
  .dropdown-content {
    display: none;
    position: absolute;
    background-color: #f1f1f1;
    min-width: 160px;
    box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
    z-index: 1;
  }

  /* Links inside the dropdown */
  .dropdown-content a {
    color: black;
    padding: 5px 2px;
    text-decoration: none;
    display: block;
  }

  /* Change color of dropdown links on hover */
  .dropdown-content a:hover {background-color: #ddd;}

  /* Show the dropdown menu on hover */
  .dropdown:hover .dropdown-content {display: block;}
</style>

<div id="animation_conv_padding" style="background: white">
  <div class="display_vis_input language-python" style="font-family: monospace; color: black; padding: 10px;">
      <!-- default -- >
      import torch<br><br>
      input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="3">))<br>
      conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
      kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                      <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
      stride=1<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="hidden" min="1" max="2" value="1"><span class="pair_hide">,
                                                                <input class="input_stridey" type="hidden" min="1" max="2" value="1">)</span></span>,
      padding=0<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="hidden" min="0" max="4" value="0"><span class="pair_hide">,
                                                                  <input class="input_paddingy" type="hidden" min="0" max="4" value="0">)</span></span>)<br>
      result = conv(input)
      -->
      <!-- padding -->
      import torch<br><br>
      input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">)<br>
      conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
      kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                      <input class="input_filtery" type="number" min="2" max="4" value="3">)</span></span>,
      stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
      padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="1"><span class="pair_hide">,
                                                                  <input class="input_paddingy" type="number" min="0" max="4" value="1">)</span></span>)<br>
      result = conv(input)

<!--
      import torch<br><br>
      input = torch.rand(1, <input class="input_matrixz" type="hidden" min="1" max="3" value="1">1, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">))<br>
      conv = torch.nn.<div class="dropdown">
<div class="dropbtn">MaxPool2d</div>
<div class="dropdown-content">
  <a class="select_maxpool" href="#">MaxPool2d</a>
  <a class="select_avgpool" href="#">AvgPool2d</a>
</div>
</div>(<input class="input_matrixzB" type="hidden" min="1" max="3" value="1"><input class="input_filterz" type="hidden" min="1" max="3" value="1">kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                      <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
      stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="2"><span class="pair_hide">,
                                                                <input class="input_stridey" type="number" min="1" max="2" value="2">)</span></span>,
      padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                  <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
      result = conv(input)
-->
  </div>
      <button class="button_play play"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 1.5,0 9.5,5 1.5,10 z"/></svg></button>
  <button class="button_play pause" style="display: none"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 0,0 4,0 4,10, 0,10 z"/><path d="M 6,0 10,0 10,10, 6,10 z"/></svg></button>
  <input type="range" min="1" max="100" value="50" class="slider" style="width: 300px; display: inline-block">
  <button class="button_play left"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,5 4,0 4,10 z"/></svg></button>
  <button class="button_play right"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,0 4,5 0,10 z"/></svg></button>
  <input type="checkbox" class="play_fast">fast play mode
  <br/>
  <svg height="0" width="0">
      <defs>
  <marker id="arrowhead" markerWidth="10" markerHeight="7"
  refX="0" refY="1.5" orient="auto" fill="rgb(236, 58, 58)">
    <polygon points="0 0, 4 1.5, 0 3" />
  </marker>
</defs>
  </svg>
  <svg class="image" height="460" width="600">

  </svg>
</div>

```

### Think! 2.2: Edge Detection

One of the simpler tasks performed by a convolutional layer is edge detection; that is, finding a place in the image where there is a large and abrupt change in color. Edge-detecting filters are usually learned by the first layers in a CNN. Observe the following simple kernel and discuss whether this will detect vertical edges (where the trace of the edge is vertical; i.e. there is a boundary between left and right), or whether it will detect horizontal edges (where the trace of the edge is horizontal; i.e., there is a boundary between top and bottom).

\begin{equation}
\textbf{Kernel} =
\begin{bmatrix} 1 & -1 \\ 1 & -1
\end{bmatrix}
\end{equation}

```
# to_remove explanation

"""
It will detect vertical edges.

Consider a part of the image where values on the left are high and values on the
right are low. In this case, applying the kernel, the high values will be
maintained and the low values will be inverted, yielding a very-positive sum.

Now reverse the situation - values on the left are low and values on the right
are high. In this case, applying the kernel, the low values will be maintained
and the high values will be inverted, yielding a very-negative sum.

In any situation on the image where the left and right sides of a 2x2 block
/aren't/ much different, applying the kernel will result in decent degree of
cancellation - i.e. if it's all, say, 5, (5 + -5 + 5 + -5) = 0.

In other words, the only situations where you'll have an extremely positive or
extremely negative score are when you have a high value on either the left/right
and a low value on the other side.
"""
```

Consider the image below, which has a black vertical stripe with white on the side. This is like a very zoomed-in vertical edge within an image!

```
# Prepare an image that's basically just a vertical black stripe
X = np.ones((6, 8))
X[:, 2:6] = 0
print(X)
plt.imshow(X, cmap=plt.get_cmap('gray'))
plt.show()
```

```
# Format the image that's basically just a vertical stripe
image = torch.from_numpy(X)
image = image.reshape(1, 1, 6, 8) # BatchSize X Channels X Height X Width

# Prepare a 2x2 kernel with 1s in the first column and -1s in the
# This exact kernel was discussed above!
kernel = torch.Tensor([[1.0, -1.0], [1.0, -1.0]])
net = Net(kernel=kernel)

# Apply the kernel to the image and prepare for display
processed_image = net(image.float())
processed_image = processed_image.reshape(5, 7).detach().numpy()
print(processed_image)
plt.imshow(processed_image, cmap=plt.get_cmap('gray'))
plt.show()
```

As you can see, this kernel detects vertical edges (the black stripe corresponds to a highly positive result, while the white stripe corresponds to a highly negative result. However, to display the image, all the pixels are normalized between 0=black and 1=white).

If the kernel were transposed (i.e. the columns become rows and the rows become columns), what would the kernel detect? What would be produced by running this kernel on the vertical edge image above?

```
# to_remove explanation

"""
If the kernel were transposed, it would detect horizontal edges.

Running it on the vertical edge would result in 0s - you'd have no change
going from top to bottom, so you'd be adding and subtracting the same value.

(If you're confused as to why this shows up as black, rather than gray,
it's because of how the image is normalized)
"""
```

---
# Section 3: Pooling and Subsampling

```
# @title Video 4: Pooling
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1264y1z7JZ", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"XOss-NUlpo0", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

To visualize the various components of a CNN, we will build a simple CNN step by step. Recall that the MNIST dataset consists of binarized images of handwritten digits. This time we will be using the EMNIST letters dataset, which consists of binarized images of handwritten characters $(A, ..., Z)$.

We will simplify the problem further by only keeping the images that correspond to $X$ (labeled as `24` in the dataset) and $O$ (labeled as `15` in the dataset). Then, we will train a CNN to classify an image either an $X$ or an $O$.

```
# @markdown ### Dataset/DataLoader Functions (run me)

# loading the dataset
def get_Xvs0_dataset():
  transform = transforms.Compose([
      transforms.ToTensor(),
      #transforms.Normalize((0.1307,), (0.3081,))
      ])
  emnist_train = datasets.EMNIST(root='./data', split='letters', download=True, train=True, transform=transform)
  emnist_test = datasets.EMNIST(root='./data', split='letters', download=True, train=False, transform=transform)

  # only want O (15) and X (24) labels
  train_idx = (emnist_train.targets == 15) | (emnist_train.targets == 24)
  emnist_train.targets = emnist_train.targets[train_idx]
  emnist_train.data = emnist_train.data[train_idx]

  # convert Xs predictions to 1, Os predictions to 0
  emnist_train.targets = (emnist_train.targets == 24).type(torch.int64)

  test_idx = (emnist_test.targets == 15) | (emnist_test.targets == 24)
  emnist_test.targets = emnist_test.targets[test_idx]
  emnist_test.data = emnist_test.data[test_idx]

  # convert Xs predictions to 1, Os predictions to 0
  emnist_test.targets = (emnist_test.targets == 24).type(torch.int64)

  return emnist_train, emnist_test

def get_data_loaders(train_dataset, test_dataset, seed, batch_size=32):

  g_seed = torch.Generator()
  g_seed.manual_seed(seed)

  train_loader = DataLoader(train_dataset, batch_size=batch_size,
                        shuffle=True, num_workers=0,
                        worker_init_fn=seed_worker,
                        generator=g_seed)
  test_loader = DataLoader(test_dataset, batch_size=batch_size,
                        shuffle=True, num_workers=0,
                        worker_init_fn=seed_worker,
                        generator=g_seed)

  return train_loader, test_loader
```

```
emnist_train, emnist_test = get_Xvs0_dataset()
train_loader, test_loader = get_data_loaders(emnist_train, emnist_test, SEED)

# index of an image in the dataset that corresponds to an X and O
x_img_idx = 4
o_img_idx = 15
```

Let's view a couple samples from the dataset.

```
fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(12, 6))
ax1.imshow(emnist_train[0][0].reshape(28, 28), cmap='gray')
ax2.imshow(emnist_train[10][0].reshape(28, 28), cmap='gray')
ax3.imshow(emnist_train[4][0].reshape(28, 28), cmap='gray')
ax4.imshow(emnist_train[6][0].reshape(28, 28), cmap='gray')
plt.show()
```

```
# @markdown ### Visualization of Convolution with Multiple Filters
# @markdown Change the number of input channels (e.g., the color channels of an image or the output channels of a previous layer) and the output channels (number of different filters to apply).
%%html
<style>
    svg {
        #border: 1px solid black;
    }
.matrix {
    font-family: sans-serif;
    transition: all 700ms ease-in-out;
}
.cell rect {
    fill:white;stroke-width:1;stroke:rgb(0,0,0)
}
.padding rect {
    stroke: rgba(0, 0, 0, 0.25);
}
.padding text {
    fill: lightgray;
}
.highlight1 {
    fill:none;stroke-width:4;stroke: rgb(236, 58, 58);stroke-dasharray:10,5;
}
.highlight2 {
    fill:rgba(229, 132, 66, 0.25);stroke-width:5;stroke: rgb(229, 132, 66);
}
.highlight3 {
    fill:rgba(236, 58, 58, 0.25);stroke-width:2;stroke: rgb(236, 58, 58);;
}
.title {
    text-anchor: middle;
}
.button_play {
    display: inline-block;
    background: none;
    border: none;
    position: relative;
    top: -3px;
}
.button_play path {
    fill: darkgray;
}
.button_play:hover path {
    fill: rgb(236, 58, 58);
}
.display_vis_input input:not(:hover)::-webkit-outer-spin-button,
.display_vis_input input:not(:hover)::-webkit-inner-spin-button {
    /* display: none; <- Crashes Chrome on hover */
    -webkit-appearance: none;
    margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
}

.display_vis_input input:not(:hover)[type=number] {
    -moz-appearance:textfield; /* Firefox */
    width: 1ch;
    margin-right: 0px;
    z-index: 0;
}
.display_vis_input input[type=number] {
    width: 4ch;
    border: 0px;
    margin-right: -3ch;
    z-index: 6;
    display: inline-block;
    position: relative;
    padding: 0;
    border-bottom: 2px solid red;
    background: white;
    color: black
}
.display_vis_input .pair {
    display: inline-block;
    white-space:nowrap;
        position: relative;
}
.display_vis_input .pair .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
    display: inline-block;
    overflow: hidden;
    position: relative;
    top: 5px;
}
.pair:not(:hover) .pair_hide {
    max-width: 0;
}
.pairX .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
}

/* Dropdown Button */
.dropbtn {
  border-bottom: 2px solid red;
}

/* The container <div> - needed to position the dropdown content */
.dropdown {
  position: relative;
  display: inline-block;
}

/* Dropdown Content (Hidden by Default) */
.dropdown-content {
  display: none;
  position: absolute;
  background-color: #f1f1f1;
  min-width: 160px;
  box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
  z-index: 1;
}

/* Links inside the dropdown */
.dropdown-content a {
  color: black;
  padding: 5px 2px;
  text-decoration: none;
  display: block;
}

/* Change color of dropdown links on hover */
.dropdown-content a:hover {background-color: #ddd;}

/* Show the dropdown menu on hover */
.dropdown:hover .dropdown-content {display: block;}

</style>

<div id="animation_conv_filters" style="background: white">
    <div class="display_vis_input language-python" style="font-family: monospace; color: black; padding: 10px;">
        <!-- default -- >
        import torch<br><br>
        input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="3">))<br>
        conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=1<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="hidden" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="hidden" min="1" max="2" value="1">)</span></span>,
        padding=0<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="hidden" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="hidden" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)
        -->
        <!-- padding
        import torch<br><br>
        input = torch.rand(1, 1<input class="input_matrixz" type="hidden" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">))<br>
        conv = torch.nn.Conv2d(in_channels=1<input class="input_matrixzB" type="hidden" min="1" max="3" value="1">, out_channels=1<input class="input_filterz" type="hidden" min="1" max="3" value="1">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="3">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="1"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="1">)</span></span>)<br>
        result = conv(input)
-->
    <!-- filters -->
        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="number" min="1" max="3" value="3">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">)<br>
        conv = torch.nn.Conv2d(in_channels=<input class="input_matrixzB" type="number" min="1" max="3" value="3">, out_channels=<input class="input_filterz" type="number" min="1" max="3" value="2">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)

 <!--
        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="hidden" min="1" max="3" value="1">1, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">))<br>
        conv = torch.nn.<div class="dropdown">
  <div class="dropbtn">MaxPool2d</div>
  <div class="dropdown-content">
    <a class="select_maxpool" href="#">MaxPool2d</a>
    <a class="select_avgpool" href="#">AvgPool2d</a>
  </div>
</div>(<input class="input_matrixzB" type="hidden" min="1" max="3" value="1"><input class="input_filterz" type="hidden" min="1" max="3" value="1">kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="2"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="2">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)
-->
    </div>
        <button class="button_play play"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 1.5,0 9.5,5 1.5,10 z"/></svg></button>
    <button class="button_play pause" style="display: none"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 0,0 4,0 4,10, 0,10 z"/><path d="M 6,0 10,0 10,10, 6,10 z"/></svg></button>
    <input type="range" min="1" max="100" value="50" class="slider" style="width: 300px; display: inline-block">
    <button class="button_play left"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,5 4,0 4,10 z"/></svg></button>
    <button class="button_play right"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,0 4,5 0,10 z"/></svg></button>
    <input type="checkbox" class="play_fast">fast play mode
    <br/>
    <svg height="0" width="0">
        <defs>
    <marker id="arrowhead" markerWidth="10" markerHeight="7"
    refX="0" refY="1.5" orient="auto" fill="rgb(236, 58, 58)">
      <polygon points="0 0, 4 1.5, 0 3" />
    </marker>
  </defs>
    </svg>
    <svg class="image" height="460" width="600">

    </svg>
</div>

```

## Section 3.1: Multiple Filters

The following network sets up 3 filters and runs them on an image of the dataset from the $X$ class.  Note that we are using the filters from the videos!

```
class Net2(nn.Module):
  def __init__(self, kernel=None, padding=0):
    super(Net2, self).__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=3,
                           padding=padding)

    # first kernel - leading diagonal
    kernel_1 = torch.Tensor([[[1, -1, -1],
                              [-1, 1, -1],
                              [-1, -1, 1]]])

    # second kernel -checkerboard pattern
    kernel_2 = torch.Tensor([[[1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, 1]]])

    # third kernel - other diagonal
    kernel_3 = torch.Tensor([[[-1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, -1]]])

    multiple_kernels = torch.stack([kernel_1, kernel_2, kernel_3], dim=0)

    self.conv1.weight = torch.nn.Parameter(multiple_kernels)
    self.conv1.bias = torch.nn.Parameter(torch.Tensor([0, 0, 0]))

  def forward(self, x):
    x = self.conv1(x)
    return x
```

Now, let's visualize the filters using the code given below.

```
net2 = Net2().to(DEVICE)
fig, (ax11, ax12, ax13) = plt.subplots(1, 3)
# show the filters
ax11.set_title("filter 1")
ax11.imshow(net2.conv1.weight[0, 0].detach().cpu().numpy(), cmap="gray")
ax12.set_title("filter 2")
ax12.imshow(net2.conv1.weight[1, 0].detach().cpu().numpy(), cmap="gray")
ax13.set_title("filter 3")
ax13.imshow(net2.conv1.weight[2, 0].detach().cpu().numpy(), cmap="gray")
```

### Think! 3.1: Do you see how these filters would help recognize an `X`?

```
# to_remove explanation

"""
These filters are tailored to have strong responses to the features of an X and
weaker responses to the features of an O. The diagonal edge filters emphasize
the diagonal edges of the X and the checkerboard pattern reacts strongly to the
center of the X, while the more round edges of the O evoke weaker responses.

So the images which have strong downward (filter 1) and upward diagonals
(filter 3), as well as an intersection of both (filter 2) are probably an X.
"""
```

We apply the filters to the images.

```
net2 = Net2().to(DEVICE)
x_img = emnist_train[x_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_x = net2(x_img)
output_x = output_x.squeeze(dim=0).detach().cpu().numpy()

o_img = emnist_train[o_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_o = net2(o_img)
output_o = output_o.squeeze(dim=0).detach().cpu().numpy()
```

Let us view the image of $X$ and $O$ and what the output of the filters applied to them looks like. Pay special attention to the areas with very high vs. very low output patterns.

```
fig, ((ax11, ax12, ax13, ax14), (ax21, ax22, ax23, ax24), (ax31, ax32, ax33, ax34)) = plt.subplots(3, 4)
# show the filters
ax11.axis("off")
ax12.set_title("filter 1")
ax12.imshow(net2.conv1.weight[0, 0].detach().cpu().numpy(), cmap="gray")
ax13.set_title("filter 2")
ax13.imshow(net2.conv1.weight[1, 0].detach().cpu().numpy(), cmap="gray")
ax14.set_title("filter 3")
ax14.imshow(net2.conv1.weight[2, 0].detach().cpu().numpy(), cmap="gray")

# show x and the filters applied to x
ax21.set_title("image x")
ax21.imshow(emnist_train[x_img_idx][0].reshape(28, 28), cmap='gray')
ax22.set_title("output filter 1")
ax22.imshow(output_x[0], cmap='gray', vmin=-3.5, vmax=1)
ax23.set_title("output filter 2")
ax23.imshow(output_x[1], cmap='gray', vmin=-3.5, vmax=1)
ax24.set_title("output filter 3")
ax24.imshow(output_x[2], cmap='gray', vmin=-3.5, vmax=1)

# show o and the filters applied to o
ax31.set_title("image o")
ax31.imshow(emnist_train[o_img_idx][0].reshape(28, 28), cmap='gray')
ax32.set_title("output filter 1")
ax32.imshow(output_o[0], cmap='gray', vmin=-3.5, vmax=1)
ax33.set_title("output filter 2")
ax33.imshow(output_o[1], cmap='gray', vmin=-3.5, vmax=1)
ax34.set_title("output filter 3")
ax34.imshow(output_o[2], cmap='gray', vmin=-3.5, vmax=1)
plt.show()
```

## Section 3.2: ReLU after convolutions

Up until now we've talked about the convolution operation, which is linear. But the real strength of neural networks comes from the incorporation of non-linear functions.  Furthermore, in the real world, we often have learning problems where the relationship between the input and output is non-linear and complex.

The ReLU (Rectified Linear Unit) introduces non-linearity into our model, allowing us to learn a more complex function that can better predict the class of an image.

The ReLU function is shown below.

The Rectified Linear Unit (ReLU) Activation Function

Now let us incorporate ReLU into our previous model and visualize the output.

```
class Net3(nn.Module):
  def __init__(self, kernel=None, padding=0):
    super(Net3, self).__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=3,
                           padding=padding)

    # first kernel - leading diagonal
    kernel_1 = torch.Tensor([[[1, -1, -1],
                              [-1, 1, -1],
                              [-1, -1, 1]]])

    # second kernel -checkerboard pattern
    kernel_2 = torch.Tensor([[[1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, 1]]])

    # third kernel - other diagonal
    kernel_3 = torch.Tensor([[[-1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, -1]]])

    multiple_kernels = torch.stack([kernel_1, kernel_2, kernel_3], dim=0)

    self.conv1.weight = torch.nn.Parameter(multiple_kernels)
    self.conv1.bias = torch.nn.Parameter(torch.Tensor([0, 0, 0]))

  def forward(self, x):
    x = self.conv1(x)
    x = F.relu(x)
    return x
```

We apply the filters and relus to the images.

```
net3 = Net3().to(DEVICE)
x_img = emnist_train[x_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_x_relu = net3(x_img)
output_x_relu = output_x_relu.squeeze(dim=0).detach().cpu().numpy()

o_img = emnist_train[o_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_o_relu = net3(o_img)
output_o_relu = output_o_relu.squeeze(dim=0).detach().cpu().numpy()
```

Let us view the image of $X$ and $O$ and what the output of the filters applied to them look like.

```
# @markdown *Execute this cell to view the filtered images*
fig, ((ax11, ax12, ax13, ax14, ax15, ax16, ax17),
      (ax21, ax22, ax23, ax24, ax25, ax26, ax27),
      (ax31, ax32, ax33, ax34, ax35, ax36, ax37)) = plt.subplots(3, 4+3, figsize=(14,6),)
# show the filters
ax11.axis("off")
ax12.set_title("filter 1")
ax12.imshow(net2.conv1.weight[0, 0].detach().cpu().numpy(), cmap="gray")
ax13.set_title("filter 2")
ax13.imshow(net2.conv1.weight[1, 0].detach().cpu().numpy(), cmap="gray")
ax14.set_title("filter 3")
ax14.imshow(net2.conv1.weight[2, 0].detach().cpu().numpy(), cmap="gray")

ax15.set_title("filter 1")
ax15.imshow(net2.conv1.weight[0, 0].detach().cpu().numpy(), cmap="gray")
ax16.set_title("filter 2")
ax16.imshow(net2.conv1.weight[1, 0].detach().cpu().numpy(), cmap="gray")
ax17.set_title("filter 3")
ax17.imshow(net2.conv1.weight[2, 0].detach().cpu().numpy(), cmap="gray")

# show x and the filters applied to x
ax21.set_title("image x")
ax21.imshow(emnist_train[x_img_idx][0].reshape(28, 28), cmap='gray')
ax22.set_title("output filter 1")
ax22.imshow(output_x[0], cmap='gray', vmin=-3, vmax=1)
ax23.set_title("output filter 2")
ax23.imshow(output_x[1], cmap='gray', vmin=-3, vmax=1)
ax24.set_title("output filter 3")
ax24.imshow(output_x[2], cmap='gray', vmin=-3, vmax=1)

ax25.set_title("filter 1 + relu")
ax25.imshow(output_x_relu[0], cmap='gray', vmin=-3, vmax=1)
ax26.set_title("filter 2 + relu")
ax26.imshow(output_x_relu[1], cmap='gray', vmin=-3, vmax=1)
ax27.set_title("filter 3 + relu")
ax27.imshow(output_x_relu[2], cmap='gray', vmin=-3, vmax=1)

# show o and the filters applied to o
ax31.set_title("image o")
ax31.imshow(emnist_train[o_img_idx][0].reshape(28, 28), cmap='gray')
ax32.set_title("output filter 1")
ax32.imshow(output_o[0], cmap='gray', vmin=-3, vmax=1)
ax33.set_title("output filter 2")
ax33.imshow(output_o[1], cmap='gray', vmin=-3, vmax=1)
ax34.set_title("output filter 3")
ax34.imshow(output_o[2], cmap='gray', vmin=-3, vmax=1)

ax35.set_title("filter 1 + relu")
ax35.imshow(output_o_relu[0], cmap='gray', vmin=-3, vmax=1)
ax36.set_title("filter 2 + relu")
ax36.imshow(output_o_relu[1], cmap='gray', vmin=-3, vmax=1)
ax37.set_title("filter 3 + relu")
ax37.imshow(output_o_relu[2], cmap='gray', vmin=-3, vmax=1)
plt.show()
```

Discuss with your pod how the ReLU activations help strengthen the features necessary to detect an $X$.

[Here](https://stats.stackexchange.com/a/226927)'s an discussion which talks about how ReLU is useful as an activation funciton.

[Here](https://stats.stackexchange.com/questions/126238/what-are-the-advantages-of-relu-over-sigmoid-function-in-deep-neural-networks?sfb=2)'s another excellent discussion about the advantages of using ReLU.

## Section 3.3: Pooling

Convolutional layers create feature maps that summarize the presence of particular features (e.g. edges) in the input. However, these feature maps record the _precise_ position of features in the input. That means that small changes to the position of an object in an image can result in a very different feature map. But a cup is a cup (and an $X$ is an $X$) no matter where it appears in the image!  We need to achieve _translational invariance_.

A common approach to this problem is called downsampling. Downsampling creates a lower-resolution version of an image, retaining the large structural elements and removing some of the fine detail that may be less relevant to the task. In CNNs, Max-Pooling and Average-Pooling are used to downsample.  These operations shrink the size of the hidden layers, and produce features that are more translationally invariant, which can be better leveraged by subsequent layers.

Like convolutional layers, pooling layers have fixed-shape windows (pooling windows) that are systematically applied to the input.  As with filters, we can change the shape of the window and the size of the stride.  And, just like with filters, every time we apply a pooling operation we produce a single output.

Pooling performs a kind of information compression that provides summary statistics for a _neighborhood_ of the input.
- In Maxpooling, we compute the maximum value of all pixels in the pooling window.
- In Avgpooling, we compute the average value of all pixels in the pooling window.

The example below shows the result of Maxpooling within the yellow pooling windows to create the red pooling output matrix.

An Example of Pooling with a kernel size of 2

Pooling gives our network translational invariance by providing a summary of the values in each pooling window. Thus, A small change in the features of the underlying image won't make a huge difference to the output.

Note that, unlike a convolutional layer, the pooling layer contains no learned parameters! Pooling just computes a pre-determined summary of the input and passes that along.  This is in contrast to the convolutional layer, where there are filters to be learned.

### Interactive Demo 3.3: The effect of the stride

The following animation depicts how changing the stride changes the output. The stride defines how much the pooling region is moved over the input matrix to produce the next output (red arrows in the animation).  Give it a try! Change the stride and see how it affects the output shape.  You can also try MaxPool or AvgPool.

```
%%html
# @markdown *Run this cell to enable the widget!*

<style>
    svg {
        #border: 1px solid black;
    }
.matrix {
    font-family: sans-serif;
    transition: all 700ms ease-in-out;
}
.cell rect {
    fill:white;stroke-width:1;stroke:rgb(0,0,0)
}
.padding rect {
    stroke: rgba(0, 0, 0, 0.25);
}
.padding text {
    fill: lightgray;
}
.highlight1 {
    fill:none;stroke-width:4;stroke: rgb(236, 58, 58);stroke-dasharray:10,5;
}
.highlight2 {
    fill:rgba(229, 132, 66, 0.25);stroke-width:5;stroke: rgb(229, 132, 66);
}
.highlight3 {
    fill:rgba(236, 58, 58, 0.25);stroke-width:2;stroke: rgb(236, 58, 58);;
}
.title {
    text-anchor: middle;
}
.button_play {
    display: inline-block;
    background: none;
    border: none;
    position: relative;
    top: -3px;
}
.button_play path {
    fill: darkgray;
}
.button_play:hover path {
    fill: rgb(236, 58, 58);
}
.display_vis_input input:not(:hover)::-webkit-outer-spin-button,
.display_vis_input input:not(:hover)::-webkit-inner-spin-button {
    /* display: none; <- Crashes Chrome on hover */
    -webkit-appearance: none;
    margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
}

.display_vis_input input:not(:hover)[type=number] {
    -moz-appearance:textfield; /* Firefox */
    width: 1ch;
    margin-right: 0px;
    z-index: 0;
}
.display_vis_input input[type=number] {
    width: 4ch;
    border: 0px;
    margin-right: -3ch;
    z-index: 6;
    display: inline-block;
    position: relative;
    padding: 0;
    border-bottom: 2px solid red;
    background: white;
    color: black
}
.display_vis_input .pair {
    display: inline-block;
    white-space:nowrap;
        position: relative;
}
.display_vis_input .pair .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
    display: inline-block;
    overflow: hidden;
    position: relative;
    top: 5px;
}
.pair:not(:hover) .pair_hide {
    max-width: 0;
}
.pairX .pair_hide {
    max-width: 4em;
    transition: max-width 1s ease-in;
}

/* Dropdown Button */
.dropbtn {
  border-bottom: 2px solid red;
}

/* The container <div> - needed to position the dropdown content */
.dropdown {
  position: relative;
  display: inline-block;
}

/* Dropdown Content (Hidden by Default) */
.dropdown-content {
  display: none;
  position: absolute;
  background-color: #f1f1f1;
  min-width: 160px;
  box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
  z-index: 1;
}

/* Links inside the dropdown */
.dropdown-content a {
  color: black;
  padding: 5px 2px;
  text-decoration: none;
  display: block;
}

/* Change color of dropdown links on hover */
.dropdown-content a:hover {background-color: #ddd;}

/* Show the dropdown menu on hover */
.dropdown:hover .dropdown-content {display: block;}

</style>

<div id="animation_conv_pool" style="background: white">
    <div class="display_vis_input language-python" style="font-family: monospace; color: black; padding: 10px;">
        <!--
        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="number" min="1" max="3" value="1">, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="3">))<br>
        conv = torch.nn.Conv2d(in_channels=<input class="input_matrixzB" type="number" min="1" max="3" value="1">, out_channels=<input class="input_filterz" type="number" min="1" max="3" value="1">,
        kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="3"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="1"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="1">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)
        -->

        import torch<br><br>
        input = torch.rand(1, <input class="input_matrixz" type="hidden" min="1" max="3" value="1">1, <input class="input_matrixx" type="number" min="3" max="5" value="4">, <input class="input_matrixy" type="number" min="3" max="5" value="4">)<br>
        conv = torch.nn.<div class="dropdown">
  <div class="dropbtn">MaxPool2d</div>
  <div class="dropdown-content">
    <a class="select_maxpool">MaxPool2d</a>
    <a class="select_avgpool">AvgPool2d</a>
  </div>
</div>(<input class="input_matrixzB" type="hidden" min="1" max="3" value="1"><input class="input_filterz" type="hidden" min="1" max="3" value="1">kernel_size=<span class="pair"><span class="pair_hide">(</span><input class="input_filterx" type="number" min="2" max="4" value="2"><span class="pair_hide">,
                                                                       <input class="input_filtery" type="number" min="2" max="4" value="2">)</span></span>,
        stride=<span class="pair"><span class="pair_hide">(</span><input class="input_stridex" type="number" min="1" max="2" value="2"><span class="pair_hide">,
                                                                  <input class="input_stridey" type="number" min="1" max="2" value="2">)</span></span>,
        padding=<span class="pair"><span class="pair_hide">(</span><input class="input_paddingx" type="number" min="0" max="4" value="0"><span class="pair_hide">,
                                                                   <input class="input_paddingy" type="number" min="0" max="4" value="0">)</span></span>)<br>
        result = conv(input)

    </div>
        <button class="button_play play"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 1.5,0 9.5,5 1.5,10 z"/></svg></button>
    <button class="button_play pause" style="display: none"><svg width="15" height="15" viewbox="0 0 10 10"><path d="M 0,0 4,0 4,10, 0,10 z"/><path d="M 6,0 10,0 10,10, 6,10 z"/></svg></button>
    <input type="range" min="1" max="100" value="50" class="slider" style="width: 300px; display: inline-block">
    <button class="button_play left"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,5 4,0 4,10 z"/></svg></button>
    <button class="button_play right"><svg width="7" height="15" viewbox="0 0 4 10"><path d="M 0,0 4,5 0,10 z"/></svg></button>
    <input type="checkbox" class="play_fast">fast play mode
    <br/>
    <svg height="0" width="0">
        <defs>
    <marker id="arrowhead" markerWidth="10" markerHeight="7"
    refX="0" refY="1.5" orient="auto" fill="rgb(236, 58, 58)">
      <polygon points="0 0, 4 1.5, 0 3" />
    </marker>
  </defs>
    </svg>
    <svg class="image" height="460" width="600">

    </svg>
</div>

```

### Coding Exercise 3.3: Implement MaxPooling

Let us now implement MaxPooling in PyTorch and observe the effects of Pooling on the dimension of the input image. Use a kernel of size 2 and stride of 2 for the MaxPooling layer.

```
class Net3(nn.Module):
  def __init__(self, kernel=None, padding=0, stride=2):
    super(Net3, self).__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=3,
                            padding=padding)

    # first kernel - leading diagonal
    kernel_1 = torch.Tensor([[[1, -1, -1],
                              [-1, 1, -1],
                              [-1, -1, 1]]])

    # second kernel -checkerboard pattern
    kernel_2 = torch.Tensor([[[1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, 1]]])

    # third kernel - other diagonal
    kernel_3 = torch.Tensor([[[-1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, -1]]])

    multiple_kernels = torch.stack([kernel_1, kernel_2, kernel_3], dim=0)

    self.conv1.weight = torch.nn.Parameter(multiple_kernels)
    self.conv1.bias = torch.nn.Parameter(torch.Tensor([0, 0, 0]))
    ####################################################################
    # Fill in missing code below (...),
    # then remove or comment the line below to test your function
    raise NotImplementedError("Define the maxpool layer")
    ####################################################################
    self.pool = nn.MaxPool2d(kernel_size=..., stride=...)

  def forward(self, x):
    x = self.conv1(x)
    x = F.relu(x)
    ####################################################################
    # Fill in missing code below (...),
    # then remove or comment the line below to test your function
    raise NotImplementedError("Define the maxpool layer")
    ####################################################################
    x = ...  # pass through a max pool layer
    return x

## check if your implementation is correct
# net = Net3().to(DEVICE)
# check_pooling_net(net, device=DEVICE)
```

```
# to_remove solution
class Net3(nn.Module):
  def __init__(self, kernel=None, padding=0, stride=2):
    super(Net3, self).__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=3,
                            padding=padding)

    # first kernel - leading diagonal
    kernel_1 = torch.Tensor([[[1, -1, -1],
                              [-1, 1, -1],
                              [-1, -1, 1]]])

    # second kernel -checkerboard pattern
    kernel_2 = torch.Tensor([[[1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, 1]]])

    # third kernel - other diagonal
    kernel_3 = torch.Tensor([[[-1, -1, 1],
                              [-1, 1, -1],
                              [1, -1, -1]]])

    multiple_kernels = torch.stack([kernel_1, kernel_2, kernel_3], dim=0)

    self.conv1.weight = torch.nn.Parameter(multiple_kernels)
    self.conv1.bias = torch.nn.Parameter(torch.Tensor([0, 0, 0]))
    self.pool = nn.MaxPool2d(kernel_size=2, stride=stride)

  def forward(self, x):
    x = self.conv1(x)
    x = F.relu(x)
    x = self.pool(x)  # pass through a max pool layer
    return x

## check if your implementation is correct
net = Net3().to(DEVICE)
check_pooling_net(net, device=DEVICE)
```

```
✅ Your network produced the correct output.
```

```
x_img = emnist_train[x_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_x_pool = net(x_img)
output_x_pool = output_x_pool.squeeze(dim=0).detach().cpu().numpy()

o_img = emnist_train[o_img_idx][0].unsqueeze(dim=0).to(DEVICE)
output_o_pool = net(o_img)
output_o_pool = output_o_pool.squeeze(dim=0).detach().cpu().numpy()
```

```
# @markdown *Run the cell to plot the outputs!*

fig, ((ax11, ax12, ax13, ax14), (ax21, ax22, ax23, ax24), (ax31, ax32, ax33, ax34)) = plt.subplots(3, 4)
# show the filters
ax11.axis("off")
ax12.set_title("filter 1")
ax12.imshow(net2.conv1.weight[0, 0].detach().cpu().numpy(), cmap="gray")
ax13.set_title("filter 2")
ax13.imshow(net2.conv1.weight[1, 0].detach().cpu().numpy(), cmap="gray")
ax14.set_title("filter 3")
ax14.imshow(net2.conv1.weight[2, 0].detach().cpu().numpy(), cmap="gray")

# show x and the filters applied to x
ax21.set_title("image x")
ax21.imshow(emnist_train[x_img_idx][0].reshape(28, 28), cmap='gray')
ax22.set_title("output filter 1")
ax22.imshow(output_x_pool[0], cmap='gray', vmin=-3.5, vmax=1)
ax23.set_title("output filter 2")
ax23.imshow(output_x_pool[1], cmap='gray', vmin=-3.5, vmax=1)
ax24.set_title("output filter 3")
ax24.imshow(output_x_pool[2], cmap='gray', vmin=-3.5, vmax=1)

# show o and the filters applied to o
ax31.set_title("image o")
ax31.imshow(emnist_train[o_img_idx][0].reshape(28, 28), cmap='gray')
ax32.set_title("output filter 1")
ax32.imshow(output_o_pool[0], cmap='gray', vmin=-3.5, vmax=1)
ax33.set_title("output filter 2")
ax33.imshow(output_o_pool[1], cmap='gray', vmin=-3.5, vmax=1)
ax34.set_title("output filter 3")
ax34.imshow(output_o_pool[2], cmap='gray', vmin=-3.5, vmax=1)
plt.show()
```

You should observe the size of the output as being half of what you saw after the ReLU section, which is due to the Maxpool layer.

Despite the reduction in the size of the output, the important or high-level features in the output still remains intact.

---
# Section 4: Putting it all together

```
# @title Video 5: Putting it all together
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1Fy4y1j7dU", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"-TJixd9fRCw", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

## Section 4.1: Number of Parameters in Convolutional vs. Fully-connected Models
Convolutional networks encourage weight-sharing by learning a single kernel that is repeated over the entire input image. In general, this kernel is just a few parameters, compared to the huge number of parameters in a dense network.

Let's use the animation below to calculate few-layer network parameters for image data of shape $32\times32$ using both convolutional layers and dense layers. The `Num_Dense` in this exercise is the number of dense layers we use in the network, with each dense layer having the same input and output dimensions. `Num_Convs` is the number of convolutional blocks in the network, with each block containing a single kernel. The kernel size is the length and width of this kernel.

**Note:** you must run the cell before you can use the sliders.

 Parameter comparison

### Interactive Demo 4.1: Number of Parameters

```
# @markdown *Run this cell to enable the widget*

#mar kdown **Input**: batch of 100 images of shape 'image_size'x'image_size'

image_size = 96 #pa ram {type:"slider", min:32, max:128, step:32}

#markdo wn ## Dense Network Parameters

batch_size = 100 #para m {type:"slider", min:10, max:100, step:10}

#print("Input Shape {}".format(sample_image.shape))

number_of_Linear = 1 #p aram {type:"slider", min:1, max:3, step:1}

#markdow n ## Convolution Network Parameters

number_of_Conv2d = 1 #para m {type:"slider", min:1, max:5, step:1}

kernel_size = 5 #pa ram {type:"slider", min:3, max:21, step:2}

pooling = True #param {type:"boolean"}

#markdown ## Final Layer

Final_Layer = True #param {type:"boolean"}

def do_plot(image_size, batch_size, number_of_Linear, number_of_Conv2d,
            kernel_size, pooling, Final_Layer):

    sample_image = torch.rand(batch_size,1,image_size,image_size)
    linear_layer = []
    linear_nets = []
    code_dense = ""
    # code_dense += f"input = torch.rand(<span class='mtk11'>{batch_size}</span>, 1, {image_size}, {image_size})\n"
    code_dense += f"model_dense = nn.Sequential(\n"
    code_dense += f"    nn.Flatten(),\n"
    for i in range(number_of_Linear):
        linear_layer.append(nn.Linear(image_size*image_size*1,image_size*image_size*1,bias=False))
        linear_nets.append(nn.Sequential(*linear_layer))
        code_dense += f"    nn.Linear({image_size}*{image_size}*1, {image_size}*{image_size}*1, bias=False),\n"
    if Final_Layer is True:
        linear_layer.append(nn.Linear(image_size*image_size*1, 10, bias=False))
        linear_nets.append(nn.Sequential(*linear_layer))
        code_dense += f"    nn.Linear({image_size}*{image_size}*1, 10, bias=False)\n"
    code_dense += ")\n"
    code_dense += "result_dense = model_dense(sample_image)\n"
    linear_layer = nn.Sequential(*linear_layer)

    conv_layer = []
    conv_nets = []
    code_conv = ""
    # code_conv += f"input = torch.rand({batch_size}, 1, {image_size}, {image_size})\n"
    code_conv += f"model_conv = nn.Sequential(\n"
    for i in range(number_of_Conv2d):
        conv_layer.append(nn.Conv2d(in_channels=1,out_channels=1,kernel_size=kernel_size,padding=kernel_size//2,bias=False))
        conv_nets.append(nn.Sequential(*conv_layer))
        code_conv += f"    nn.Conv2d(in_channels=1, out_channels=1, kernel_size={kernel_size}, padding={kernel_size//2}, bias=False),\n"
        if pooling > 0:
            conv_layer.append(nn.MaxPool2d(2, 2))
            code_conv += f"    nn.MaxPool2d(2, 2),\n"
        conv_nets.append(nn.Sequential(*conv_layer))
    if Final_Layer is True:
        conv_layer.append(nn.Flatten())
        code_conv += f"    nn.Flatten(),\n"
        conv_nets.append(nn.Sequential(*conv_layer))
        shape_conv = conv_nets[-1](sample_image).shape
        conv_layer.append(nn.Linear(shape_conv[1], 10, bias=False))
        code_conv += f"    nn.Linear({shape_conv[1]}, 10, bias=False),\n"
        conv_nets.append(nn.Sequential(*conv_layer))
    conv_layer = nn.Sequential(*conv_layer)
    code_conv += ")\n"
    code_conv += "result_conv = model_conv(sample_image)\n"

    t_1=time.time()
    shape_linear = linear_layer(torch.flatten(sample_image,1)).shape
    #print("\nOutput From Linear {}".format(shape_linear))
    t_2=time.time()
    shape_conv = conv_layer(sample_image).shape
    #print("Output From Conv Layer {}".format(shape_conv))
    t_3=time.time()
    #print("")
    print("Time taken by Dense Layer {}".format(t_2-t_1))
    print("Time taken by Conv Layer  {}".format(t_3-t_2))

    #print("")
    #print("-------------------------------------------")
    #print("Total Parameters in Linear Layer {:10d}".format(sum(p.numel() for p in linear_layer.parameters())))
    #print("Total Parameters in Conv Layer   {:10d}".format(sum(p.numel() for p in conv_layer.parameters())))
    #print("-------------------------------------------")

    ax = plt.axes((0, 0, 1, 1))
    ax.spines["left"].set_visible(False)
    plt.yticks([])
    ax.spines["bottom"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.spines["top"].set_visible(False)
    plt.xticks([])
    p1 = sum(p.numel() for p in linear_layer.parameters())
    nl = '\n'
    p2 = sum(p.numel() for p in conv_layer.parameters())
    plt.text(0.1, 0.8, f"Total Parameters in Dense Layer {p1:10,d}{nl}Total Parameters in Conv Layer   {p2:10,d}")

    plt.text(0.23,0.62, "Dense Net", rotation=90, color='k', ha="center", va="center")

    def addBox(x, y, w, h, color, text1, text2, text3):
        ax.add_patch(plt.Rectangle((x,y),w,h,
                                        fill=True, color=color, alpha=0.5, zorder=1000, clip_on=False
                                        ))
        plt.text(x + 0.02, y+h/2, text1, rotation=90, va="center", ha="center", size=12)
        plt.text(x + 0.05, y+h/2, text2, rotation=90, va="center", ha="center")
        plt.text(x + 0.08, y+h/2, text3, rotation=90, va="center", ha="center", size=12)

    x = 0.25
    if 1:
        addBox(x,0.5, 0.08, 0.25, [1, 0.5, 0],
            "Flatten",
            tuple(torch.flatten(sample_image,1).shape),
            ""
            )
        x += 0.08+0.01

    for i in range(number_of_Linear):
        addBox(x,0.5, 0.1, 0.25, "g",
            "Dense",
            tuple(linear_nets[i](torch.flatten(sample_image,1)).shape),
            list(linear_layer.parameters())[i].numel()
            )
        x += 0.11

    if Final_Layer is True:
        i = number_of_Linear
        addBox(x,0.5, 0.1, 0.25, "g",
            "Dense",
            tuple(linear_nets[i](torch.flatten(sample_image,1)).shape),
            list(linear_layer.parameters())[i].numel()
            )

    plt.text(0.23,0.1+0.35/2, "Conv Net", rotation=90, color='k', ha="center", va="center")
    x = 0.25
    for i in range(number_of_Conv2d):
        addBox(x,0.1, 0.1, 0.35, "r",
            "Conv",
            tuple(conv_nets[i*2](sample_image).shape),
            list(conv_nets[i*2].parameters())[-1].numel()
            )
        x += 0.11
        if pooling > 0:
            addBox(x,0.1, 0.08, 0.35, [0, 0.5, 1],
            "Pooling",
            tuple(conv_nets[i*2+1](sample_image).shape),
            ""
            )
            x += 0.08+0.01

    if Final_Layer is True:
        i = number_of_Conv2d
        addBox(x,0.1, 0.08, 0.35, [1, 0.5, 0],
            "Flatten",
            tuple(conv_nets[i*2](sample_image).shape),
            ""
            )
        x += 0.08+0.01
        addBox(x,0.1, 0.1, 0.35, "g",
            "Dense",
            tuple(conv_nets[i*2+1](sample_image).shape),
            list(conv_nets[i*2+1].parameters())[-1].numel()
            )
        x += 0.11

    plt.text(0.08,0.3+0.35/2, "Input", rotation=90, color='b', ha="center", va="center")
    ax.add_patch(plt.Rectangle((0.1,0.3),0.1,0.35,
                                        fill=True, color='b', alpha=0.5, zorder=1000,
                                        clip_on=False))
    plt.text(0.1+0.1/2,0.3+0.35/2, tuple(sample_image.shape), rotation=90, va="center", ha="center")
    #plt.xlim(0.8, 1)
    #plt.ylim(0.05, 0.8)
    import io, base64
    plt.gcf().set_tight_layout(False)
    my_stringIObytes = io.BytesIO()
    plt.savefig(my_stringIObytes, format='png', dpi=90)
    my_stringIObytes.seek(0)
    my_base64_jpgData = base64.b64encode(my_stringIObytes.read())
    del linear_layer,conv_layer
    plt.close()
    return code_dense, code_conv, """"""

from ipywidgets import interact, interactive, fixed, interact_manual
%matplotlib inline

caption = widgets.Label(value='The values of range1 and range2 are synchronized')
slider_batch_size = widgets.IntSlider(value=100, min=10, max=100, step=10, description="BatchSize")
slider_image_size = widgets.IntSlider(value=32, min=32, max=128, step=32, description="ImageSize")
slider_number_of_Linear = widgets.IntSlider(value=1, min=1, max=3, step=1, description="NumDense")
slider_number_of_Conv2d = widgets.IntSlider(value=1, min=1, max=5, step=1, description="NumConv")
slider_kernel_size = widgets.IntSlider(value=5, min=3, max=21, step=2, description="KernelSize")
input_pooling = widgets.Checkbox(value=False, description="Pooling")
input_Final_Layer = widgets.Checkbox(value=False, description="Final_Layer")

output_code1 = widgets.HTML(
    value="",
)

output_plot = widgets.HTML(
    value="",
)

image_size = 96 #pa ram {type:"slider", min:32, max:128, step:32}
#markdo wn ## Dense Network Parameters
batch_size = 100 #para m {type:"slider", min:10, max:100, step:10}
#print("Input Shape {}".format(sample_image.shape))
number_of_Linear = 1 #p aram {type:"slider", min:1, max:3, step:1}
#markdow n ## Convolution Network Parameters
number_of_Conv2d = 1 #para m {type:"slider", min:1, max:5, step:1}
kernel_size = 5 #pa ram {type:"slider", min:3, max:21, step:2}
pooling = True #param {type:"boolean"}
#markdown ## Final Layer
Final_Layer = True #param {type:"boolean"}

def plot_func(batch_size, image_size, number_of_Linear, number_of_Conv2d, kernel_size,
              pooling, Final_Layer
              ):
    code1, code2, plot = do_plot(
        image_size, batch_size, number_of_Linear, number_of_Conv2d,
            kernel_size, pooling, Final_Layer)
    output_plot.value = plot
    output_code1.value = """
 <!DOCTYPE html>
<html>
<head>
<style>
* {
    box-sizing: border-box;
}

.column {
    float: left;
    /*width: 33.33%;*/
    padding: 5px;
}

/* Clearfix (clear floats) */
.row::after {
    content: "";
    clear: both;
    display: table;
}

pre {
    line-height: 1.2em;
}
</style>
</head>
<body>

<div class="row">
    <div class="column" style="overflow-x: scroll;">
    <h2>Code for Dense Network</h2>
  <pre>"""+code1+"""</pre>
  </div>
    <div class="column" style="overflow-x: scroll;">
        <h2>Code for Conv Network</h2>
  <pre>"""+code2+"""</pre>
  </div>
</div>
</body>
</html>
"""
    #plt.clf()
    #x = np.linspace(0, 2*np.pi)
    #y = np.sin(x * freq)
    #plt.plot(x, y)

#slider.observe(plot_func, names='value')

#display(caption, slider)
out = widgets.interactive_output(plot_func, {
    "batch_size": slider_batch_size,
    "image_size": slider_image_size,
    "number_of_Linear": slider_number_of_Linear,
    "number_of_Conv2d": slider_number_of_Conv2d,
    "kernel_size": slider_kernel_size,
    "pooling": input_pooling,
    "Final_Layer": input_Final_Layer,
})

ui = widgets.VBox([slider_batch_size, slider_image_size, slider_number_of_Linear, widgets.HBox([slider_number_of_Conv2d, slider_kernel_size, input_pooling]), input_Final_Layer])
display(widgets.HBox([output_plot, output_code1]), ui)
display(out)
```

The difference in parameters is huge, and it continues to increase as the input image size increases. Larger images require that the linear layer use a matrix that can be directly multiplied with the input pixels.

While pooling does not reduce the number of parameters for a subsequent convolutional layer, it does decreases the image size. Therefore, later dense layers will need fewer parameters.

The CNN parameter size, however, is invariant of the image size, as irrespective of the input that it gets, it keeps sliding the same learnable filter over the images.

The reduced parameter set not only brings down memory usage by huge chunks, but it also allows the model to generalize better.

```
# @title Video 6: Implement your own CNN
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV18f4y1j7e4", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"_gkF9Vv7MgE", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

## Coding Exercise 4: Implement your own CNN

Let's stack up all we have learnt. Create a CNN with the following structure.

- Convolution `nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3)`
- Convolution `nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3)`
- Pool Layer `nn.MaxPool2d(kernel_size=2)`
- Fully Connected Layer `nn.Linear(in_features=9216, out_features=128)`
- Fully Connected layer `nn.Linear(in_features=128, out_features=2)`

Note: As discussed in the video, we would like to flatten the output from the Convolutional Layers before passing on the Linear layers, thereby converting an input of shape [BatchSize, Channels, Height, Width] to [BatchSize, Channels\\*Height\\*Width], which in this case would be from [32, 64, 12, 12] (output of second convolution layer) to [32, 64\*12\*12] = [32, 9216]. Recall that the input images have size [28, 28].

Hint: You could use `torch.flatten(x, 1)` in order to flatten the input at this stage. The 1 means it flattens dimensions starting with dimensions 1 in order to exclude the batch dimension from the flattening.

We should also stop to think about how we get the output of the pooling layer to be 12x12. It is because first, the two `Conv2d` with a `kernel_size=3` operations cause the image to be reduced to 26x26 and the second `Conv2d` reduces it to 24x24. Finally, the `MaxPool2d` operation reduces the output size by half to 12x12.

Also, don't forget the ReLUs (use e.g. `F.ReLU`)! No need to add a ReLU after the final fully connected layer.

```
# @markdown ### Train/Test Functions (Run Me)
# @markdown  Double-click to see the contents!

# loading the dataset
def get_Xvs0_dataset():
  transform = transforms.Compose([
      transforms.ToTensor(),
      transforms.Normalize((0.1307,), (0.3081,))
      ])
  emnist_train = datasets.EMNIST(root='./data', split='letters',
                                 download=True, train=True, transform=transform)
  emnist_test = datasets.EMNIST(root='./data', split='letters',
                                download=True, train=False, transform=transform)

  # only want O (15) and X (24) labels
  train_idx = (emnist_train.targets == 15) | (emnist_train.targets == 24)
  emnist_train.targets = emnist_train.targets[train_idx]
  emnist_train.data = emnist_train.data[train_idx]

  # convert Xs predictions to 1, Os predictions to 0
  emnist_train.targets = (emnist_train.targets == 24).type(torch.int64)

  test_idx = (emnist_test.targets == 15) | (emnist_test.targets == 24)
  emnist_test.targets = emnist_test.targets[test_idx]
  emnist_test.data = emnist_test.data[test_idx]

  # convert Xs predictions to 1, Os predictions to 0
  emnist_test.targets = (emnist_test.targets == 24).type(torch.int64)

  return emnist_train, emnist_test

def get_data_loaders(train_dataset, test_dataset, batch_size=32, seed=0):
  g = torch.Generator()
  g.manual_seed(seed)

  train_loader = DataLoader(train_dataset, batch_size=batch_size,
                            shuffle=True, num_workers=0,
                            worker_init_fn=seed_worker,
                            generator=g)
  test_loader = DataLoader(test_dataset, batch_size=batch_size,
                           shuffle=True, num_workers=0,
                           worker_init_fn=seed_worker,
                           generator=g)

  return train_loader, test_loader

def train(model, device, train_loader, epochs):
  model.train()

  criterion = nn.CrossEntropyLoss()
  optimizer = torch.optim.SGD(model.parameters(),
                            lr=0.01)
  for epoch in range(epochs):
    with tqdm(train_loader, unit='batch') as tepoch:
      for data, target in tepoch:
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)

        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        tepoch.set_postfix(loss=loss.item())
        sleep(0.1)

def test(model, device, data_loader):
  model.eval()
  correct = 0
  total = 0
  for data in data_loader:
    inputs, labels = data
    inputs = inputs.to(device).float()
    labels = labels.to(device).long()

    outputs = model(inputs)
    _, predicted = torch.max(outputs, 1)
    total += labels.size(0)
    correct += (predicted == labels).sum().item()

  acc = 100 * correct / total
  return acc
```

```
class EMNIST_Net(nn.Module):
  def __init__(self):
    super(EMNIST_Net, self).__init__()

    ####################################################################
    # Fill in missing code below (...),
    # then remove or comment the line below to test your function
    raise NotImplementedError("Define the required layers")
    ####################################################################
    self.conv1 = nn.Conv2d(...)
    self.conv2 = nn.Conv2d(...)
    self.fc1 = nn.Linear(...)
    self.fc2 = nn.Linear(...)
    self.pool = nn.MaxPool2d(...)

  def forward(self, x):
    ####################################################################
    # Fill in missing code below (...),
    # then remove or comment the line below to test your function
    # Hint: Do not forget to flatten the image as it goes from
    # Convolution Layers to Linear Layers!
    raise NotImplementedError("Define forward pass for any input x")
    ####################################################################
    x = self.conv1(x)
    x = F.relu(x)
    x = self.conv2(x)
    x = ...
    x = ...
    x = ...
    x = ...
    x = ...
    x = ...
    return x

set_seed(SEED)
emnist_train, emnist_test = get_Xvs0_dataset()
train_loader, test_loader = get_data_loaders(emnist_train, emnist_test, seed=SEED)
## Uncomment the lines below to train your network
# emnist_net = EMNIST_Net().to(DEVICE)
# print("Total Parameters in Network {:10d}".format(sum(p.numel() for p in emnist_net.parameters())))
# train(emnist_net, DEVICE, train_loader, 1)
```

```
# to_remove solution
class EMNIST_Net(nn.Module):
  def __init__(self):
    super(EMNIST_Net, self).__init__()
    self.conv1 = nn.Conv2d(1, 32, 3)
    self.conv2 = nn.Conv2d(32, 64, 3)
    self.fc1 = nn.Linear(9216, 128)
    self.fc2 = nn.Linear(128, 2)
    self.pool = nn.MaxPool2d(2)

  def forward(self, x):
    x = self.conv1(x)
    x = F.relu(x)
    x = self.conv2(x)
    x = F.relu(x)
    x = self.pool(x)
    x = torch.flatten(x, 1)
    x = self.fc1(x)
    x = F.relu(x)
    x = self.fc2(x)
    return x

set_seed(SEED)
emnist_train, emnist_test = get_Xvs0_dataset()
train_loader, test_loader = get_data_loaders(emnist_train, emnist_test, seed=SEED)
## Uncomment the lines below to train your network
emnist_net = EMNIST_Net().to(DEVICE)
print("Total Parameters in Network {:10d}".format(sum(p.numel() for p in emnist_net.parameters())))
train(emnist_net, DEVICE, train_loader, 1)
```

Now, let's run the network on the test data!

```
test(emnist_net, DEVICE, test_loader)
```

You should have been able to get a test accuracy of around $99%$!

NOTE: We are using a softmax function here which converts a real value to a value between 0 and 1, which can be interpreted as a probability.

```
# index of an image in the dataset that corresponds to an X and O
x_img_idx = 11
o_img_idx = 0

print("Input:")
x_img = emnist_train[x_img_idx][0].unsqueeze(dim=0).to(DEVICE)
plt.imshow(emnist_train[x_img_idx][0].reshape(28, 28),
           cmap=plt.get_cmap('gray'))
plt.show()
output = emnist_net(x_img)
result = F.softmax(output, dim=1)
print("\nResult:", result)
print("Confidence of image being an 'O':", result[0, 0].item())
print("Confidence of image being an 'X':", result[0, 1].item())
```

The network is quite confident that this image is an $X$!

Note that this is evident from the softmax output, which shows the probabilities of the image belonging to each of the classes. There is a higher probability of belonging to class 1; i.e., class $X$.

Let us also test the network on an $O$ image.

```
print("Input:")
o_img = emnist_train[o_img_idx][0].unsqueeze(dim=0).to(DEVICE)
plt.imshow(emnist_train[o_img_idx][0].reshape(28, 28),
           cmap=plt.get_cmap('gray'))
plt.show()
output = emnist_net(o_img)
result = F.softmax(output, dim=1)
print("\nResult:", result)
print("Confidence of image being an 'O':", result[0, 0].item())
print("Confidence of image being an 'X':", result[0, 1].item())
```

---
# Summary

In this Tutorial we have familiarized ouselves with CNNs. We have leaned how the convolution operation works and be applied in various images. Also, we have learned to implement our own CNN. In the next Tutorial, we will go deeper in the training of CNNs!