**Agenda:**

For this tutorial in  Deep Learning(DL) with Pytorch, we are going to explore Multi Layered Perceptron architecture and learn Pytorch by implementing  algorithms under a certain usecase.We will cover the following:
1. Deep Learning basics with Pytorch
2. Multilayered Perceptron (MLP) implemention on  MNIST

Lets get started !!

# **1. Deep Learning basics with Pytorch**

In this part we will cover the following:

1. Learn to play with tensors on numpy and pytorch
2. Learn to build a simple feed forward network from scratch with random data
3. Learn to build an end to end MLP for MNIST dataset

##  Imports

```python
#!pip3 install torch torchvision

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
#print("List of files",os.listdir("../input"))
import torch
import numpy as np
print("Torch Version:",torch.__version__)
# Any results you write to the current directory are saved as output.
```

    Torch Version: 1.1.0

## Helper Functions

```python
import matplotlib.pyplot as plt
import numpy as np
from torch import nn, optim
from torch.autograd import Variable

def test_network(net, trainloader):

    criterion = nn.MSELoss()
    optimizer = optim.Adam(net.parameters(), lr=0.001)

    dataiter = iter(trainloader)
    images, labels = dataiter.next()

    # Create Variables for the inputs and targets
    inputs = Variable(images)
    targets = Variable(images)

    # Clear the gradients from all Variables
    optimizer.zero_grad()

    # Forward pass, then backward pass, then update weights
    output = net.forward(inputs)
    loss = criterion(output, targets)
    loss.backward()
    optimizer.step()

    return True

def imshow(image, ax=None, title=None, normalize=True):
    """Imshow for Tensor."""
    if ax is None:
        fig, ax = plt.subplots()
    image = image.numpy().transpose((1, 2, 0))

    if normalize:
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])
        image = std * image + mean
        image = np.clip(image, 0, 1)

    ax.imshow(image)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['left'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.tick_params(axis='both', length=0)
    ax.set_xticklabels('')
    ax.set_yticklabels('')

    return ax

def view_recon(img, recon):
    ''' Function for displaying an image (as a PyTorch Tensor) and its
        reconstruction also a PyTorch Tensor
    '''

    fig, axes = plt.subplots(ncols=2, sharex=True, sharey=True)
    axes[0].imshow(img.numpy().squeeze())
    axes[1].imshow(recon.data.numpy().squeeze())
    for ax in axes:
        ax.axis('off')
        ax.set_adjustable('box-forced')

def view_classify(img, ps, version="MNIST"):
    ''' Function for viewing an image and it's predicted classes.
    '''
    ps = ps.data.numpy().squeeze()

    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)
    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())
    ax1.axis('off')
    ax2.barh(np.arange(10), ps)
    ax2.set_aspect(0.1)
    ax2.set_yticks(np.arange(10))
    if version == "MNIST":
        ax2.set_yticklabels(np.arange(10))
    elif version == "Fashion":
        ax2.set_yticklabels(['T-shirt/top',
                            'Trouser',
                            'Pullover',
                            'Dress',
                            'Coat',
                            'Sandal',
                            'Shirt',
                            'Sneaker',
                            'Bag',
                            'Ankle Boot'], size='small');
    ax2.set_title('Class Probability')
    ax2.set_xlim(0, 1.1)

    plt.tight_layout()

```

## Tensors
It turns out neural network computations are just a bunch of linear algebra operations on *tensors*, a generalization of matrices. A vector is a 1-dimensional tensor, a matrix is a 2-dimensional tensor, an array with three indices is a 3-dimensional tensor (RGB color images for example). The fundamental data structure for neural networks are tensors and PyTorch (as well as pretty much every other deep learning framework) is built around tensors.

Tensors are similar to NumPy’s ndarrays, with the addition being that Tensors can also be used on a GPU to accelerate computing.

### Construct a randomly initialized 5x3 matrix:

```python
x = torch.rand(5, 3)
print(x)
```

    tensor([[0.9768, 0.4626, 0.4697],
            [0.6264, 0.5623, 0.8074],
            [0.2413, 0.3968, 0.8782],
            [0.5647, 0.7554, 0.1240],
            [0.1027, 0.4617, 0.4611]])

### Construct a tensor directly from data:

```python
x = torch.tensor([5.5, 3])
print(x)
```

    tensor([5.5000, 3.0000])

### Numpy to Torch and back

PyTorch has a great feature for converting between Numpy arrays and Torch tensors. Let us see how easy it is to switch between the two

### Ceate a tensor using numpy array

```python
np_array=np.random.randn(5,3)
print(f' Numpy array:\n {np_array}')
```

     Numpy array:
     [[-0.02647265  0.10582354  0.13038732]
     [-1.34244932  0.35982855  0.70407106]
     [ 1.52964114 -0.29781545 -0.62033849]
     [ 0.96108999 -1.24624232  1.94094033]
     [-0.51059626  0.36401938 -0.5627018 ]]

### Convert to torch tensor

```python
torch_tensor=torch.from_numpy(np_array)
print(f'Torch tensor:\n {torch_tensor}')
```

    Torch tensor:
     tensor([[-0.0265,  0.1058,  0.1304],
            [-1.3424,  0.3598,  0.7041],
            [ 1.5296, -0.2978, -0.6203],
            [ 0.9611, -1.2462,  1.9409],
            [-0.5106,  0.3640, -0.5627]], dtype=torch.float64)

### Convert back to numpy array

```python
torch_tensor.numpy()
```

    array([[-0.02647265,  0.10582354,  0.13038732],
           [-1.34244932,  0.35982855,  0.70407106],
           [ 1.52964114, -0.29781545, -0.62033849],
           [ 0.96108999, -1.24624232,  1.94094033],
           [-0.51059626,  0.36401938, -0.5627018 ]])

***An important thing to note here is memory is shared between the Numpy array and Torch tensor, so if you change the values in-place of one object, the other will change as well.*
Let see what does it mean**

```python
# Add 2 to PyTorch Tensor, in place
torch_tensor.add_(2)
```

    tensor([[1.9735, 2.1058, 2.1304],
            [0.6576, 2.3598, 2.7041],
            [3.5296, 1.7022, 1.3797],
            [2.9611, 0.7538, 3.9409],
            [1.4894, 2.3640, 1.4373]], dtype=torch.float64)

###  Numpy array matches new values from Tensor

```python
np_array
```

    array([[1.97352735, 2.10582354, 2.13038732],
           [0.65755068, 2.35982855, 2.70407106],
           [3.52964114, 1.70218455, 1.37966151],
           [2.96108999, 0.75375768, 3.94094033],
           [1.48940374, 2.36401938, 1.4372982 ]])

 ## Simple Neural Network using Pytorch
 Let us see how we can use PyTorch to build a simple neural network.

Mathematically this looks like:

$$
\begin{align}
y &= f(w_1 x_1 + w_2 x_2 + b) \\
y &= f\left(\sum_i w_i x_i +b \right)
\end{align}
$$

With vectors this is the dot/inner product of two vectors:

$$
h = \begin{bmatrix}
x_1 \, x_2 \cdots  x_n
\end{bmatrix}
\cdot
\begin{bmatrix}
           w_1 \\
           w_2 \\
           \vdots \\
           w_n
\end{bmatrix}
$$

With the basics covered, it's time to explore how we can use PyTorch to build a simple neural network.

###  Generate some random data
 We will create a tensor with shape (1, 5), one row and five columns, that contains values randomly distributed according to the normal distribution with a mean of zero and standard deviation of one.

```python
features=torch.randn(1,3)
print(f'Number of Inout features:{features.shape[1]}')
```

    Number of Inout features:3

### Initialize Weights and Biases

Weights = torch.randn_like(features) creates another tensor with the same shape as features, again containing values from a normal distribution.

Finally, bias = torch.randn((1, 1)) creates a single value from a normal distribution.

```python
n_input=features.shape[1]   #3 input neuron
n_hidden=2     #two hidden neuron
n_output=1     #one output neuron
#Weights for input to hidden layer
W1=torch.randn(n_input,n_hidden)
W2=torch.randn(n_hidden,n_output)
#Bias term for hidden and output layer
B1=torch.randn(n_hidden)
B2=torch.randn(n_output)
```

```python
#Using a Sigmoid Activation Function
def activation(x):
    return(1/1+torch.exp(-x))
```

### Calculate Weight and Biases
We will calculate the output for this multi-layer network using the weights `W1` & `W2`, and the biases, `B1` & `B2`.

```python
h1=activation(torch.matmul(features,W1)+B1)
print(f'Hidden Layer activations:{h1}')
out=activation(torch.matmul(h1,W2)+B2)
print(f'Output of the network:{out}')
```

    Hidden Layer activations:tensor([[2.9791, 1.2008]])
    Output of the network:tensor([[1.1576]])

## Building our Network
Now we're going to build a larger network that can solve a (formerly) difficult problem, identifying text in an image using MNIST data
For now our goal will be to build a neural network that can take one of these images and predict the digit in the image.First, let's try to build this network for this dataset using weight matrices and matrix multiplications. Then, we'll see how to do it using PyTorch's `nn` module which provides a much more convenient and powerful method for defining network architectures.

```python
# Import necessary packages

%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import numpy as np
import torch
import helper
import matplotlib.pyplot as plt
```

### Load Dataset
First up, we need to get our dataset.Right now we will be using MNIST dataset which is already in`torchvision` package. The code below will download the MNIST dataset, then create training and test datasets for us.

```python
from torchvision import datasets, transforms

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                              ])

# Download and load the training data
trainset = datasets.MNIST('MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
```

    0it [00:00, ?it/s]

    Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to MNIST_data/MNIST/raw/train-images-idx3-ubyte.gz

    9920512it [00:00, 27181756.88it/s]

    Extracting MNIST_data/MNIST/raw/train-images-idx3-ubyte.gz

    32768it [00:00, 447175.53it/s]
      1%|          | 16384/1648877 [00:00<00:11, 145553.26it/s]

    Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to MNIST_data/MNIST/raw/train-labels-idx1-ubyte.gz
    Extracting MNIST_data/MNIST/raw/train-labels-idx1-ubyte.gz
    Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to MNIST_data/MNIST/raw/t10k-images-idx3-ubyte.gz

    1654784it [00:00, 7246505.93it/s]
    8192it [00:00, 180111.75it/s]

    Extracting MNIST_data/MNIST/raw/t10k-images-idx3-ubyte.gz
    Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to MNIST_data/MNIST/raw/t10k-labels-idx1-ubyte.gz
    Extracting MNIST_data/MNIST/raw/t10k-labels-idx1-ubyte.gz
    Processing...
    Done!

#### what is dataset ?
Dataset contains two data methods `__getitem__` and `__len__` so using these methods. we can directly call a single data with index

```python
# like this way
trainset[0]
```

    (tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0118, 0.0706, 0.0706, 0.0706,
               0.4941, 0.5333, 0.6863, 0.1020, 0.6510, 1.0000, 0.9686, 0.4980,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.1176, 0.1412, 0.3686, 0.6039, 0.6667, 0.9922, 0.9922, 0.9922,
               0.9922, 0.9922, 0.8824, 0.6745, 0.9922, 0.9490, 0.7647, 0.2510,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1922,
               0.9333, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922,
               0.9922, 0.9843, 0.3647, 0.3216, 0.3216, 0.2196, 0.1529, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0706,
               0.8588, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922, 0.7765, 0.7137,
               0.9686, 0.9451, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.3137, 0.6118, 0.4196, 0.9922, 0.9922, 0.8039, 0.0431, 0.0000,
               0.1686, 0.6039, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0549, 0.0039, 0.6039, 0.9922, 0.3529, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.5451, 0.9922, 0.7451, 0.0078, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0431, 0.7451, 0.9922, 0.2745, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.1373, 0.9451, 0.8824, 0.6275,
               0.4235, 0.0039, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3176, 0.9412, 0.9922,
               0.9922, 0.4667, 0.0980, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1765, 0.7294,
               0.9922, 0.9922, 0.5882, 0.1059, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0627,
               0.3647, 0.9882, 0.9922, 0.7333, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.9765, 0.9922, 0.9765, 0.2510, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1804, 0.5098,
               0.7176, 0.9922, 0.9922, 0.8118, 0.0078, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.1529, 0.5804, 0.8980, 0.9922,
               0.9922, 0.9922, 0.9804, 0.7137, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0941, 0.4471, 0.8667, 0.9922, 0.9922, 0.9922,
               0.9922, 0.7882, 0.3059, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0902, 0.2588, 0.8353, 0.9922, 0.9922, 0.9922, 0.9922, 0.7765,
               0.3176, 0.0078, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0706, 0.6706,
               0.8588, 0.9922, 0.9922, 0.9922, 0.9922, 0.7647, 0.3137, 0.0353,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.2157, 0.6745, 0.8863, 0.9922,
               0.9922, 0.9922, 0.9922, 0.9569, 0.5216, 0.0431, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.5333, 0.9922, 0.9922, 0.9922,
               0.8314, 0.5294, 0.5176, 0.0627, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000],
              [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
               0.0000, 0.0000, 0.0000, 0.0000]]]), 5)

#### what is dataloader?

It simply uses the generator to provide data giving single- or multi-process iterators over the dataset.

We have the training data loaded into trainloader

With dataloaded we make  an iterator with iter(trainloader). Later, we'll use this to loop through the dataset for training, like below:

```python
dataiter = iter(trainloader)
images, labels = dataiter.next()
print(type(images))
print(images.shape)
print(labels.shape)
#Printing the size of one image
print(images[1].numpy().squeeze().shape)
```

    <class 'torch.Tensor'>
    torch.Size([64, 1, 28, 28])
    torch.Size([64])
    (28, 28)

```python
#Look at the image
plt.imshow(images[1].numpy().squeeze(), cmap='Greys_r');
```

```python
#Sigmoid Activation Function
def activation(x):
    return (1/(1+torch.exp(-x)))

#Input 64x784
inputs=images.view(images.shape[0],-1)
#Number of input features-784
n_input=inputs.shape[1]
#Number of neurons in hidden layer-256
n_hidden=256
#Number of output neuron-10
n_out=10
#Weight at hidden neuron-784x256
W1=torch.randn(n_input,n_hidden)
#Bias at hidden neuron-256
B1=torch.randn(n_hidden)
#Weight at output neuron-256x10
W2=torch.randn(n_hidden,n_out)
#Bias at output neuron-10
B2=torch.randn(n_out)

```

```python
print("Shape of a batch of an image:",images.shape)
print("Shape of the input to the network:",inputs.shape)
print("Shape of the input features:",n_input)
print("Shape of the Weight matrix of neurons in the hidden layer",W1.shape)
print("Shape of the Bias vector of neurons in the hidden layer",B1.shape)
print("Shape of the Weight matrix of neurons in the output layer",W2.shape)
print("Shape of the Bias vector of neurons in the output layer",W2.shape)

```

    Shape of a batch of an image: torch.Size([64, 1, 28, 28])
    Shape of the input to the network: torch.Size([64, 784])
    Shape of the input features: 784
    Shape of the Weight matrix of neurons in the hidden layer torch.Size([784, 256])
    Shape of the Bias vector of neurons in the hidden layer torch.Size([256])
    Shape of the Weight matrix of neurons in the output layer torch.Size([256, 10])
    Shape of the Bias vector of neurons in the output layer torch.Size([256, 10])

```python
#Hidden layer activations
h1=activation(torch.mm(inputs,W1)+B1)
#Output layer activations
out=activation(torch.mm(h1,W2)+B2)
```

```python
print(f'Shape of the Hidden activation of the network{h1.shape}')
print(f'Shape of the Output of the network{out.shape}')
```

    Shape of the Hidden activation of the networktorch.Size([64, 256])
    Shape of the Output of the networktorch.Size([64, 10])

```python
#Let us see the network output to one of the feeded input image
out[1]
```

    tensor([9.2154e-01, 4.1177e-05, 1.0000e+00, 9.9978e-01, 7.6175e-01, 1.3460e-03,
            1.0000e+00, 1.4779e-06, 6.7857e-04, 1.0000e+00])

Now we have 10 outputs for our network. This raw output is usually called **logits or scores**.

However,We want to pass in an image to our network and get out a probability distribution over the classes that tells us the likely class(es) the image belongs to.

### Probability Distribution using Softmax
To calculate this probability distribution, we often use the [softmax function](https://en.wikipedia.org/wiki/Softmax_function)
$$
\Large \sigma(x_i) = \cfrac{e^{x_i}}{\sum_k^K{e^{x_k}}}
$$
What this does is squish each input $x_i$ between 0 and 1 and normalizes the values to give you a proper probability distribution where the probabilites sum up to one.

```python
def softmax(x):
    return(torch.exp(x)/torch.sum(torch.exp(x),dim=1).view(-1,1))
```

Let us understand what we are doing above by an example

Step 1:Calculating the numerator of the softmax function

```python
torch.exp(out[1:3])
```

    tensor([[2.5132, 1.0000, 2.7183, 2.7177, 2.1420, 1.0013, 2.7183, 1.0000, 1.0007,
             2.7183],
            [2.7082, 1.0000, 2.7157, 2.7183, 1.0645, 1.0001, 2.7183, 1.0054, 1.0000,
             2.7181]])

Step 2:For every predicted image output, calculate the sum over the predicted values over all classes

```python
#print(torch.sum(torch.exp(out[1:3])))
#Dim=1 says, we want to take the sum across all columns
torch.sum(torch.exp(out[1:3]),dim=1)
```

    tensor([19.5298, 18.6485])

Step3:Rearrange the sums in an order for broadcasting to work

```python
torch.sum(torch.exp(out[1:3]),dim=1).view(-1,1)
```

    tensor([[19.5298],
            [18.6485]])

Step 3:For every predicted image output, divide the predictions of each class with the sum over all classes.

```python
#print(torch.exp(out[1:3])/torch.sum(torch.exp(out[1:3]),dim=1))
temp=torch.exp(out[1:3])/torch.sum(torch.exp(out[1:3]),dim=1).view(-1,1)
print(temp)
```

    tensor([[0.1287, 0.0512, 0.1392, 0.1392, 0.1097, 0.0513, 0.1392, 0.0512, 0.0512,
             0.1392],
            [0.1452, 0.0536, 0.1456, 0.1458, 0.0571, 0.0536, 0.1458, 0.0539, 0.0536,
             0.1458]])

Voila!! We got the softmax output .One last thing to do is check whether the sum across all classes sum to 1 for understanding the predicted class

```python
temp.sum(dim=1)
```

    tensor([1., 1.])

```python
probabilities = softmax(out)
# Does it have the right shape? Should be (64, 10)
print(probabilities.shape)
# Does it sum to 1?
#print(probabilities.sum(dim=1))
```

    torch.Size([64, 10])

## Building our Network with Pytorch

PyTorch provides a module `nn` that makes building networks much simpler. Here I'll show you how to build the same one as above with 784 inputs, 256 hidden units, 10 output units and a softmax output.

```python
from torch import nn
```

```python
class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden=nn.Linear(784,256)
        self.output=nn.Linear(256,10)
        self.sigmoid=nn.Sigmoid()
        self.softmax=nn.Softmax(dim=1)

    def forward(self,x):
        x=self.hidden(x)
        x=self.sigmoid(x)
        x=self.output(x)
        x=self.softmax(x)
        return x
```

Let's go through this bit by bit.

```python
class Network(nn.Module):
```

Here we're inheriting from `nn.Module`. Combined with `super().__init__()` this creates a class that tracks the architecture and provides a lot of useful methods and attributes. It is mandatory to inherit from `nn.Module` when you're creating a class for your network. The name of the class itself can be anything.

```python
self.hidden = nn.Linear(784, 256)
```

This line creates a module for a linear transformation, $x\mathbf{W} + b$, with 784 inputs and 256 outputs and assigns it to `self.hidden`. The module automatically creates the weight and bias tensors which we'll use in the `forward` method. You can access the weight and bias tensors once the network once it's create at `net.hidden.weight` and `net.hidden.bias`.

```python
self.output = nn.Linear(256, 10)
```

Similarly, this creates another linear transformation with 256 inputs and 10 outputs.

```python
self.sigmoid = nn.Sigmoid()
self.softmax = nn.Softmax(dim=1)
```

Here I defined operations for the sigmoid activation and softmax output. Setting `dim=1` in `nn.Softmax(dim=1)` calculates softmax across the columns.

```python
def forward(self, x):
```

PyTorch networks created with `nn.Module` must have a `forward` method defined. It takes in a tensor `x` and passes it through the operations you defined in the `__init__` method.

```python
x = self.hidden(x)
x = self.sigmoid(x)
x = self.output(x)
x = self.softmax(x)
```

Here the input tensor `x` is passed through each operation a reassigned to `x`. We can see that the input tensor goes through the hidden layer, then a sigmoid function, then the output layer, and finally the softmax function. It doesn't matter what you name the variables here, as long as the inputs and outputs of the operations match the network architecture you want to build. The order in which you define things in the `__init__` method doesn't matter, but you'll need to sequence the operations correctly in the `forward` method.

Now we can create a `Network` object.

```python
model=Network()
model
```

    Network(
      (hidden): Linear(in_features=784, out_features=256, bias=True)
      (output): Linear(in_features=256, out_features=10, bias=True)
      (sigmoid): Sigmoid()
      (softmax): Softmax()
    )

We can define the network somewhat more concisely and clearly using the `torch.nn.functional` module. This is the most common way you'll see networks defined as many operations are simple element-wise functions. We normally import this module as `F`, `import torch.nn.functional as F`.

```python
import torch.nn.functional as F
```

```python
import torch.nn.functional as F

class Network(nn.Module):
    def __init__(self):
        super().__init__()
        # Inputs to hidden layer linear transformation
        self.hidden = nn.Linear(784, 128)
        # Output layer, 10 units - one for each digit
        self.output = nn.Linear(128, 10)

    def forward(self, x):
        # Hidden layer with sigmoid activation
        x = F.sigmoid(self.hidden(x))
        # Output layer with softmax activation
        x = F.softmax(self.output(x), dim=1)

        return x
```

```python
model=Network()
model
```

    Network(
      (hidden): Linear(in_features=784, out_features=128, bias=True)
      (output): Linear(in_features=128, out_features=10, bias=True)
    )

### Initializing weights and biases

The weights and bias are automatically initialized for you, but it's possible to customize how they are initialized. The weights and biases are tensors attached to the layer you defined, you can get them with `model.fc1.weight` for instance.

```python
print(model.hidden.weight,model.hidden.weight.shape)
print(model.hidden.bias,model.hidden.bias.shape)
```

    Parameter containing:
    tensor([[ 0.0197, -0.0116,  0.0058,  ...,  0.0302,  0.0299, -0.0343],
            [ 0.0206,  0.0135,  0.0345,  ...,  0.0247, -0.0280,  0.0309],
            [ 0.0328,  0.0228,  0.0160,  ..., -0.0250, -0.0251,  0.0253],
            ...,
            [ 0.0225,  0.0256,  0.0272,  ...,  0.0159,  0.0133,  0.0227],
            [ 0.0063,  0.0204,  0.0154,  ...,  0.0120,  0.0228,  0.0085],
            [ 0.0172, -0.0254, -0.0014,  ..., -0.0239, -0.0123, -0.0082]],
           requires_grad=True) torch.Size([128, 784])
    Parameter containing:
    tensor([-3.4129e-02, -3.5023e-02, -2.4994e-02,  6.9522e-03,  2.4256e-02,
            -3.0995e-02,  1.8428e-02,  1.7995e-04,  3.5690e-02,  1.1080e-02,
            -2.5361e-02, -1.9550e-02, -4.1704e-03, -3.0963e-02, -2.3094e-03,
             2.3683e-02, -8.2089e-03,  3.5467e-02,  6.6356e-03,  9.0419e-04,
             9.8338e-03, -1.3634e-02, -2.2661e-02, -9.3643e-03,  2.4917e-02,
             3.0572e-02, -6.5517e-03,  3.5520e-02,  3.5385e-02, -3.1636e-02,
             1.2763e-02, -2.5896e-02, -1.5290e-02,  7.1542e-03,  2.3027e-02,
             3.5029e-02, -3.1634e-03,  2.5170e-02,  1.0522e-02,  2.0760e-02,
            -1.7747e-02, -9.3821e-03,  1.6665e-02, -1.5528e-02,  3.5188e-02,
             2.0664e-02, -3.6067e-03,  1.4825e-02, -3.5022e-03,  3.2112e-02,
            -9.9952e-03,  1.8579e-02,  1.9001e-02,  2.7324e-02, -1.8744e-02,
            -3.1447e-02,  2.7373e-02,  1.2326e-02, -3.1618e-02,  3.5516e-02,
             1.8448e-02, -2.7804e-02,  3.4334e-02,  2.3076e-03,  1.8247e-02,
            -2.7856e-02, -1.1554e-02,  6.2440e-03, -5.8380e-04,  2.4348e-02,
             3.0930e-02, -2.4141e-02,  3.0912e-02,  1.2217e-02,  3.9276e-03,
            -2.8238e-02,  4.9517e-03,  8.7882e-04, -1.6903e-02,  1.7382e-02,
             4.3225e-03, -2.3541e-02, -1.6702e-03,  3.1075e-02, -1.7210e-02,
             3.4483e-02,  1.8049e-04, -1.4049e-02,  1.4035e-02,  2.5542e-03,
             3.3312e-02, -1.0237e-02, -1.8189e-02, -9.2125e-03, -2.2061e-02,
             1.0555e-02,  2.4672e-02, -2.9224e-02, -2.6461e-02, -1.1879e-02,
             1.2724e-02,  1.0099e-02,  3.4147e-03, -1.5321e-02,  2.0307e-03,
             1.0033e-02, -2.2252e-02,  3.1660e-02,  2.3012e-02,  1.4720e-02,
             3.4967e-02, -2.4062e-02,  8.7397e-03, -3.2874e-02,  2.1446e-05,
             3.5473e-02,  1.5873e-02, -2.6940e-02, -1.2387e-02,  2.6201e-02,
            -3.1600e-02,  3.4718e-03, -8.8702e-03, -2.5180e-02,  2.0655e-02,
             1.7065e-03,  2.7802e-02,  2.8921e-02], requires_grad=True) torch.Size([128])

For custom initialization, we can these tensors in place.

```python
# Set biases to all zeros
model.hidden.bias.data.fill_(0)
```

    tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0., 0., 0., 0., 0., 0., 0., 0.])

```python
# sample from random normal with standard dev = 0.01
model.hidden.weight.data.normal_(std=0.01)
```

    tensor([[-1.2083e-02,  1.3362e-02, -4.7883e-04,  ..., -1.9470e-02,
             -7.9714e-05, -2.3377e-02],
            [ 1.9217e-03, -7.4535e-03,  1.4575e-03,  ..., -3.4555e-03,
              5.4716e-03, -2.1791e-03],
            [-2.0988e-03,  9.7672e-03, -3.6484e-03,  ..., -6.3477e-03,
             -1.8276e-02,  9.4072e-03],
            ...,
            [ 3.9926e-03,  1.2083e-02, -5.4276e-03,  ..., -7.9789e-03,
              1.0731e-03,  3.9884e-03],
            [-2.4229e-03, -3.3990e-03,  6.5320e-03,  ...,  9.5462e-04,
              6.1829e-03, -1.6516e-02],
            [-2.5104e-03, -9.3102e-03,  5.5190e-03,  ..., -1.0905e-02,
             -1.3265e-02, -5.5200e-03]])

```python
netowrk=Network()
```

### Forward pass

Now that we have a network, let's see what happens when we pass in an image.

```python
# Grab some data
dataiter = iter(trainloader)
images, labels = dataiter.next()

# Resize images into a 1D vector, new shape is (batch size, color channels, image pixels)
images.resize_(images.shape[0], 1, 784)

# Forward pass through the network
img_idx = 0
ps = model.forward(images[img_idx,:])

img = images[img_idx]
view_classify(img.view(1, 28, 28), ps)
```

As you can see above, our network has basically no idea what this digit is. It's because we haven't trained it yet, , all the weights are random!

## Add-on: People from the keras would love this!!!
PyTorch provides a convenient way to build networks like this where a tensor is passed sequentially through operations, `nn.Sequential`.
Lets try to build the above network using this method:

```python
# Hyperparameters for our network
input_size = 784
hidden_sizes = [128]
output_size = 10

model=nn.Sequential(nn.Linear(input_size,hidden_sizes[0]),
                    nn.ReLU(),
                    nn.Linear(hidden_sizes[0],output_size),
                    nn.ReLU(),
                    nn.Softmax(dim=1))
print(model)
```

    Sequential(
      (0): Linear(in_features=784, out_features=128, bias=True)
      (1): ReLU()
      (2): Linear(in_features=128, out_features=10, bias=True)
      (3): ReLU()
      (4): Softmax()
    )

```python
# Forward pass through the network and display output
images, labels = next(iter(trainloader))
images.resize_(images.shape[0], 1, 784)
ps = model.forward(images[0,:])
view_classify(images[0].view(1, 28, 28), ps)
```

### Access Layers of the network
We can access layers  by integer

```python
print(model[0])
model[0].weight
```

    Linear(in_features=784, out_features=128, bias=True)

    Parameter containing:
    tensor([[-0.0193,  0.0175, -0.0097,  ..., -0.0264,  0.0135,  0.0245],
            [ 0.0010, -0.0323, -0.0246,  ...,  0.0255, -0.0211, -0.0345],
            [ 0.0170, -0.0030, -0.0328,  ...,  0.0156, -0.0089, -0.0116],
            ...,
            [ 0.0051, -0.0267, -0.0117,  ..., -0.0142,  0.0277,  0.0269],
            [-0.0048,  0.0291, -0.0253,  ...,  0.0246,  0.0038,  0.0083],
            [-0.0341, -0.0225, -0.0207,  ..., -0.0338,  0.0086,  0.0154]],
           requires_grad=True)

### Ordered Dict- Better way to create a network
We can also pass in an `OrderedDict` to name the individual layers and operations, instead of using incremental integers. Note that dictionary keys must be unique, so _each operation must have a different name_.

```python
from collections import OrderedDict
model = nn.Sequential(OrderedDict([
                      ('hidden', nn.Linear(input_size, hidden_sizes[0])),
                      ('relu1', nn.ReLU()),
                      ('output', nn.Linear(hidden_sizes[0], output_size)),
                      ('softmax', nn.Softmax(dim=1))]))
model
```

    Sequential(
      (hidden): Linear(in_features=784, out_features=128, bias=True)
      (relu1): ReLU()
      (output): Linear(in_features=128, out_features=10, bias=True)
      (softmax): Softmax()
    )

### Access Layers using integer or name
Now we can access layers  either by integer or name

```python
print(model[0])
print(model.hidden)
print(model.hidden.weight)
```

    Linear(in_features=784, out_features=128, bias=True)
    Linear(in_features=784, out_features=128, bias=True)
    Parameter containing:
    tensor([[-0.0213,  0.0055,  0.0044,  ...,  0.0216,  0.0212,  0.0184],
            [-0.0282,  0.0110, -0.0335,  ..., -0.0296, -0.0002, -0.0258],
            [-0.0075,  0.0346,  0.0275,  ...,  0.0115,  0.0269, -0.0081],
            ...,
            [-0.0204,  0.0250, -0.0245,  ...,  0.0287,  0.0189,  0.0337],
            [-0.0119,  0.0049, -0.0011,  ..., -0.0234, -0.0316,  0.0198],
            [ 0.0213, -0.0012, -0.0262,  ...,  0.0164,  0.0224,  0.0268]],
           requires_grad=True)

### Recollect everything
Before we go ahead and train a neural network to accuractly predict the numbers appearing in the MNIST images,let us recollect the important modules that is necessary for any model training exercise

#### Imports

```python
import torch
from torch import nn
import torch.nn.functional as F
from torchvision import datasets,transforms
```

#### Load Data

```python
transform=transforms.Compose([transforms.ToTensor()])
trainset=datasets.MNIST('~/.pytorch/MNIST_data/',train=True,transform=transform,download=True)
testset=datasets.MNIST('~/.pytorch/MNIST_data/',train=False,transform=transform,download=True)

trainloader=torch.utils.data.DataLoader(trainset,batch_size=64,shuffle=True,num_workers=0)
#will explain later
testloader=torch.utils.data.DataLoader(testset,batch_size=64,shuffle=True,num_workers=0)
```

      0%|          | 16384/9912422 [00:00<01:11, 138201.55it/s]

    Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to /root/.pytorch/MNIST_data/MNIST/raw/train-images-idx3-ubyte.gz

    9920512it [00:00, 31253957.04it/s]

    Extracting /root/.pytorch/MNIST_data/MNIST/raw/train-images-idx3-ubyte.gz

    32768it [00:00, 445049.83it/s]
      1%|          | 16384/1648877 [00:00<00:11, 140432.12it/s]

    Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to /root/.pytorch/MNIST_data/MNIST/raw/train-labels-idx1-ubyte.gz
    Extracting /root/.pytorch/MNIST_data/MNIST/raw/train-labels-idx1-ubyte.gz
    Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to /root/.pytorch/MNIST_data/MNIST/raw/t10k-images-idx3-ubyte.gz

    1654784it [00:00, 6967617.90it/s]
    8192it [00:00, 160376.67it/s]

    Extracting /root/.pytorch/MNIST_data/MNIST/raw/t10k-images-idx3-ubyte.gz
    Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to /root/.pytorch/MNIST_data/MNIST/raw/t10k-labels-idx1-ubyte.gz
    Extracting /root/.pytorch/MNIST_data/MNIST/raw/t10k-labels-idx1-ubyte.gz
    Processing...
    Done!

#### Build a feedforward Network

```python
# TODO: Build a feed-forward network in one of the three ways mentioned above:
model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 64),
                      nn.ReLU(),
                      nn.Linear(64, 10),
                      nn.LogSoftmax(dim=1))

```

#### Lets run one image through the network to check our work

```python
# Get our data
images, labels = next(iter(trainloader))
# Flatten images
images = images.view(images.shape[0], -1)

# Forward pass, get our logits
logits = model(images)
print(logits.shape)
```

    torch.Size([64, 10])

#### Define a loss function

```python
criterion = nn.NLLLoss()
```

```python
# Calculate the loss with the logits and the labels
loss=criterion(logits,labels)
print(loss)
```

    tensor(2.3049, grad_fn=<NllLossBackward>)

## Autograd

Now that we know how to calculate a loss, how do we use it to perform backpropagation? Torch provides a module, `autograd`, for automatically calculating the gradients of tensors. We can use it to calculate the gradients of all our parameters with respect to the loss. Autograd works by keeping track of operations performed on tensors, then going backwards through those operations, calculating gradients along the way.

PyTorch keeps track of operations on a tensor and calculates the gradients, you need to set `requires_grad = True` on a tensor. You can do this at creation with the `requires_grad` keyword, or at any time with `x.requires_grad_(True)`.

Let's see an example to understand it better.Then again we will head back to our modelling task

```python
x = torch.randn(2,2, requires_grad=True)
print("x:",x)
y = x**2
print("y:",y)
## grad_fn shows the function that generated this variable
print("y.grad_fn:",y.grad_fn)
z = y.mean()
print("z:",z)
print("x.grad:",x.grad)
z.backward()
print("x.grad:",x.grad)
print("x/2:",x/2)
```

    x: tensor([[ 0.5090, -0.1186],
            [-0.6671, -0.0789]], requires_grad=True)
    y: tensor([[0.2591, 0.0141],
            [0.4450, 0.0062]], grad_fn=<PowBackward0>)
    y.grad_fn: <PowBackward0 object at 0x7f3d1d46e6a0>
    z: tensor(0.1811, grad_fn=<MeanBackward0>)
    x.grad: None
    x.grad: tensor([[ 0.2545, -0.0593],
            [-0.3335, -0.0394]])
    x/2: tensor([[ 0.2545, -0.0593],
            [-0.3335, -0.0394]], grad_fn=<DivBackward0>)

## Loss and Autograd together

```python
# Build a feed-forward network
model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 64),
                      nn.ReLU(),
                      nn.Linear(64, 10),
                      nn.LogSoftmax(dim=1))

criterion = nn.NLLLoss()
images, labels = next(iter(trainloader))
images = images.view(images.shape[0], -1)

logits = model(images)
loss = criterion(logits, labels)

print('Before backward pass: \n', model[0].weight.grad)

loss.backward()

print('After backward pass: \n', model[0].weight.grad)
```

    Before backward pass:
     None
    After backward pass:
     tensor([[0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            ...,
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.]])

## Defining the optimizer

```python
from torch import optim
optimizer=optim.Adam(model.parameters(),lr=0.1)
```

## Training for real

```python
epochs = 5
for e in range(epochs):
    running_loss = 0
    for images, labels in trainloader:
        # Flatten MNIST images into a 784 long vector
        images = images.view(images.shape[0], -1)
        optimizer.zero_grad()
        output=model.forward(images)
        # TODO: Training pass

        loss = criterion(output,labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()*images.shape[0]
    else:
        print(f"Epoch:{e} Training loss: {running_loss/len(trainloader.dataset)}")
```

    Epoch:0 Training loss: 1.1391830668767293
    Epoch:1 Training loss: 1.2300454666773477
    Epoch:2 Training loss: 1.4402017743428548
    Epoch:3 Training loss: 1.687267345937093
    Epoch:4 Training loss: 1.6394379685719809

```python
%matplotlib inline
#import helper

images, labels = next(iter(trainloader))

img = images[0].view(1, 784)
# Turn off gradients to speed up this part
with torch.no_grad():
    logits = model.forward(img)

# Output of the network are logits, need to take softmax for probabilities
ps = F.softmax(logits, dim=1)
#helper.view_classify(img.view(1, 28, 28), ps)
```

## Inference and Validation

The goal of validation is to measure the model's performance on data that isn't part of the training set. Typically this is just accuracy, the percentage of classes the network predicted correctly. Other options are precision and recall and top-5 error rate. We'll focus on accuracy here. First I'll do a forward pass with one batch from the test set.

### Inference on a batch of images
Let us try to do this for a batch of images.Before that we will make some changes in our architecture

```python
images, labels = next(iter(testloader))
images.shape,labels.shape
```

    (torch.Size([64, 1, 28, 28]), torch.Size([64]))

```python
images, labels = next(iter(testloader))
img = images.view(images.shape[0], 784)
# Get the class probabilities
ps = torch.exp(model(img))
# Make sure the shape is appropriate, we should get 10 class probabilities for 64 examples
print(ps.shape)
```

    torch.Size([64, 10])

```python
top_prob,top_class=ps.topk(1,dim=1)
top_prob.shape,top_class.shape
```

    (torch.Size([64, 1]), torch.Size([64, 1]))

```python
top_class.view(64)
```

    tensor([6, 1, 3, 6, 3, 6, 6, 6, 6, 6, 6, 3, 6, 3, 6, 6, 6, 6, 6, 6, 6, 3, 6, 3,
            6, 1, 6, 6, 3, 6, 6, 3, 1, 6, 3, 3, 3, 6, 6, 6, 3, 6, 6, 6, 6, 1, 6, 3,
            6, 3, 6, 1, 6, 6, 6, 6, 3, 0, 3, 3, 6, 6, 6, 3])

```python
import pandas as pd
pd.DataFrame({"Predicted":top_class.view(top_class.shape[0]),"Actual":labels})
```

      Predicted
      Actual

      0
      6
      2

      1
      1
      1

      2
      3
      5

      3
      6
      1

      4
      3
      9

      5
      6
      6

      6
      6
      5

      7
      6
      0

      8
      6
      8

      9
      6
      0

      10
      6
      7

      11
      3
      8

      12
      6
      3

      13
      3
      9

      14
      6
      6

      15
      6
      6

      16
      6
      6

      17
      6
      2

      18
      6
      7

      19
      6
      7

      20
      6
      8

      21
      3
      1

      22
      6
      8

      23
      3
      5

      24
      6
      7

      25
      1
      1

      26
      6
      7

      27
      6
      9

      28
      3
      4

      29
      6
      0

      ...
      ...
      ...

      34
      3
      8

      35
      3
      5

      36
      3
      5

      37
      6
      6

      38
      6
      2

      39
      6
      7

      40
      3
      3

      41
      6
      2

      42
      6
      9

      43
      6
      4

      44
      6
      8

      45
      1
      1

      46
      6
      2

      47
      3
      5

      48
      6
      2

      49
      3
      3

      50
      6
      6

      51
      1
      1

      52
      6
      7

      53
      6
      5

      54
      6
      6

      55
      6
      4

      56
      3
      3

      57
      0
      0

      58
      3
      8

      59
      3
      5

      60
      6
      7

      61
      6
      4

      62
      6
      6

      63
      3
      4

64 rows × 2 columns

```python
equals=top_class == labels.view(*top_class.shape)
accuracy=torch.mean(equals.type(torch.FloatTensor))
accuracy.item()
```

    0.265625

```python
class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(784, 256)
        self.fc2 = nn.Linear(256, 128)
        self.fc3 = nn.Linear(128, 64)
        self.fc4 = nn.Linear(64, 10)

        # Dropout module with 0.2 drop probability
        self.dropout = nn.Dropout(p=0.2)

    def forward(self, x):
        # make sure input tensor is flattened
        x = x.view(x.shape[0], -1)

        # Now with dropout
        x = self.dropout(F.relu(self.fc1(x)))
        x = self.dropout(F.relu(self.fc2(x)))
        x = self.dropout(F.relu(self.fc3(x)))

        # output so no dropout here
        x = F.log_softmax(self.fc4(x), dim=1)

        return x

model=Network()
optimizer=optim.Adam(model.parameters(),lr=0.01)
criterion=nn.NLLLoss()
```

```python
epochs=5
train_losses,test_losses=[],[]
for e in range(epochs):
    running_loss=0
    for images,labels in trainloader:
        optimizer.zero_grad()
        #images=images.view(images.shape[0],-1)
        log_ps=model(images)
        loss=criterion(log_ps,labels) # a single value for ex 2.33
        loss.backward()
        optimizer.step()
        running_loss += loss.item() * images.shape[0] ## (2.33*64 + 2.22*64 + 2.12*33) / 138

    else:
        test_loss=0
        accuracy=0

        with torch.no_grad():
            model.eval()
            for images,labels in testloader:
                log_ps=model(images)
                test_loss+=criterion(log_ps,labels) *images.shape[0]
                ps=torch.exp(log_ps)
                top_p,top_class=ps.topk(1,dim=1)
                equals=top_class==labels.view(*top_class.shape)
                accuracy+=torch.sum(equals).item()
        model.train()
        train_losses.append(running_loss/len(trainloader.dataset))
        test_losses.append(test_loss.item()/len(testloader.dataset))

        print("Epoch: {}/{}.. ".format(e+1, epochs),
              "Training Loss: {:.3f}.. ".format(running_loss/len(trainloader.dataset)),
              "Test Loss: {:.3f}.. ".format(test_loss/len(testloader.dataset)),
              "Test Accuracy: {:.3f}".format(accuracy/len(testloader.dataset)))
```

    Epoch: 1/5..  Training Loss: 0.402..  Test Loss: 0.204..  Test Accuracy: 0.947
    Epoch: 2/5..  Training Loss: 0.300..  Test Loss: 0.193..  Test Accuracy: 0.952
    Epoch: 3/5..  Training Loss: 0.319..  Test Loss: 0.197..  Test Accuracy: 0.955
    Epoch: 4/5..  Training Loss: 0.276..  Test Loss: 0.190..  Test Accuracy: 0.954
    Epoch: 5/5..  Training Loss: 0.279..  Test Loss: 0.190..  Test Accuracy: 0.960

```python
running_loss
```

    16732.30222570896

```python
import matplotlib.pyplot as plt
%matplotlib inline
plt.plot(train_losses, label='Training loss')
plt.plot(test_losses, label='Validation loss')
plt.legend(frameon=False)
```

## Inference time

```python
model.eval()

dataiter = iter(testloader)
images, labels = dataiter.next()
img = images[0]
# Convert 2D image to 1D vector
img = img.view(1, 784)

# Calculate the class probabilities (softmax) for img
with torch.no_grad():
    output = model.forward(img)

ps = torch.exp(output)
top_prob,top_class=ps.topk(1,dim=1)
top_class.item(),labels[0]
```

    (8, tensor(8))

The parameters for PyTorch networks are stored in a model's state_dict
 Optimizer objects (torch.optim) also have a state_dict, which contains information about the optimizer’s state, as well as the hyperparameters used.

Because state_dict objects are Python dictionaries, they can be easily saved, updated, altered, and restored, adding a great deal of modularity to PyTorch models and optimizers.

```python
print("Model's state_dict:")
for param_tensor in model.state_dict():
    print(param_tensor, "\t", model.state_dict()[param_tensor].size())
```

    Model's state_dict:
    fc1.weight 	 torch.Size([256, 784])
    fc1.bias 	 torch.Size([256])
    fc2.weight 	 torch.Size([128, 256])
    fc2.bias 	 torch.Size([128])
    fc3.weight 	 torch.Size([64, 128])
    fc3.bias 	 torch.Size([64])
    fc4.weight 	 torch.Size([10, 64])
    fc4.bias 	 torch.Size([10])

```python
# Print optimizer's state_dict
print("Optimizer's state_dict:")
for var_name in optimizer.state_dict():
    print(var_name, "\t", optimizer.state_dict()[var_name])
```

    Optimizer's state_dict:
    state 	 {139900461526256: {'step': 4690, 'exp_avg': tensor([[0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            ...,
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.]]), 'exp_avg_sq': tensor([[0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            ...,
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.],
            [0., 0., 0.,  ..., 0., 0., 0.]])}, 139900461526328: {'step': 4690, 'exp_avg': tensor([ 5.6052e-45, -9.0633e-05, -2.0838e-18,  5.6052e-45,  5.6052e-45,
            -2.1870e-05, -1.6081e-04, -1.2594e-03, -1.9198e-04,  3.1295e-38,
            -4.9848e-04,  2.1075e-03, -1.6403e-23, -4.4465e-04,  2.8305e-04,
            -6.0509e-41,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,
             5.7103e-04, -4.9331e-04, -1.4567e-03,  5.6052e-45, -9.3776e-07,
             4.8159e-04, -1.2833e-03,  2.9540e-11, -1.3162e-04,  5.6052e-45,
             5.6052e-45,  5.6052e-45,  5.6052e-45, -1.6582e-04, -1.9093e-04,
            -2.7883e-03, -6.7744e-04,  5.6052e-45, -3.7190e-04,  5.6052e-45,
            -9.4714e-05,  5.6052e-45,  5.6052e-45,  5.1016e-26,  5.6052e-45,
            -1.4458e-03, -7.5890e-06, -3.3384e-07,  5.6052e-45, -4.6484e-04,
            -3.9815e-24,  5.0237e-07,  1.0383e-04, -1.4806e-04, -1.3148e-03,
            -5.6060e-06,  1.4278e-03,  5.6052e-45,  5.6052e-45, -1.8632e-06,
             5.6052e-45,  5.6052e-45, -1.8906e-04,  1.6491e-04,  5.6052e-45,
            -6.2496e-05, -3.7904e-05,  5.6052e-45,  1.0470e-03,  5.6052e-45,
             5.5002e-04,  5.6052e-45,  5.6052e-45,  5.6052e-45, -3.3427e-04,
            -7.0477e-05,  5.6052e-45,  5.6052e-45, -1.1798e-03, -1.4826e-04,
             2.2910e-34, -2.2005e-04,  1.5249e-03, -4.2334e-10,  5.0154e-04,
            -6.2739e-04,  5.6052e-45, -1.9690e-03, -5.6052e-45, -4.7854e-04,
             5.6052e-45,  5.6052e-45, -3.1894e-04,  2.7461e-03, -5.2178e-04,
            -2.8835e-04, -3.6933e-04, -1.4136e-07,  1.8333e-05,  1.9115e-04,
            -9.9939e-05,  5.6052e-45,  5.6052e-45,  5.6052e-45,  3.3981e-04,
            -1.8277e-04,  7.2033e-04,  5.6052e-45,  3.9305e-17,  5.6052e-45,
            -7.5574e-33,  1.9815e-04,  5.6052e-45,  3.8151e-04,  1.2540e-04,
             5.6052e-45,  8.0551e-04,  1.3884e-03, -2.5297e-04, -1.6292e-04,
             5.6052e-45,  5.1199e-04,  9.6204e-07,  5.6052e-45,  5.6052e-45,
            -6.0065e-04, -1.3282e-03,  4.7743e-19, -7.5781e-08,  6.1700e-04,
             5.4625e-04, -2.2623e-06, -2.2372e-06,  3.6068e-04, -2.5489e-04,
             1.1144e-05,  5.6052e-45,  5.6052e-45,  5.1736e-04,  5.6052e-45,
             5.6052e-45, -5.1086e-05,  5.6052e-45, -1.2228e-03, -8.9460e-16,
            -2.4032e-07,  5.6052e-45,  5.6052e-45, -3.8343e-07,  5.6052e-45,
             5.6052e-45,  5.6052e-45, -4.0377e-04, -1.6527e-04, -7.4107e-04,
            -2.5274e-04,  1.3999e-03,  5.6052e-45,  5.6052e-45,  2.2554e-04,
             5.6052e-45,  5.6052e-45, -2.1454e-18,  5.6052e-45,  5.6052e-45,
             5.6052e-45,  5.6052e-45,  5.6052e-45,  4.8885e-11, -1.1367e-04,
             5.6052e-45,  5.6052e-45, -5.6052e-45, -2.6906e-04,  3.3655e-04,
             6.2474e-04,  2.1049e-04,  5.6052e-45,  6.8644e-04, -5.0611e-04,
            -4.8471e-04, -1.3922e-03,  5.6052e-45, -5.0407e-04,  6.8073e-04,
             5.6052e-45,  4.0440e-05,  5.6052e-45,  5.6052e-45,  4.1358e-04,
             3.2968e-04, -4.0805e-04,  4.0927e-05,  5.6052e-45,  5.6052e-45,
             5.6052e-45,  4.3170e-28, -3.0520e-04,  5.6052e-45,  1.7056e-03,
             5.6052e-45,  8.5415e-07,  5.6052e-45,  1.4115e-03,  7.3234e-04,
             5.6052e-45, -3.1110e-05,  5.6052e-45,  5.6052e-45, -5.0643e-05,
             5.6052e-45,  4.6739e-06, -3.0619e-12,  5.6052e-45, -5.6052e-45,
             3.9847e-04,  5.6052e-45,  5.6052e-45,  1.2318e-03, -2.2216e-03,
             5.6052e-45, -4.9994e-04, -1.0524e-08,  5.6052e-45,  1.1751e-04,
            -7.9131e-04, -2.4598e-04, -1.5322e-04, -8.4084e-07, -4.0460e-04,
             5.6052e-45,  5.6052e-45,  1.2803e-16,  5.6052e-45, -1.1575e-03,
            -2.4894e-04, -1.0898e-12,  1.1172e-03, -7.2015e-06, -7.3483e-05,
            -1.4377e-04,  5.6052e-45, -2.0602e-05,  1.1014e-17,  5.6052e-45,
            -2.6173e-05,  3.4547e-04, -8.5269e-04,  2.3448e-24, -1.7872e-05,
             6.9672e-05,  5.6052e-45, -9.6096e-04,  3.7348e-04, -2.4987e-04,
             1.5859e-03]), 'exp_avg_sq': tensor([2.0160e-11, 2.1008e-06, 1.2720e-07, 8.1500e-12, 7.7864e-11, 6.0178e-06,
            2.5515e-06, 3.0802e-05, 7.8533e-06, 6.0529e-08, 1.0237e-05, 2.6508e-05,
            5.3380e-08, 1.1060e-05, 2.3833e-05, 2.8277e-08, 2.1225e-12, 6.3951e-12,
            1.9235e-11, 1.2461e-10, 1.9305e-05, 9.7068e-06, 2.0388e-05, 7.6511e-13,
            1.2754e-06, 7.6364e-06, 1.4660e-05, 8.6858e-08, 1.3502e-05, 4.6380e-12,
            1.3879e-11, 1.5353e-09, 1.8678e-11, 6.9552e-07, 9.7127e-06, 3.0020e-05,
            5.7284e-06, 4.3043e-11, 1.4089e-05, 6.8081e-12, 1.0286e-06, 7.7298e-14,
            5.0995e-08, 7.3316e-07, 2.8563e-11, 2.0212e-05, 3.1870e-06, 1.9183e-06,
            8.6649e-12, 1.1835e-05, 1.6193e-07, 1.6386e-05, 2.3852e-05, 1.3492e-06,
            1.9181e-05, 4.1632e-06, 2.3369e-05, 7.9263e-13, 1.5977e-11, 1.3816e-06,
            5.0661e-12, 6.0988e-12, 1.1294e-05, 1.5424e-05, 1.0193e-12, 4.2645e-06,
            1.9336e-05, 6.4664e-12, 1.1046e-05, 8.1666e-11, 1.2267e-05, 1.2778e-10,
            3.8073e-08, 1.8938e-13, 6.6783e-06, 2.7694e-05, 1.9329e-12, 2.1069e-07,
            6.5967e-06, 2.3123e-05, 1.1079e-07, 2.0135e-05, 1.1243e-05, 9.4523e-08,
            4.4590e-06, 7.9489e-06, 1.0393e-11, 2.7172e-05, 6.8948e-08, 8.8119e-06,
            1.0062e-10, 5.3416e-10, 5.1183e-06, 1.9165e-05, 1.1489e-05, 1.2823e-05,
            2.1543e-05, 3.0763e-06, 1.5846e-05, 3.0736e-05, 2.7274e-05, 1.1226e-11,
            8.0351e-12, 1.6604e-11, 1.7078e-05, 1.1072e-05, 1.0692e-05, 3.9794e-08,
            1.0354e-08, 2.2004e-08, 1.5030e-08, 1.6517e-06, 3.4152e-12, 3.4401e-05,
            1.1921e-05, 2.5089e-11, 2.3087e-05, 1.1648e-05, 1.4296e-05, 2.0518e-06,
            2.1707e-07, 1.5897e-05, 1.3711e-05, 6.6077e-12, 4.2163e-12, 1.3388e-05,
            1.1029e-05, 6.3954e-07, 4.5353e-06, 2.8117e-05, 2.0157e-05, 3.5936e-06,
            5.7601e-06, 2.5753e-05, 2.8817e-05, 1.0194e-05, 7.6028e-12, 8.0913e-11,
            1.0208e-05, 6.8632e-12, 3.5391e-12, 6.2860e-06, 7.1457e-11, 6.1707e-06,
            3.5247e-08, 5.4233e-06, 1.7449e-11, 3.3822e-07, 6.5717e-06, 7.5350e-12,
            9.1023e-09, 1.5383e-08, 3.3088e-05, 8.6880e-07, 7.1587e-06, 1.0490e-05,
            2.2026e-05, 8.9084e-12, 3.1643e-11, 1.0487e-05, 3.5097e-09, 2.2138e-11,
            1.0931e-06, 1.1192e-09, 5.8274e-11, 3.3750e-12, 7.1008e-12, 7.5826e-12,
            1.7436e-06, 1.6265e-05, 3.0266e-12, 3.2127e-12, 2.2556e-08, 1.2205e-05,
            1.2238e-05, 7.5882e-06, 2.7635e-05, 5.8003e-11, 2.2000e-05, 1.8573e-05,
            6.4704e-06, 2.3237e-05, 4.5839e-09, 1.2010e-05, 3.1836e-05, 6.4275e-12,
            9.3727e-06, 1.5362e-15, 5.8506e-08, 1.4453e-05, 2.0889e-05, 1.3129e-05,
            3.2019e-06, 1.4059e-11, 2.8743e-11, 3.5082e-10, 5.6549e-08, 7.5633e-06,
            2.9774e-11, 2.7921e-05, 5.1786e-12, 6.7535e-07, 1.0596e-11, 1.5241e-05,
            5.0772e-06, 4.5000e-12, 4.4615e-06, 6.0049e-11, 2.2792e-10, 1.7014e-05,
            6.7951e-08, 9.5301e-07, 2.1374e-07, 5.7570e-12, 6.9692e-09, 8.8905e-06,
            1.4452e-11, 3.7032e-08, 1.4276e-05, 2.1252e-05, 3.8762e-12, 7.0415e-06,
            3.4788e-07, 1.0391e-11, 1.6392e-06, 2.1311e-05, 1.4133e-05, 1.7716e-06,
            1.8970e-05, 1.2725e-05, 1.2128e-11, 1.9355e-11, 3.7479e-12, 3.6879e-08,
            1.5979e-05, 5.8590e-06, 5.7794e-08, 1.0580e-05, 8.3609e-06, 1.1806e-05,
            6.4650e-06, 5.5602e-08, 1.2513e-06, 4.5440e-08, 1.5930e-11, 7.5485e-06,
            1.5942e-05, 1.0589e-05, 4.1900e-07, 1.9761e-05, 2.1407e-05, 2.5238e-11,
            2.2480e-05, 7.5331e-06, 1.8760e-05, 1.4502e-05])}, 139900461525392: {'step': 4690, 'exp_avg': tensor([[ 5.6052e-45,  5.9859e-10, -5.6052e-45,  ..., -7.5535e-23,
             -8.2570e-22, -1.4431e-24],
            [-5.6052e-45,  5.6052e-45,  5.6052e-45,  ..., -9.4061e-06,
             -6.5321e-04, -1.9733e-03],
            [-5.6052e-45,  4.9055e-07, -5.6052e-45,  ...,  5.4969e-06,
             -4.9657e-05, -5.3686e-06],
            ...,
            [ 5.6052e-45, -5.6052e-45, -5.6052e-45,  ...,  8.9802e-07,
              1.7440e-05,  3.9829e-05],
            [-5.6052e-45,  8.4629e-07,  5.6052e-45,  ...,  3.0286e-21,
             -1.1601e-07, -1.0881e-09],
            [-5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,
              6.1732e-06,  7.2658e-05]]), 'exp_avg_sq': tensor([[3.3201e-13, 5.3940e-07, 3.7244e-08,  ..., 2.8396e-08, 2.9961e-06,
             6.8652e-07],
            [6.6013e-12, 4.0142e-08, 2.2513e-08,  ..., 2.0858e-06, 9.8170e-05,
             4.5702e-05],
            [1.3497e-12, 1.2152e-06, 1.1506e-08,  ..., 6.8172e-08, 2.0846e-05,
             3.8204e-05],
            ...,
            [1.8454e-12, 4.7357e-10, 7.1464e-08,  ..., 4.1053e-08, 1.8037e-06,
             1.4767e-06],
            [1.3383e-13, 3.9913e-09, 4.3661e-08,  ..., 1.3015e-08, 1.8022e-06,
             7.7080e-07],
            [8.0929e-13, 1.0355e-07, 6.2716e-08,  ..., 3.7291e-09, 1.6681e-06,
             2.5748e-06]])}, 139900461526112: {'step': 4690, 'exp_avg': tensor([-1.2909e-04,  2.4660e-04,  1.5988e-04,  7.3528e-05,  1.8472e-06,
            -1.9826e-05, -2.0145e-04, -3.3048e-05, -5.6509e-04, -1.9983e-04,
            -1.5285e-04, -4.6748e-05,  8.3083e-04, -4.4042e-06,  8.9194e-06,
             4.8256e-04, -1.1667e-04, -1.1729e-04, -1.4312e-04,  5.7649e-04,
             1.0127e-04, -1.6241e-06, -3.4087e-04,  2.3752e-04, -3.0360e-05,
             3.3750e-07, -5.4022e-04, -1.5422e-05,  5.6052e-45,  3.6990e-04,
             3.6851e-04, -4.8492e-04, -3.4300e-05, -1.4492e-05, -5.2315e-04,
            -1.7087e-07, -1.8778e-04, -1.1772e-04,  7.2602e-04,  8.2353e-05,
             1.1801e-04, -7.6495e-05, -6.1823e-04, -1.9737e-10, -1.3041e-04,
            -1.8452e-03, -6.7736e-06, -5.3150e-05, -2.4154e-04,  1.1406e-04,
             3.3539e-04, -4.8865e-04, -2.3958e-05, -4.2819e-05, -3.1120e-04,
            -1.2215e-05, -1.8287e-04,  1.6531e-04,  9.1417e-05, -4.5539e-04,
             1.3994e-04,  4.5136e-05,  9.5200e-06,  3.8422e-05, -1.9843e-04,
             1.8433e-04, -9.6962e-04, -1.0862e-04,  5.6458e-04,  2.3610e-04,
            -3.7210e-04, -6.0189e-05,  1.0946e-03, -1.0759e-05,  6.2165e-04,
            -3.3889e-04, -1.6923e-03,  5.9440e-05,  1.8638e-05,  1.1267e-04,
             2.4674e-04,  9.2008e-07,  1.2399e-04,  2.5901e-04, -2.0639e-04,
            -6.7167e-04,  5.2466e-04, -2.4921e-04,  2.1985e-04,  4.2930e-04,
            -5.4526e-05, -7.0575e-06, -1.0473e-05,  4.1598e-05,  5.5857e-05,
            -3.4982e-05,  2.9785e-05,  1.0195e-05, -8.1172e-05, -9.5797e-05,
             1.7819e-04,  5.3293e-04,  1.6696e-04,  3.5086e-05,  4.0574e-04,
             4.1801e-04,  1.4045e-05,  1.3502e-04,  3.3930e-04,  5.3970e-04,
             4.8366e-06, -2.2800e-05,  1.7637e-04, -1.3367e-04,  6.6134e-07,
            -7.6568e-05,  2.6509e-05, -1.7254e-04,  5.7555e-04, -4.1443e-05,
             8.4460e-05,  7.0156e-04, -2.0258e-05, -6.5623e-04, -9.8701e-05,
            -8.1752e-05,  4.7091e-06, -1.0151e-06]), 'exp_avg_sq': tensor([7.6907e-07, 5.6437e-06, 3.1294e-06, 4.6518e-06, 6.0130e-06, 1.5373e-06,
            1.0934e-06, 6.7901e-06, 4.5485e-06, 1.8672e-06, 5.1357e-06, 3.7233e-06,
            1.5402e-06, 1.8828e-06, 9.2130e-07, 4.2199e-06, 5.5776e-06, 1.1961e-06,
            6.4486e-06, 5.2119e-06, 5.0504e-06, 6.1551e-07, 5.6928e-06, 4.0122e-06,
            3.5543e-06, 2.7474e-06, 6.1314e-06, 1.0417e-06, 9.0383e-11, 3.3043e-06,
            5.8399e-06, 3.6004e-06, 5.0794e-07, 2.6088e-06, 4.9966e-06, 5.0437e-07,
            4.2044e-06, 3.0332e-06, 8.4421e-06, 8.7030e-07, 4.9481e-06, 3.1368e-08,
            5.8931e-06, 4.3666e-09, 5.1971e-06, 9.2423e-06, 6.4350e-07, 2.8997e-06,
            2.5190e-06, 3.8955e-07, 2.1700e-06, 4.1142e-06, 3.0168e-07, 3.9488e-06,
            1.2919e-06, 6.6820e-07, 2.9851e-06, 1.9923e-06, 1.5860e-06, 5.5307e-06,
            5.0241e-07, 7.1609e-06, 1.2631e-06, 3.4589e-06, 3.9030e-06, 1.2609e-06,
            6.8581e-06, 1.1345e-06, 5.5086e-06, 5.2100e-07, 3.1832e-06, 3.3962e-06,
            3.9177e-06, 6.0988e-07, 4.0214e-06, 3.9222e-06, 4.3557e-06, 1.6606e-06,
            9.7263e-07, 8.2460e-06, 3.2286e-06, 5.8069e-07, 2.3252e-06, 5.9235e-06,
            4.2597e-07, 6.0506e-06, 5.4456e-06, 5.8790e-06, 2.0797e-06, 4.8118e-06,
            6.4540e-07, 5.6915e-07, 1.6480e-07, 4.4769e-06, 1.2334e-07, 6.0121e-07,
            1.1480e-06, 9.6727e-07, 2.4759e-06, 1.1378e-05, 2.1714e-06, 7.4186e-07,
            4.1942e-06, 1.1270e-06, 9.1601e-07, 6.8907e-06, 4.8703e-07, 1.9495e-06,
            2.0869e-06, 5.0826e-06, 7.2962e-07, 2.1816e-07, 4.7466e-06, 4.6380e-06,
            1.2959e-06, 2.4143e-06, 1.1162e-06, 1.8269e-06, 4.9407e-06, 1.3954e-06,
            7.0093e-06, 6.1579e-06, 1.3861e-06, 6.8897e-06, 1.0757e-06, 1.7799e-06,
            3.2545e-07, 8.2589e-07])}, 139900461526040: {'step': 4690, 'exp_avg': tensor([[-3.0439e-08, -1.2409e-08,  3.4250e-04,  ..., -5.5884e-12,
             -8.4041e-07, -5.6052e-45],
            [ 4.3944e-40,  3.4023e-05, -1.9848e-04,  ...,  1.1912e-06,
              3.4964e-08,  4.7000e-07],
            [ 1.1539e-38, -4.2105e-08,  1.0535e-03,  ..., -2.6653e-07,
             -5.0220e-07, -3.4162e-05],
            ...,
            [ 5.6052e-45, -3.8936e-33,  1.7960e-03,  ...,  3.6836e-08,
              3.6358e-10, -1.0499e-06],
            [-6.7221e-37,  5.6052e-45, -6.8850e-07,  ..., -4.7001e-26,
              5.6052e-45, -1.4941e-31],
            [-1.3073e-19,  3.9909e-16, -3.0864e-06,  ...,  4.5565e-09,
             -1.1513e-06, -3.4462e-05]]), 'exp_avg_sq': tensor([[5.5368e-06, 1.3295e-06, 3.6648e-05,  ..., 4.4462e-07, 4.1638e-06,
             3.2714e-06],
            [1.6893e-08, 3.1863e-07, 1.0157e-04,  ..., 2.0351e-06, 9.8976e-06,
             8.9255e-06],
            [2.3075e-07, 7.5221e-06, 1.1583e-04,  ..., 1.7064e-06, 4.6344e-06,
             9.6035e-06],
            ...,
            [8.4200e-08, 1.6446e-06, 6.8540e-05,  ..., 1.3699e-06, 8.0277e-07,
             1.1968e-06],
            [2.1728e-10, 3.1512e-08, 1.0583e-05,  ..., 1.3092e-06, 1.2653e-08,
             1.5370e-09],
            [1.7704e-06, 6.0989e-07, 1.3985e-05,  ..., 1.5147e-07, 4.0570e-07,
             1.7411e-06]])}, 139900461525248: {'step': 4690, 'exp_avg': tensor([-2.2409e-04, -1.0514e-04, -2.2531e-05,  1.1361e-04,  2.1007e-05,
            -7.5251e-05, -1.6207e-04, -2.6700e-05, -5.5658e-04, -3.2277e-04,
            -4.9651e-16, -3.2694e-04,  2.3576e-04,  2.2112e-04, -6.9036e-04,
            -3.5538e-06, -1.1465e-04,  1.7443e-04,  4.8515e-04, -6.5063e-04,
            -3.7166e-04,  1.1352e-11,  2.0714e-07, -4.5680e-05, -5.8665e-04,
            -1.0959e-06,  1.3370e-03, -7.2549e-04,  5.1173e-05,  5.9709e-04,
            -9.6832e-04,  6.1406e-04, -4.4309e-04, -6.8583e-06, -6.7363e-04,
             1.0745e-39,  6.8978e-05, -2.8864e-04, -6.5218e-07, -4.4942e-05,
            -8.8874e-05, -2.6537e-04,  2.1660e-08, -3.9327e-04,  4.3083e-18,
            -6.9570e-05,  1.8185e-04,  6.1726e-05,  8.5478e-10,  4.1923e-07,
             2.6255e-04,  7.3841e-06,  1.7282e-05,  1.2687e-08,  1.2880e-04,
            -3.3075e-05, -9.1281e-04,  7.3394e-04,  1.1153e-10,  4.5372e-04,
             1.5997e-03,  4.6362e-04, -1.8101e-05, -8.4351e-04]), 'exp_avg_sq': tensor([4.0402e-06, 3.5730e-06, 6.8574e-06, 7.6486e-07, 1.6507e-06, 3.3100e-06,
            6.2310e-06, 5.0227e-06, 8.1812e-06, 3.7582e-06, 2.7292e-07, 3.3632e-06,
            3.7643e-06, 1.1877e-06, 6.9635e-06, 5.0682e-06, 2.3374e-06, 5.3952e-06,
            1.2488e-06, 9.0824e-06, 3.4307e-06, 2.0671e-07, 6.5342e-07, 4.5825e-06,
            3.4935e-06, 2.0613e-07, 8.4145e-06, 6.0097e-06, 2.6297e-06, 7.9023e-06,
            1.1026e-05, 5.2180e-06, 3.0439e-06, 1.0718e-06, 4.8543e-06, 3.6846e-09,
            4.3594e-06, 7.5386e-06, 4.8594e-07, 4.6684e-06, 6.2848e-07, 7.2960e-06,
            5.4032e-07, 3.3385e-06, 4.4825e-07, 3.0315e-06, 6.7744e-06, 4.7726e-07,
            6.9378e-07, 8.7345e-08, 2.4895e-06, 1.3262e-06, 1.1211e-06, 4.1199e-07,
            3.7932e-06, 3.8700e-06, 6.2931e-06, 4.7584e-06, 1.1123e-07, 5.7224e-06,
            7.9200e-06, 4.3516e-06, 7.1062e-07, 4.8290e-06])}, 139900461525680: {'step': 4690, 'exp_avg': tensor([[ 6.0655e-05, -1.2791e-02, -9.2147e-04,  2.1673e-05, -4.1058e-04,
              2.2092e-04,  9.3630e-04, -8.4244e-04,  1.1652e-03, -2.9285e-02,
              3.1146e-20,  2.5363e-05, -2.6613e-03,  7.2318e-05,  4.5017e-04,
             -4.3325e-04,  4.0996e-07, -2.2132e-03,  6.3804e-05,  3.9406e-03,
             -8.2670e-03,  7.2933e-12,  1.5404e-10, -4.9675e-03,  3.6926e-04,
              2.1772e-08, -2.3323e-03,  3.3699e-04,  1.5331e-05,  1.0594e-03,
              2.3482e-03, -2.7367e-03,  3.2953e-04,  1.3379e-06,  2.6193e-04,
              6.3114e-42,  1.3882e-04,  9.2218e-04,  8.5543e-09,  1.8975e-03,
              3.7938e-05, -1.7319e-03,  4.5561e-07, -3.8550e-03,  2.8505e-20,
              6.2950e-02, -9.5540e-05,  9.2063e-06,  4.8318e-12,  2.0646e-09,
              3.6704e-04, -1.2727e-03,  1.1521e-10,  2.3292e-08, -8.8260e-03,
             -6.6115e-05,  4.3013e-04, -4.1527e-03,  1.2339e-20, -1.0495e-02,
              7.0002e-04, -3.3688e-03,  2.8396e-06,  1.6550e-04],
            [ 6.4342e-04, -7.0615e-04, -1.7810e-03,  3.9195e-05,  1.8642e-04,
              3.3421e-04,  2.8177e-04, -1.4400e-02,  3.6164e-05,  1.5856e-04,
              2.2252e-21,  2.1819e-04,  3.7965e-04,  1.6275e-03,  8.4986e-04,
              3.5819e-03, -4.8866e-04,  2.3106e-03,  3.1386e-05,  1.4170e-03,
             -4.9496e-04,  9.1555e-13,  3.0923e-09,  1.5971e-03, -1.3562e-02,
              2.4855e-08,  1.0632e-02,  8.3045e-04, -2.3467e-04,  1.0786e-03,
              2.4902e-03,  6.2346e-04,  4.5365e-04,  1.3266e-06,  3.2794e-04,
              6.1097e-42, -2.6325e-03,  1.1108e-03,  2.8952e-08,  1.4671e-02,
              2.4318e-05, -2.0775e-04,  1.5423e-06,  2.2969e-04,  4.5046e-21,
              2.8734e-04, -9.6467e-03,  7.2479e-05,  6.7430e-11,  2.7392e-06,
              4.4603e-02,  3.7826e-05,  9.4647e-10,  5.6905e-12, -3.3269e-03,
              3.9343e-03,  2.8985e-03, -5.2099e-03,  6.3759e-11, -3.5141e-03,
              1.1085e-03, -2.1191e-03,  8.5595e-06, -6.2178e-03],
            [-1.1279e-03,  1.0543e-04,  1.6612e-03,  3.4101e-05,  2.0064e-05,
              3.1949e-03,  1.2260e-03,  3.9997e-03,  2.9635e-03,  4.1801e-03,
              3.0246e-18,  6.1391e-04,  1.8425e-04,  9.9576e-06, -9.1689e-04,
              7.2249e-05,  1.9875e-05,  1.2135e-03,  1.2557e-04,  4.0164e-03,
              5.7576e-03, -6.3479e-10, -9.9287e-07, -1.8478e-03,  1.7606e-03,
              4.1317e-07, -1.0264e-02,  3.6387e-03, -1.1257e-02,  2.7415e-04,
             -2.1603e-03,  1.0014e-02,  3.1992e-04,  2.4815e-06, -7.5049e-04,
              5.8587e-41,  7.6634e-04,  4.6535e-04,  5.2662e-08,  2.2997e-03,
              2.3940e-05,  1.5604e-03,  2.8048e-06,  2.3474e-03,  3.3118e-20,
             -7.3655e-02,  1.3692e-03,  1.2989e-05,  7.3255e-12, -7.9618e-06,
              2.5449e-04, -3.6612e-04,  6.9422e-10,  1.9942e-10,  7.1586e-04,
              1.0598e-02,  2.2502e-02, -3.0736e-03,  5.1294e-11,  8.0015e-04,
              3.7301e-03,  3.4588e-04,  4.6960e-05,  5.2406e-03],
            [ 9.7803e-04, -5.7591e-03,  2.8518e-03, -7.3398e-05,  2.5469e-05,
              1.7901e-03,  2.4835e-03,  3.5200e-04,  3.3197e-03, -1.0286e-03,
              5.8132e-20,  1.9056e-04,  1.8554e-04,  3.3539e-05,  3.4421e-03,
             -1.6499e-02,  4.6298e-05,  3.6397e-03,  2.0179e-05, -2.4890e-03,
              2.0947e-04,  7.2090e-12,  1.2930e-08,  4.4958e-03,  2.3325e-04,
              7.2660e-07, -5.8211e-03,  7.4462e-04,  3.0254e-02, -1.2488e-03,
              3.8391e-03,  1.3244e-04,  1.8818e-03,  6.5810e-06,  5.2162e-04,
              2.2190e-41, -1.9189e-03,  3.3726e-03,  1.9136e-07,  1.9461e-03,
              5.6933e-06,  3.4045e-04,  1.0193e-05,  9.9102e-04,  2.0472e-20,
             -1.7063e-04,  6.7598e-03,  9.5516e-05, -2.4221e-09,  4.9838e-06,
             -4.8239e-04, -1.6009e-03,  2.7525e-10,  1.0924e-10,  3.1977e-04,
             -1.6310e-02, -3.8442e-02,  3.0997e-02,  1.2784e-10,  6.0948e-03,
              1.9830e-03, -1.9069e-03,  2.3035e-05,  7.5605e-04],
            [-4.5912e-05,  4.8757e-04,  1.2643e-04,  5.5841e-05,  1.1495e-05,
              1.3659e-04, -4.0086e-03,  1.6297e-03,  5.6420e-04,  8.8125e-04,
              3.0042e-16,  1.9286e-04, -1.4206e-02,  3.5235e-03,  4.1370e-02,
              2.1637e-03,  8.2001e-05,  2.1773e-03,  3.8327e-03,  6.3733e-03,
             -1.8607e-03,  2.4579e-13,  3.5993e-08,  3.1144e-04, -1.2892e-02,
              1.5482e-07, -9.1971e-04,  2.5765e-02,  1.8947e-05,  3.6488e-04,
              2.5485e-02,  8.4330e-04,  1.3146e-03, -2.0034e-05, -6.5345e-03,
              2.3262e-41,  3.1501e-03, -2.3027e-02, -7.7443e-07,  6.3521e-04,
             -4.2126e-04, -1.5364e-02, -4.2412e-05, -1.6689e-03, -1.2324e-17,
              3.5199e-04,  3.9292e-02, -6.4763e-04,  1.5920e-10,  1.0208e-07,
             -2.6229e-03,  2.8661e-05,  1.1051e-08,  5.4056e-12, -1.3071e-02,
             -3.2830e-04,  6.1126e-03,  4.1686e-04,  1.8591e-10,  4.6347e-04,
              1.2427e-02,  1.2250e-04,  3.1720e-06,  7.9513e-03],
            [ 8.6165e-04,  1.1891e-02, -5.9651e-03,  6.7274e-05, -1.2139e-04,
              5.6340e-04, -1.7675e-02,  1.5725e-03, -1.5175e-02,  2.0766e-02,
              4.0862e-14,  1.1383e-04, -8.0797e-03,  1.2325e-05,  6.3148e-03,
              3.5843e-03,  4.1690e-06,  8.0564e-04, -3.6879e-03,  8.4207e-03,
              2.3316e-04,  1.0453e-13,  1.6824e-09,  5.8055e-04,  1.8235e-03,
             -7.2947e-06, -6.8145e-04, -5.1994e-03, -1.8644e-02, -4.0514e-03,
             -1.8055e-04,  1.0952e-03, -1.9157e-04,  2.1720e-06,  6.6323e-04,
              4.0198e-41,  5.6886e-04, -6.1873e-03,  6.8599e-08,  1.6068e-03,
              1.8089e-05,  3.1182e-04,  3.8937e-06,  1.5155e-03,  2.0718e-20,
              1.2753e-02, -1.5339e-02,  6.1533e-05,  2.4695e-09,  4.0058e-09,
              5.7953e-04,  1.7506e-03, -2.6567e-05, -2.5048e-08,  3.4162e-03,
              5.8968e-04,  6.1135e-04, -2.0262e-02,  3.2702e-15, -9.9731e-05,
              1.3353e-03,  6.3181e-03,  4.5248e-06,  3.0445e-04],
            [ 8.3590e-05,  1.5211e-03,  8.4773e-04,  1.7138e-05,  6.4443e-04,
             -9.4895e-03,  7.9392e-03,  2.7951e-03,  3.9809e-03,  5.4222e-03,
              2.7446e-20,  2.6707e-05,  2.2772e-02,  1.2406e-06,  1.4238e-04,
              2.3956e-03,  2.8610e-07,  7.5274e-04,  4.0841e-05,  9.5999e-04,
              3.0001e-03,  3.2045e-14,  1.5596e-11,  1.4300e-04,  1.6838e-02,
              1.4031e-07, -2.4844e-03,  5.2618e-04,  9.2018e-07, -1.6985e-04,
              8.8030e-04, -1.3556e-03, -5.0838e-03,  6.1005e-08,  5.1764e-03,
              3.7975e-43,  2.9285e-05,  1.1486e-04,  2.3927e-10, -1.8018e-02,
              5.3103e-06,  1.8788e-02,  9.3153e-07, -1.5259e-03,  7.4597e-25,
             -5.0114e-03,  4.8150e-04,  1.1210e-06, -2.8871e-10,  3.5563e-09,
             -4.3094e-02,  7.8910e-05,  2.6555e-05,  1.5047e-10,  1.9773e-02,
              2.1260e-04,  2.6984e-04,  2.5723e-04,  3.0052e-22,  9.8502e-05,
             -9.6666e-04,  4.8448e-05,  3.6815e-07,  6.8641e-05],
            [-3.0544e-03,  3.7554e-04,  1.9600e-04, -2.3234e-04,  5.2573e-07,
              2.4601e-04,  3.1509e-05,  1.3949e-03,  3.9377e-04, -6.7361e-03,
              2.6346e-21, -1.2944e-03,  9.5232e-05,  9.0406e-06,  7.2170e-03,
              2.8510e-03, -2.4961e-04,  1.4493e-03,  5.4571e-05,  7.2805e-03,
              3.2949e-04,  6.1733e-10,  4.8037e-07,  1.5017e-02,  4.9188e-03,
              5.0866e-07, -9.1993e-03,  4.7141e-03, -2.8186e-04,  3.5682e-04,
              4.1818e-03, -9.4393e-03,  1.5570e-04, -2.0763e-06,  5.3853e-04,
             -2.0982e-40,  5.1340e-05,  2.8537e-03,  6.5580e-08,  3.8495e-03,
              9.7204e-05,  2.1208e-04,  3.4928e-06,  2.7584e-04,  1.1974e-17,
              4.2425e-04, -4.9946e-03,  1.0017e-04,  1.4705e-12,  1.2567e-07,
             -1.2158e-04,  6.3444e-04,  3.7228e-11,  4.3327e-10,  2.6831e-04,
              5.1502e-04,  2.3114e-03,  9.1195e-04, -7.4339e-10,  1.1900e-03,
              9.0714e-03,  1.3563e-04, -1.1799e-04, -1.1163e-02],
            [ 4.6634e-04,  1.6397e-03,  1.1240e-03,  2.6158e-05, -3.5681e-04,
              2.5724e-03,  6.3749e-03,  1.5753e-03,  1.5058e-03,  2.0195e-03,
              1.0866e-20,  4.2717e-05,  2.6325e-04,  2.5235e-04,  1.4457e-03,
              1.9416e-03,  9.8937e-07,  9.1179e-04,  9.9483e-05, -2.5729e-04,
              9.2125e-04,  2.8425e-13,  1.1911e-09,  6.8082e-04,  1.3983e-03,
              1.7459e-07,  2.3272e-02,  1.2354e-03,  4.7895e-05,  5.0578e-03,
              1.6723e-03, -7.3700e-04,  4.8377e-04,  1.2666e-06, -2.8294e-04,
              2.3780e-42, -2.1041e-04,  9.5778e-04,  3.1583e-08, -6.5431e-03,
              1.9087e-05, -4.9716e-03,  1.6822e-06,  2.4183e-03,  1.0229e-22,
              1.7014e-03,  1.1845e-03,  8.0721e-05,  8.8632e-11,  1.0727e-09,
              7.4483e-04,  3.1628e-04, -3.2164e-10,  3.9585e-10,  3.1032e-04,
              5.6509e-04,  1.5096e-03,  3.7662e-04,  2.1429e-17,  2.6395e-04,
              2.1676e-03,  2.2728e-04,  2.4134e-05,  7.8617e-04],
            [ 1.1345e-03,  3.2359e-03,  1.8605e-03,  4.4360e-05,  3.6574e-07,
              4.3105e-04,  2.4107e-03,  1.9232e-03,  1.2456e-03,  3.6221e-03,
              2.1307e-21, -1.2978e-04,  1.0668e-03, -5.5418e-03, -6.0315e-02,
              3.4172e-04,  5.8420e-04, -1.1047e-02, -5.8058e-04, -2.9662e-02,
              1.7160e-04,  1.3778e-12,  4.5743e-07, -1.6010e-02, -8.8802e-04,
              5.1299e-06, -2.2019e-03, -3.2592e-02,  7.9672e-05, -2.7215e-03,
             -3.8556e-02,  1.5604e-03,  3.3649e-04,  6.8843e-06,  7.8335e-05,
              5.0396e-41,  5.7114e-05,  1.9418e-02,  3.2690e-07, -2.3451e-03,
              1.8968e-04,  1.0626e-03,  1.7417e-05, -7.2778e-04,  2.4169e-19,
              3.6981e-04, -1.9011e-02,  2.1390e-04,  9.0513e-11,  3.2138e-10,
             -2.2816e-04,  3.9303e-04,  1.0763e-11,  4.5653e-10,  4.2132e-04,
              2.8941e-04,  1.7965e-03, -2.6072e-04,  3.2005e-10,  5.1979e-03,
             -3.1556e-02,  1.9705e-04,  4.4002e-06,  2.1080e-03]]), 'exp_avg_sq': tensor([[2.4164e-05, 1.5559e-04, 4.9004e-04, 4.2817e-03, 4.2696e-03, 5.6991e-05,
             7.0878e-04, 2.2812e-03, 4.1725e-04, 1.0499e-02, 1.2936e-05, 8.0487e-04,
             1.0903e-02, 5.0636e-06, 6.3305e-05, 7.6545e-04, 6.0395e-05, 3.6449e-04,
             1.1528e-05, 4.7764e-04, 1.1837e-02, 4.2758e-06, 2.2903e-06, 1.3890e-04,
             3.8392e-04, 1.6142e-06, 6.0205e-04, 2.6468e-05, 7.2946e-06, 3.0632e-04,
             2.9315e-04, 2.2921e-04, 9.0373e-04, 3.9224e-05, 2.0212e-04, 2.8421e-10,
             1.9488e-04, 1.4723e-04, 1.0936e-04, 8.0113e-05, 2.1675e-05, 1.5725e-03,
             1.4320e-04, 7.7003e-03, 5.7100e-06, 1.0897e-02, 9.5812e-05, 2.5235e-05,
             1.6631e-05, 1.4318e-06, 9.2698e-05, 2.4159e-04, 1.5199e-05, 2.6125e-06,
             8.6782e-04, 3.5281e-04, 6.5236e-05, 1.3798e-03, 9.2791e-08, 1.5916e-04,
             9.3519e-05, 9.0455e-05, 1.1131e-05, 1.2766e-03],
            [3.8587e-04, 1.4561e-04, 1.3880e-04, 1.7641e-05, 1.3557e-04, 6.7983e-03,
             8.5566e-03, 3.0924e-04, 2.6288e-04, 2.4972e-05, 5.4221e-06, 1.4623e-04,
             2.8628e-05, 1.6132e-04, 9.1792e-05, 2.5228e-03, 6.9330e-03, 5.2217e-04,
             2.0897e-05, 3.0695e-04, 4.2437e-05, 6.0857e-06, 8.2916e-06, 5.9907e-04,
             2.1935e-04, 5.2916e-07, 7.7861e-03, 1.3107e-04, 4.8548e-05, 6.3602e-04,
             3.9591e-04, 5.4950e-04, 8.4956e-05, 5.5988e-06, 1.6118e-04, 1.4550e-11,
             3.7867e-03, 9.7725e-05, 4.9165e-07, 7.7912e-03, 2.6635e-06, 5.5492e-04,
             6.8113e-05, 2.1415e-04, 4.1019e-07, 1.1067e-04, 8.8134e-03, 1.5215e-04,
             8.7193e-06, 1.1796e-06, 7.8883e-03, 9.6584e-06, 5.7494e-05, 1.5763e-05,
             1.4339e-04, 3.8076e-03, 4.4477e-03, 1.6352e-04, 3.9365e-07, 5.3115e-04,
             3.2113e-04, 1.9658e-04, 5.0106e-06, 2.9971e-03],
            [8.7349e-04, 2.3135e-04, 2.6037e-04, 8.6410e-06, 7.6225e-04, 7.8016e-04,
             8.6167e-04, 1.2492e-03, 3.8084e-04, 5.5150e-04, 7.7308e-06, 3.1419e-04,
             1.8014e-03, 5.4168e-06, 3.2804e-05, 1.5210e-02, 1.8171e-04, 1.1814e-02,
             7.8643e-05, 6.6446e-04, 6.4680e-04, 9.3659e-05, 5.1863e-04, 1.1150e-02,
             6.9256e-04, 2.6569e-05, 1.5326e-02, 6.4267e-04, 3.4477e-04, 2.5281e-03,
             4.6229e-04, 3.1563e-03, 5.0051e-05, 1.5863e-04, 2.2743e-03, 9.8059e-07,
             1.2086e-02, 8.6680e-05, 3.5019e-05, 4.5565e-04, 9.2292e-05, 1.5800e-03,
             2.9017e-05, 1.3727e-03, 6.8679e-06, 6.3272e-03, 7.8440e-04, 3.1849e-05,
             3.4480e-05, 1.5630e-05, 3.4464e-04, 4.5204e-05, 3.0621e-03, 1.6238e-06,
             3.5814e-03, 7.5220e-03, 2.9363e-03, 1.3859e-03, 4.6726e-06, 8.8095e-04,
             2.9528e-03, 1.9689e-03, 1.8962e-04, 6.4042e-04],
            [1.6313e-03, 5.7322e-03, 1.2792e-02, 1.2909e-05, 7.4644e-05, 2.7810e-04,
             2.1825e-03, 6.4218e-05, 3.1835e-03, 1.1975e-03, 7.9596e-06, 2.0242e-04,
             2.0739e-04, 1.0373e-06, 1.3368e-03, 2.7470e-03, 3.8988e-04, 4.6680e-03,
             3.8751e-05, 4.5940e-03, 1.4962e-04, 1.6152e-04, 2.2023e-04, 5.1635e-03,
             6.4286e-05, 9.9271e-06, 5.5220e-03, 9.1891e-05, 2.0917e-03, 5.1408e-03,
             1.1389e-03, 5.9105e-04, 2.6039e-03, 1.1157e-04, 1.7617e-04, 1.6455e-11,
             5.1003e-03, 6.9596e-04, 3.7354e-05, 1.6628e-04, 7.4673e-05, 5.3079e-04,
             1.8227e-05, 2.0775e-04, 1.0798e-06, 2.2224e-03, 4.2448e-04, 3.2899e-05,
             2.2465e-06, 4.6429e-06, 1.2599e-04, 6.5327e-04, 4.7178e-06, 1.5826e-05,
             1.0118e-03, 1.8432e-03, 1.2135e-02, 4.9859e-03, 5.5584e-06, 1.5345e-02,
             6.4265e-03, 5.6669e-03, 4.7562e-05, 4.9257e-04],
            [3.3014e-04, 4.8402e-05, 4.1317e-05, 6.2709e-06, 9.0085e-04, 2.3317e-04,
             1.1321e-03, 4.9914e-03, 1.4513e-04, 4.8733e-05, 1.4850e-05, 3.6200e-03,
             1.9137e-03, 1.8203e-05, 3.6546e-03, 4.3683e-04, 1.3802e-04, 6.5039e-04,
             6.0821e-04, 1.5325e-03, 2.4236e-04, 7.1910e-06, 2.1294e-04, 3.5259e-04,
             4.8553e-03, 3.2577e-05, 3.2449e-04, 7.9946e-03, 6.4820e-07, 1.8158e-04,
             9.2142e-03, 2.2787e-04, 8.3153e-05, 9.6754e-05, 7.4774e-03, 9.8169e-07,
             3.5174e-04, 8.7753e-03, 3.8230e-05, 5.7265e-03, 3.4879e-05, 5.3198e-03,
             4.6136e-04, 5.1934e-03, 4.3138e-06, 2.3609e-04, 1.5536e-02, 2.5896e-05,
             1.3583e-03, 3.5339e-07, 3.6742e-04, 7.7368e-06, 3.8906e-05, 9.8683e-06,
             1.9202e-03, 1.8554e-04, 2.8577e-04, 1.0176e-04, 9.7532e-07, 1.7562e-05,
             1.1038e-03, 7.7931e-06, 1.3094e-05, 7.2228e-04],
            [3.1145e-04, 7.3497e-03, 1.5531e-02, 4.4441e-03, 3.8136e-03, 4.9348e-04,
             1.8772e-02, 2.0943e-03, 1.2333e-02, 1.1015e-02, 7.4249e-06, 3.2444e-04,
             2.4073e-03, 2.9479e-07, 1.8238e-03, 3.6008e-04, 1.9831e-04, 1.9424e-04,
             5.2191e-05, 4.7311e-03, 2.9817e-03, 1.9319e-06, 5.5457e-06, 6.3013e-04,
             2.3417e-04, 1.7613e-06, 6.3445e-04, 1.6621e-04, 9.7612e-04, 1.3269e-03,
             5.1305e-04, 2.3724e-04, 8.8587e-03, 1.1898e-04, 8.9317e-04, 2.6590e-09,
             1.3051e-04, 2.7351e-04, 7.1842e-06, 5.0460e-04, 4.7999e-06, 6.8255e-03,
             5.3463e-04, 3.9659e-04, 3.3184e-07, 8.5956e-04, 4.3205e-04, 3.6133e-05,
             1.8923e-04, 4.9614e-06, 3.3949e-04, 1.0495e-03, 2.0700e-04, 1.1520e-05,
             2.8843e-03, 1.8133e-04, 5.6089e-03, 1.8066e-03, 2.0722e-06, 1.1993e-02,
             7.4974e-04, 2.7766e-03, 4.6744e-05, 1.6356e-04],
            [2.6879e-05, 6.1430e-04, 8.0586e-04, 6.8625e-06, 6.3279e-03, 4.8947e-04,
             1.9013e-02, 1.2145e-02, 1.2341e-03, 1.8458e-03, 5.7258e-05, 1.1582e-04,
             1.4309e-02, 6.4429e-07, 4.7125e-05, 7.6980e-03, 3.2862e-05, 2.4731e-04,
             4.6082e-05, 3.2475e-04, 1.9050e-03, 4.7876e-06, 2.5992e-06, 2.3457e-05,
             1.0385e-03, 4.0828e-06, 4.9824e-04, 4.5297e-04, 1.2199e-05, 1.3741e-04,
             1.2760e-04, 1.6559e-04, 2.2031e-03, 2.0861e-06, 2.2734e-03, 7.5066e-11,
             8.8238e-04, 2.3420e-05, 6.7687e-06, 9.3732e-04, 5.4162e-06, 1.5915e-02,
             1.3420e-03, 2.3563e-03, 6.8302e-07, 2.2981e-03, 1.7282e-04, 8.3618e-05,
             1.2795e-03, 4.1999e-06, 1.7416e-03, 3.0051e-04, 3.3097e-03, 1.2884e-06,
             8.1157e-03, 1.3881e-04, 2.3500e-05, 1.7407e-04, 2.2427e-07, 1.6615e-05,
             6.9270e-06, 5.4169e-06, 1.9349e-05, 4.5128e-06],
            [5.0843e-03, 1.2056e-05, 7.7608e-05, 4.4238e-07, 3.6010e-05, 7.6961e-04,
             4.6192e-04, 1.4441e-04, 7.4425e-05, 1.9038e-03, 5.2849e-06, 4.1401e-03,
             3.4912e-04, 6.1149e-06, 1.7145e-03, 1.4852e-03, 1.3541e-03, 8.4019e-03,
             9.8086e-05, 1.4835e-02, 1.6850e-03, 1.6450e-04, 6.5570e-04, 1.1556e-02,
             9.1652e-04, 1.3033e-05, 1.5697e-03, 1.3320e-03, 2.8942e-04, 8.6521e-05,
             2.2637e-03, 7.3743e-04, 2.0854e-04, 1.0903e-03, 5.9329e-04, 3.3587e-09,
             1.1817e-03, 2.2900e-03, 8.7853e-05, 1.0963e-03, 3.2603e-04, 3.6227e-05,
             1.8476e-04, 9.6693e-04, 6.4800e-06, 2.8987e-04, 3.5510e-03, 4.4714e-05,
             2.1768e-06, 1.2286e-05, 3.2108e-03, 1.9310e-04, 4.4242e-05, 2.5936e-06,
             3.5847e-05, 2.5237e-03, 1.4915e-03, 1.8955e-04, 1.6115e-05, 4.8602e-04,
             1.1438e-02, 1.4020e-04, 2.5649e-04, 6.1717e-03],
            [1.0891e-04, 7.6459e-04, 1.2062e-03, 2.6042e-06, 5.1874e-04, 5.0040e-03,
             3.8758e-03, 1.7160e-03, 1.0466e-02, 1.2049e-03, 1.4538e-05, 8.4335e-05,
             5.5011e-04, 1.5327e-04, 5.6150e-04, 8.3712e-04, 5.7238e-03, 3.1252e-04,
             1.4978e-05, 7.8303e-04, 6.0714e-04, 1.3621e-05, 4.1742e-06, 2.5527e-04,
             1.1156e-04, 4.3493e-07, 8.8517e-03, 1.2171e-04, 5.6126e-04, 6.3933e-03,
             5.8039e-03, 4.5366e-03, 2.9123e-03, 2.5333e-06, 2.8578e-04, 4.6301e-10,
             3.8033e-04, 6.2766e-04, 2.9803e-06, 1.1533e-03, 5.8022e-07, 6.4181e-03,
             6.5587e-05, 5.5509e-04, 8.7293e-07, 1.4502e-03, 1.2992e-03, 1.5177e-04,
             3.4890e-06, 3.8776e-06, 1.9870e-03, 1.1059e-04, 3.4291e-05, 2.2495e-07,
             4.5465e-04, 4.1793e-04, 8.5964e-04, 1.3323e-03, 3.5328e-07, 4.8181e-04,
             4.1762e-04, 2.4789e-04, 1.1756e-05, 1.2546e-04],
            [2.2024e-03, 2.9902e-04, 1.9882e-03, 1.8885e-05, 9.8413e-06, 3.6813e-04,
             6.4192e-04, 5.0374e-04, 7.0906e-04, 2.1616e-03, 8.7569e-06, 1.1760e-03,
             1.1912e-03, 3.7721e-05, 8.6972e-03, 1.7531e-05, 2.6118e-04, 6.7387e-04,
             4.6890e-04, 1.5202e-02, 5.4383e-03, 6.6620e-06, 1.9690e-04, 1.8621e-03,
             1.6581e-03, 1.1339e-05, 2.8631e-04, 7.0603e-03, 1.3334e-05, 1.0759e-03,
             1.5021e-02, 3.3081e-04, 6.6050e-04, 6.8880e-04, 2.8214e-03, 4.5196e-10,
             4.2421e-05, 1.2701e-02, 4.4204e-05, 7.8267e-04, 9.9904e-05, 3.1756e-04,
             1.0566e-04, 3.2786e-03, 6.5071e-06, 2.0823e-03, 1.2363e-02, 2.8420e-06,
             6.0493e-06, 1.4831e-06, 2.7018e-04, 4.4944e-05, 3.4502e-05, 2.0980e-05,
             1.6774e-04, 3.1850e-05, 1.8613e-03, 1.0287e-03, 6.3262e-06, 1.1435e-03,
             4.6527e-03, 2.1994e-04, 1.7640e-05, 6.2338e-03]])}, 139900461525608: {'step': 4690, 'exp_avg': tensor([-1.3817e-03, -1.5057e-03,  5.2111e-03,  1.2903e-03,  4.3707e-03,
            -8.3946e-05, -1.6424e-03, -1.2887e-03,  5.8877e-04, -5.5585e-03]), 'exp_avg_sq': tensor([1.1082e-04, 9.8747e-05, 2.1626e-04, 2.3502e-04, 1.9139e-04, 2.0084e-04,
            1.3883e-04, 1.6299e-04, 2.7077e-04, 2.8351e-04])}}
    param_groups 	 [{'lr': 0.01, 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False, 'params': [139900461526256, 139900461526328, 139900461525392, 139900461526112, 139900461526040, 139900461525248, 139900461525680, 139900461525608]}]

#  Try Running the below codes on  Kaggle Kernel

[Check for reference](https://www.kaggle.com/u6yuvi/digit-recognizer-pytorch/output?scriptVersionId=16485375)

# Kaggle- Multilayered Perceptron (MLP) implemention on MNIST dataset
Untill now we were using the MNIST dataset that is available in torchvision.dataset.Let us now load the dataset from Kaggle repo and train our model

```python
import os
from pathlib import Path
import torch
from torch.utils.data import TensorDataset ,DataLoader
from torch import nn,optim
import torch.nn.functional as F
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
PATH=Path("../input/digit-recognizer")
print(os.listdir("../input/digit-recognizer"))
```

## Load Data

```python
train=pd.read_csv(PATH/'train.csv')
test=pd.read_csv(PATH/'test.csv')
train.shape,test.shape
```

## Extracting Input and Target Variable

```python
x=train.drop("label",axis=1)
y=np.array(train['label'])
x.shape,y.shape
```

## Normalization

```python
#x_train=x/255
```

## Train Test Split

```python
#x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.15)
```

## Train Test in Pytorch

```python
'''
# create feature and targets tensor for train set.
torch_X_train = torch.from_numpy(x_train.values).type(torch.FloatTensor)
torch_y_train = torch.from_numpy(y_train).type(torch.LongTensor)

# create feature and targets tensor for test set.
torch_X_test = torch.from_numpy(x_test.values).type(torch.FloatTensor)
torch_y_test = torch.from_numpy(y_test).type(torch.LongTensor)

# Pytorch train and test sets
train = torch.utils.data.TensorDataset(torch_X_train,torch_y_train)
test = torch.utils.data.TensorDataset(torch_X_test,torch_y_test)
'''
```

```python
'''
BATCH_SIZE=64
# data loader
train_loader = torch.utils.data.DataLoader(train, batch_size = BATCH_SIZE, shuffle = False)
test_loader = torch.utils.data.DataLoader(test, batch_size = BATCH_SIZE, shuffle = False)
'''
```

## Train -Test Split -Pytorch

```python
torch_X_train = torch.from_numpy(x.values).type(torch.FloatTensor)/255
torch_y_train = torch.from_numpy(y).type(torch.LongTensor)
myDataset = torch.utils.data.TensorDataset(torch_X_train,torch_y_train)
valid_no  = int(0.2 * len(myDataset))
# so divide the data into trainset and testset
trainSet,testSet = torch.utils.data.random_split(myDataset,(len(myDataset)-valid_no,valid_no))
print(f"len of trainSet {len(trainSet)} , len of testSet {len(testSet)}")
batch_size=64
train_loader  = DataLoader(trainSet , batch_size=batch_size ,shuffle=True)
test_loader  = DataLoader(testSet , batch_size=batch_size ,shuffle=True)
```

trainData = torch.from_numpy(x_train.values)
trainLabel=torch.from_numpy(y_train)
testData = torch.from_numpy(x_test.values)
testLabel = torch.from_numpy(y_test)
trainData, testData = trainData.type(torch.FloatTensor), testData.type(torch.LongTensor)
trainLabel, testLabel = trainLabel.type(torch.FloatTensor), testLabel.type(torch.LongTensor)
trainData.shape,testData.shape
trainData = trainData.unsqueeze_(dim=1)
testData = testData.unsqueeze_(dim=1)
trainData.shape,testData.shape
transforms =transforms.Compose(transforms.ToTensor())
train_dataset = TensorDataset(trainData,trainLabel)
train_loader = DataLoader(train_dataset,batch_size=64,shuffle=True)

test_dataset = TensorDataset(testData,testLabel)
test_loader = DataLoader(test_dataset,batch_size=64,shuffle=True)

## Network

```python
class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(784, 256)
        self.fc2 = nn.Linear(256, 128)
        self.fc3 = nn.Linear(128, 64)
        self.fc4 = nn.Linear(64, 10)

        # Dropout module with 0.2 drop probability
        self.dropout = nn.Dropout(p=0.2)

    def forward(self, x):
        # make sure input tensor is flattened
        x = x.view(x.shape[0], -1)

        # Now with dropout
        x = self.dropout(F.relu(self.fc1(x)))
        x = self.dropout(F.relu(self.fc2(x)))
        x = self.dropout(F.relu(self.fc3(x)))

        # output so no dropout here
        x = F.log_softmax(self.fc4(x), dim=1)

        return x

model=Network()
optimizer=optim.Adam(model.parameters(),lr=0.01)
criterion=nn.NLLLoss()
```

## Train

```python
epochs=5
train_losses,test_losses=[],[]
for e in range(epochs):
    running_loss=0
    for images,labels in train_loader:
        optimizer.zero_grad()
        log_ps=model(images)
        loss=criterion(log_ps,labels)
        loss.backward()
        optimizer.step()
        running_loss+=loss.item()

    else:
        test_loss=0
        accuracy=0

        with torch.no_grad():
            model.eval()
            for images,labels in test_loader:
                log_ps=model(images)
                test_loss+=criterion(log_ps,labels)
                ps=torch.exp(log_ps)
                top_p,top_class=ps.topk(1,dim=1)
                equals=top_class==labels.view(*top_class.shape)
                accuracy+=torch.mean(equals.type(torch.FloatTensor))
        model.train()
        train_losses.append(running_loss/len(train_loader))
        test_losses.append(test_loss/len(test_loader))

        print("Epoch: {}/{}.. ".format(e+1, epochs),
              "Training Loss: {:.3f}.. ".format(running_loss/len(train_loader)),
              "Test Loss: {:.3f}.. ".format(test_loss/len(test_loader)),
              "Test Accuracy: {:.3f}".format(accuracy/len(test_loader)))
```

## Save our model

```python
print("Our model: \n\n", model, '\n')
print("The state dict keys: \n\n", model.state_dict().keys())
```

```python
torch.save(model.state_dict(), 'checkpoint.pth')
```

## Load our model

```python
state_dict = torch.load('checkpoint.pth')
print(state_dict.keys())
```

```python
model.load_state_dict(state_dict)
```

```python
checkpoint = {'input_size': 784,
              'output_size': 10,
              'hidden_layers': [256,128,64],
              'state_dict': model.state_dict()}

torch.save(checkpoint, 'checkpoint.pth')
```

## Load Test Data

```python
test_images = pd.read_csv("../input/digit-recognizer/test.csv")
test_image = test_images.loc[:,test_images.columns != "label"].values
test_dataset = torch.from_numpy(test_image).type(torch.FloatTensor)/255
print(test_dataset.shape)
#test_dataset = torch.utils.data.TensorDataset(test_dataset)
new_test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = 100, shuffle = False)
```

```python
results = []
with torch.no_grad():
    model.eval()
    for images in new_test_loader:
        output = model(images)
        ps = torch.exp(output)
        top_p, top_class = ps.topk(1, dim = 1)
        results += top_class.numpy().tolist()
```

## Check the results

```python
predictions = np.array(results).flatten()
print(predictions[:5])
print(predictions.shape)
```

## Submit for Scoring

```python
submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)),
                         "Label": predictions})
submissions.to_csv("my_submissions.csv", index=False, header=True)
```

# Reference

[Introduction to Pytorch-Udacity](https://github.com/udacity/deep-learning-v2-pytorch/tree/master/intro-to-pytorch)