```python
from sympy import *
```

```python
x = symbols('x')
tol = 1e-12

def check_zero(f):
    if f == 0:
        return ""
    return f" - {f}"

def write_test_dual(f, x0, method, additional_param = "", index=None):
    test = "#[test]\n"
    test += f"fn test_dual_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = Dual64::from({x0}).derive().{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_dual_vec(f, x0, method, additional_param = "", index=None):
    test = "#[test]\n"
    test += f"fn test_dual_vec_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = DualVec64::<2>::new({x0}, StaticVec::new_vec([1.0, 1.0])).{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps[1]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_hyperdual(f, x0, method, additional_param = "", index=None):
    test = "#[test]\n"
    test += f"fn test_hyperdual_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = HyperDual64::from({x0}).derive1().derive2().{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps2[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1eps2[(0,0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_dual2_vec(f, x0, method, additional_param = "", index=None):
    test = "#[test]\n"
    test += f"fn test_dual2_vec_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = Dual2Vec64::<2>::new({x0}, StaticVec::new_vec([1.0, 1.0]), StaticMat::zero()).{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v1[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v1[1]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2[(0, 0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2[(0, 1)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2[(1, 0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2[(1, 1)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_hyperdual_vec(f, x0, method, additional_param = "", index=None):
    test = "#[test]\n"
    test += f"fn test_hyperdual_vec_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = HyperDualVec64::<2, 2>::new({x0}, StaticVec::new_vec([1.0, 1.0]), StaticVec::new_vec([1.0, 1.0]), StaticMat::new([[0.0, 0.0], [0.0, 0.0]])).{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1[1]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps2[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps2[1]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1eps2[(0, 0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1eps2[(0, 1)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1eps2[(1, 0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.eps1eps2[(1, 1)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_dual2(f, x0, method, additional_param="", index=None):
    test = "#[test]\n"
    test += f"fn test_dual2_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = Dual2_64::from({x0}).derive().{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v1[0]{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2[(0,0)]{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_test_dual3(f, x0, method, additional_param="", index=None):
    test = "#[test]\n"
    test += f"fn test_dual3_{method}{'' if index is None else f'_{index}'}() {{\n"
    test += f"    let res = Dual3_64::from({x0}).derive().{method}({additional_param});\n"
    test += f"    assert!((res.re{check_zero(f.evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v1{check_zero(f.diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v2{check_zero(f.diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += f"    assert!((res.v3{check_zero(f.diff().diff().diff().evalf(subs={x: x0}))}).abs() < {tol});\n"
    test += "}\n\n"
    return test

def write_all_tests(number):
    write_test = {'dual': write_test_dual, 'dual_vec': write_test_dual_vec, 'hyperdual': write_test_hyperdual, 'dual2_vec': write_test_dual2_vec, 'hyperdual_vec': write_test_hyperdual_vec, 'dual2': write_test_dual2, 'dual3': write_test_dual3}[number]
    test = ""

    if number == 'dual2_vec':
        test += "use num_traits::Zero;\n"
    test += "use num_dual::*;\n\n"
    test += write_test(1/x, 1.2, "recip")
    test += write_test(exp(x), 1.2, "exp")
    test += write_test(exp(x)-1, 1.2, "exp_m1")
    test += write_test(2**x, 1.2, "exp2")
    test += write_test(ln(x), 1.2, "ln")
    test += write_test(log(x, 4.2), 1.2, "log", 4.2)
    test += write_test(ln(1+x), 1.2, "ln_1p")
    test += write_test(log(x,2), 1.2, "log2")
    test += write_test(log(x,10), 1.2, "log10")
    test += write_test(sqrt(x), 1.2, "sqrt")
    test += write_test(cbrt(x), 1.2, "cbrt")
    test += write_test(x**4.2, 1.2, "powf", 4.2)
    test += write_test(x**0, 0.0, "powf", 0.0, 0)
    test += write_test(x**1, 0.0, "powf", 1.0, 1)
    test += write_test(x**2, 0.0, "powf", 2.0, 2)
    test += write_test(x**3, 0.0, "powf", 3.0, 3)
    test += write_test(x**4, 0.0, "powf", 4.0, 4)
    test += write_test(x**6, 1.2, "powi", 6)
    test += write_test(x**0, 0.0, "powi", 0, 0)
    test += write_test(x**1, 0.0, "powi", 1, 1)
    test += write_test(x**2, 0.0, "powi", 2, 2)
    test += write_test(x**3, 0.0, "powi", 3, 3)
    test += write_test(x**4, 0.0, "powi", 4, 4)
    test += write_test(sin(x), 1.2, "sin")
    test += write_test(cos(x), 1.2, "cos")
    test += write_test(tan(x), 1.2, "tan")
    test += write_test(asin(x), 0.2, "asin")
    test += write_test(acos(x), 0.2, "acos")
    test += write_test(atan(x), 0.2, "atan")
    test += write_test(sinh(x), 1.2, "sinh")
    test += write_test(cosh(x), 1.2, "cosh")
    test += write_test(tanh(x), 1.2, "tanh")
    test += write_test(asinh(x), 1.2, "asinh")
    test += write_test(acosh(x), 1.2, "acosh")
    test += write_test(atanh(x), 0.2, "atanh")
    test += write_test(jn(0,x), 1.2, "sph_j0")
    test += write_test(jn(1,x), 1.2, "sph_j1")
    test += write_test(jn(2,x), 1.2, "sph_j2")

    with open(f'tests/test_{number}.rs', 'w') as f:
        f.write(test)
```

```python
for number in ['dual', 'dual_vec', 'dual2', 'dual2_vec', 'hyperdual', 'hyperdual_vec', 'dual3']:
    write_all_tests(number)
```

```python

```


# <center> ЛАБОРАТОРНАЯ  РАБОТА 2 </center>
# <center> РЕШЕНИЕ  НЕЛИНЕЙНЫХ  УРАВНЕНИЙ </center>

Теоретический материал к данной теме содержится в [1, глава 4].

```python
from sympy.solvers import solve
from sympy import exp, sin, cos, sqrt, log, ln, pi
from sympy import Rational as syR
from sympy import Symbol, diff
from scipy.misc import derivative

import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline
```

## Задание 1
Даны два уравнения  f(x)=0 и g(x)=0. Найти с точностью $10^{-10}$  все корни уравнений, содержащиеся на отрезке [a, b]. Для решения  задачи использовать  метод бисекции.  Найти корни с помощью встроенной функции.
## ПОРЯДОК  РЕШЕНИЯ  ЗАДАЧИ:
1. Найти аналитическое решение уравнения  f(x)=0.
2. Локализовать корни  f(x)=0   графически.
3. Найти корни уравнения f(x)=0  с точностью    с помощью метода бисекции.
4. Используя  встроенную функцию найти корни уравнения f(x)=0 с точностью  .
5. Аналогично п. 1-4 попытаться найти корни уравнения  g(x)=0. Объяснить полученные результаты.

Ваши функции:
$f(x) = {Task1[TASK_VARIANT][f_latex]}$
$g(x) = {Task1[TASK_VARIANT][f_latex]}$

Пределы:
$[a;b] = [ {Task1[TASK_VARIANT][a]};{Task1[TASK_VARIANT][b]} ] $

```python
epselon1 = 10**(-10)
# Пределы:
a1,b1 = {Task1[TASK_VARIANT][a]}, {Task1[TASK_VARIANT][b]}
```

Запишите $f(x)$ и $g(x)$ как python'ские функции

```python
def f1(x):
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

```python
def g1(x):
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

```python
Локализуйте корни уравнения графически, используя пакет matplotlib
```

```python
## ЛОКАЛИЗУЙТЕ КОРНИ ГРАФИЧЕСКИ ##
```

Допишите функцию BisecMethod, которая реализует метод бисекции.
BisecMethod - должен вохвращать точку и количество шагов итераций, необходимое для достижения заданной точности

```python
def BisecMethod():
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

```python
## Тут решение ##
bisec_method_x_1, bisec_method_n_1  = BisecMethod(f1,a1,b1,epselon1)
```

```python
from nose.tools import assert_equal

assert_equal(abs(f1(bisec_method_x_1)) < epselon1, True)

### BEGIN HIDDEN TESTS
#assert_equal(square(5),[25])
### END HIDDEN TESTS
```

```python
## Решение встроенной функцией ##
integrated_x_1 =
```

```python
## Тест ##
from numpy.testing import assert_allclose

assert_allclose(abs(f1(integrated_x_1)), 0, atol=epselon1)
```

```python
## Простор для функции g ##
```

```python
## Простор для функции g ##
```

```python
## Простор для функции g ##
```

Что не так с функцией $g(x)$?

Запишите свой ответ здесь

## Задание 2
Найти указанный в варианте  корень уравнения $f(x)=0$ с точностью $10^{-6}$ ,  двумя  способами.

   а) Использовать метод бисекции. Предварительно определить отрезок локализации  [a, b].

   b) Использовать метод Ньютона. В качестве начального приближения для метода Ньютона взять  середину отрезка локализации из п. а).

   Сравнить число итераций в п. a), b).

Ваша функция $f(x) = {Task2[TASK_VARIANT][f_latex]}$
Корень, который Вам нужен: {Task2[TASK_VARIANT][which]}

```python
epselon2 = 10**(-6)
```

Запишите $f(x)$ как python'ские функцию

```python
def f2(x):
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

Определите отрезок локализации
Сохраните как $a2,b2$

```python
## Определите отрезок локализации ##
```

Решите задачу используя метод бисекции

```python
bisec_method_x_2, bisec_method_n_2  = BisecMethod(f2,a2,b2,epselon2)
```

```python
assert_equal(abs(f2(bisec_method_x_2)) < epselon2, True)
```

Определите метод Ньютона решения нелинейных уравнений
Он должен возвращать два числа - точку x и количество итераций

```python
def NewtoonMethod():
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

```python
newtoon_method_x_2, newtoon_method_n_2  = NewtoonMethod()
## Тут тест ##
assert_equal(abs(f2(newtoon_method_x_2)) < epselon2, True)
```

Сравните число итераций в двух методах

```python
## ТЕСТ ДЛЯ ЧИСЛА ИТЕРАЦИЙ ##
assert_equal(anewtoon_method_n_2 < bisec_method_n_2, True)
```

## Задание 3

Локализовать корни уравнения $f(x)=0$ и найти их с точностью $10^{-5}$ , используя метод простой итерации. К виду $x=\phi(x)$, удобному для итераций, уравнение $ f(x)=0$ привести двумя способами.

a) Преобразовать уравнение к виду  $x=x-\alpha f(x)$,    где $\alpha=\frac{2}{M+m}$, $0 < m \leq f'(x) \leq M$, а $x$ принадлежит отрезку    локализации [a, b].

b) Любым  другим  преобразованием  уравнения. Проверить достаточное условие сходимости метода.

Использовать критерий окончания итерационного процесса вида  , где  в  п. a)  $q=\frac{(M-m)}{(M+m)}$, в п. b) $q = \max\limits_{x \in [a,b]}{\phi'(x)}$

Ваша функция:
$ f(x) = {Task3[TASK_VARIANT][f_latex]} $

```python
epselon3 = 10**(-5)
```

Запишите $f(x)$ как python'ские функцию

```python
def f3(x):
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

Реализуйте метод простой итерации

```python
def SimpleIteration():
    ### BEGIN SOLUTION
    return x
    ### END SOLUTION
```

```python
## Тут все для SimpleIteration
simple_iteration_x_3_1 = SimpleIteration()
```

```python
## Первое решение тест ##
assert_equal(abs(f3(simple_iteration_x_3_1)) < epselon3, True)
```

```python
## Тут все для SimpleIteration
simple_iteration_x_3_2 = SimpleIteration()
```

```python
## Второе решение тест ##
assert_equal(abs(f3(simple_iteration_x_3_2)) < epselon3, True)
```

Проверьте условие сходимости метода.

```python

```

#### ЛИТЕРАТУРА:
1. Амосов А.А., Дубинский Ю.А., Копченова Н.В. Вычислительные методы для инженеров. М.: Высшая школа, 1994.


# Batch Normalization
One way to make deep networks easier to train is to use more sophisticated optimization procedures such as SGD+momentum, RMSProp, or Adam. Another strategy is to change the architecture of the network to make it easier to train.
One idea along these lines is batch normalization which was proposed by [1] in 2015.

The idea is relatively straightforward. Machine learning methods tend to work better when their input data consists of uncorrelated features with zero mean and unit variance. When training a neural network, we can preprocess the data before feeding it to the network to explicitly decorrelate its features; this will ensure that the first layer of the network sees data that follows a nice distribution. However, even if we preprocess the input data, the activations at deeper layers of the network will likely no longer be decorrelated and will no longer have zero mean or unit variance since they are output from earlier layers in the network. Even worse, during the training process the distribution of features at each layer of the network will shift as the weights of each layer are updated.

The authors of [1] hypothesize that the shifting distribution of features inside deep neural networks may make training deep networks more difficult. To overcome this problem, [1] proposes to insert batch normalization layers into the network. At training time, a batch normalization layer uses a minibatch of data to estimate the mean and standard deviation of each feature. These estimated means and standard deviations are then used to center and normalize the features of the minibatch. A running average of these means and standard deviations is kept during training, and at test time these running averages are used to center and normalize features.

It is possible that this normalization strategy could reduce the representational power of the network, since it may sometimes be optimal for certain layers to have features that are not zero-mean or unit variance. To this end, the batch normalization layer includes learnable shift and scale parameters for each feature dimension.

[1] [Sergey Ioffe and Christian Szegedy, "Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shift", ICML 2015.](https://arxiv.org/abs/1502.03167)

```python
# As usual, a bit of setup
import time
import numpy as np
import matplotlib.pyplot as plt
from cs231n.classifiers.fc_net import *
from cs231n.data_utils import get_CIFAR10_data
from cs231n.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array
from cs231n.solver import Solver

%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

def rel_error(x, y):
    """ returns relative error """
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

def print_mean_std(x,axis=0):
    print('  means: ', x.mean(axis=axis))
    print('  stds:  ', x.std(axis=axis))
    print()
```

```python
# Load the (preprocessed) CIFAR10 data.
data = get_CIFAR10_data()
for k, v in data.items():
  print('%s: ' % k, v.shape)
```

    X_train:  (49000, 3, 32, 32)
    y_train:  (49000,)
    X_val:  (1000, 3, 32, 32)
    y_val:  (1000,)
    X_test:  (1000, 3, 32, 32)
    y_test:  (1000,)

## Batch normalization: forward
In the file `cs231n/layers.py`, implement the batch normalization forward pass in the function `batchnorm_forward`. Once you have done so, run the following to test your implementation.

Referencing the paper linked to above in [1] may be helpful!

```python
# Check the training-time forward pass by checking means and variances
# of features both before and after batch normalization

# Simulate the forward pass for a two-layer network
np.random.seed(231)
N, D1, D2, D3 = 200, 50, 60, 3
X = np.random.randn(N, D1)
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)
a = np.maximum(0, X.dot(W1)).dot(W2)

print('Before batch normalization:')
print_mean_std(a,axis=0)

gamma = np.ones((D3,))
beta = np.zeros((D3,))
# Means should be close to zero and stds close to one
print('After batch normalization (gamma=1, beta=0)')
a_norm, _ = batchnorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=0)

gamma = np.asarray([1.0, 2.0, 3.0])
beta = np.asarray([11.0, 12.0, 13.0])
# Now means should be close to beta and stds close to gamma
print('After batch normalization (gamma=', gamma, ', beta=', beta, ')')
a_norm, _ = batchnorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=0)
```

    Before batch normalization:
      means:  [ -2.3814598  -13.18038246   1.91780462]
      stds:   [27.18502186 34.21455511 37.68611762]

    After batch normalization (gamma=1, beta=0)
      means:  [5.32907052e-17 7.04991621e-17 1.85962357e-17]
      stds:   [0.99999999 1.         1.        ]

    After batch normalization (gamma= [1. 2. 3.] , beta= [11. 12. 13.] )
      means:  [11. 12. 13.]
      stds:   [0.99999999 1.99999999 2.99999999]

```python
# Check the test-time forward pass by running the training-time
# forward pass many times to warm up the running averages, and then
# checking the means and variances of activations after a test-time
# forward pass.

np.random.seed(231)
N, D1, D2, D3 = 200, 50, 60, 3
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)

bn_param = {'mode': 'train'}
gamma = np.ones(D3)
beta = np.zeros(D3)

for t in range(500):
  X = np.random.randn(N, D1)
  a = np.maximum(0, X.dot(W1)).dot(W2)
  batchnorm_forward(a, gamma, beta, bn_param)

bn_param['mode'] = 'test'
X = np.random.randn(N, D1)
a = np.maximum(0, X.dot(W1)).dot(W2)
a_norm, _ = batchnorm_forward(a, gamma, beta, bn_param)

# Means should be close to zero and stds close to one, but will be
# noisier than training-time forward passes.
print('After batch normalization (test-time):')
print_mean_std(a_norm,axis=0)
```

    After batch normalization (test-time):
      means:  [-0.06521252 -0.05248973  0.06914059]
      stds:   [0.96434578 1.05556362 0.93836445]

## Batch normalization: backward
Now implement the backward pass for batch normalization in the function `batchnorm_backward`.

To derive the backward pass you should write out the computation graph for batch normalization and backprop through each of the intermediate nodes. Some intermediates may have multiple outgoing branches; make sure to sum gradients across these branches in the backward pass.

Once you have finished, run the following to numerically check your backward pass.

```python
# Gradient check batchnorm backward pass
np.random.seed(231)
N, D = 4, 5
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

bn_param = {'mode': 'train'}
fx = lambda x: batchnorm_forward(x, gamma, beta, bn_param)[0]
fg = lambda a: batchnorm_forward(x, a, beta, bn_param)[0]
fb = lambda b: batchnorm_forward(x, gamma, b, bn_param)[0]

dx_num = eval_numerical_gradient_array(fx, x, dout)
da_num = eval_numerical_gradient_array(fg, gamma.copy(), dout)
db_num = eval_numerical_gradient_array(fb, beta.copy(), dout)

_, cache = batchnorm_forward(x, gamma, beta, bn_param)
dx, dgamma, dbeta = batchnorm_backward(dout, cache)
#You should expect to see relative errors between 1e-13 and 1e-8
print('dx error: ', rel_error(dx_num, dx))
print('dgamma error: ', rel_error(da_num, dgamma))
print('dbeta error: ', rel_error(db_num, dbeta))
```

    dx error:  1.7029235612572515e-09
    dgamma error:  7.420414216247087e-13
    dbeta error:  2.8795057655839487e-12

## Batch normalization: alternative backward
In class we talked about two different implementations for the sigmoid backward pass. One strategy is to write out a computation graph composed of simple operations and backprop through all intermediate values. Another strategy is to work out the derivatives on paper. For example, you can derive a very simple formula for the sigmoid function's backward pass by simplifying gradients on paper.

Surprisingly, it turns out that you can do a similar simplification for the batch normalization backward pass too!

In the forward pass, given a set of inputs $X=\begin{bmatrix}x_1\\x_2\\...\\x_N\end{bmatrix}$,

we first calculate the mean $\mu$ and variance $v$.
With $\mu$ and $v$ calculated, we can calculate the standard deviation $\sigma$  and normalized data $Y$.
The equations and graph illustration below describe the computation ($y_i$ is the i-th element of the vector $Y$).

\begin{align}
& \mu=\frac{1}{N}\sum_{k=1}^N x_k  &  v=\frac{1}{N}\sum_{k=1}^N (x_k-\mu)^2 \\
& \sigma=\sqrt{v+\epsilon}         &  y_i=\frac{x_i-\mu}{\sigma}
\end{align}

The meat of our problem during backpropagation is to compute $\frac{\partial L}{\partial X}$, given the upstream gradient we receive, $\frac{\partial L}{\partial Y}.$ To do this, recall the chain rule in calculus gives us $\frac{\partial L}{\partial X} = \frac{\partial L}{\partial Y} \cdot \frac{\partial Y}{\partial X}$.

The unknown/hart part is $\frac{\partial Y}{\partial X}$. We can find this by first deriving step-by-step our local gradients at
$\frac{\partial v}{\partial X}$, $\frac{\partial \mu}{\partial X}$,
$\frac{\partial \sigma}{\partial v}$,
$\frac{\partial Y}{\partial \sigma}$, and $\frac{\partial Y}{\partial \mu}$,
and then use the chain rule to compose these gradients (which appear in the form of vectors!) appropriately to compute $\frac{\partial Y}{\partial X}$.

If it's challenging to directly reason about the gradients over $X$ and $Y$ which require matrix multiplication, try reasoning about the gradients in terms of individual elements $x_i$ and $y_i$ first: in that case, you will need to come up with the derivations for $\frac{\partial L}{\partial x_i}$, by relying on the Chain Rule to first calculate the intermediate $\frac{\partial \mu}{\partial x_i}, \frac{\partial v}{\partial x_i}, \frac{\partial \sigma}{\partial x_i},$ then assemble these pieces to calculate $\frac{\partial y_i}{\partial x_i}$.

You should make sure each of the intermediary gradient derivations are all as simplified as possible, for ease of implementation.

After doing so, implement the simplified batch normalization backward pass in the function `batchnorm_backward_alt` and compare the two implementations by running the following. Your two implementations should compute nearly identical results, but the alternative implementation should be a bit faster.

```python
np.random.seed(231)
N, D = 100, 500
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

bn_param = {'mode': 'train'}
out, cache = batchnorm_forward(x, gamma, beta, bn_param)

t1 = time.time()
dx1, dgamma1, dbeta1 = batchnorm_backward(dout, cache)
t2 = time.time()
dx2, dgamma2, dbeta2 = batchnorm_backward_alt(dout, cache)
t3 = time.time()

print('dx difference: ', rel_error(dx1, dx2))
print('dgamma difference: ', rel_error(dgamma1, dgamma2))
print('dbeta difference: ', rel_error(dbeta1, dbeta2))
print('speedup: %.2fx' % ((t2 - t1) / (t3 - t2)))
```

    dx difference:  7.682750402447207e-13
    dgamma difference:  0.0
    dbeta difference:  0.0
    speedup: 1.49x

## Fully Connected Nets with Batch Normalization
Now that you have a working implementation for batch normalization, go back to your `FullyConnectedNet` in the file `cs231n/classifiers/fc_net.py`. Modify your implementation to add batch normalization.

Concretely, when the `normalization` flag is set to `"batchnorm"` in the constructor, you should insert a batch normalization layer before each ReLU nonlinearity. The outputs from the last layer of the network should not be normalized. Once you are done, run the following to gradient-check your implementation.

HINT: You might find it useful to define an additional helper layer similar to those in the file `cs231n/layer_utils.py`. If you decide to do so, do it in the file `cs231n/classifiers/fc_net.py`.

```python
np.random.seed(231)
N, D, H1, H2, C = 2, 15, 20, 30, 10
X = np.random.randn(N, D)
y = np.random.randint(C, size=(N,))

# You should expect losses between 1e-4~1e-10 for W,
# losses between 1e-08~1e-10 for b,
# and losses between 1e-08~1e-09 for beta and gammas.
for reg in [0, 3.14]:
  print('Running check with reg = ', reg)
  model = FullyConnectedNet([H1, H2], input_dim=D, num_classes=C,
                            reg=reg, weight_scale=5e-2, dtype=np.float64,
                            normalization='batchnorm')

  loss, grads = model.loss(X, y)
  print('Initial loss: ', loss)

  for name in sorted(grads):
    f = lambda _: model.loss(X, y)[0]
    grad_num = eval_numerical_gradient(f, model.params[name], verbose=False, h=1e-5)
    print('%s relative error: %.2e' % (name, rel_error(grad_num, grads[name])))
  if reg == 0: print()
```

    Running check with reg =  0
    Initial loss:  2.2611955101340957
    W1 relative error: 1.10e-04
    W2 relative error: 2.85e-06
    W3 relative error: 4.05e-10
    b1 relative error: 2.66e-07
    b2 relative error: 2.22e-08
    b3 relative error: 1.01e-10
    beta1 relative error: 7.33e-09
    beta2 relative error: 1.89e-09
    gamma1 relative error: 6.96e-09
    gamma2 relative error: 1.96e-09

    Running check with reg =  3.14
    Initial loss:  6.996533220108303
    W1 relative error: 1.98e-06
    W2 relative error: 2.28e-06
    W3 relative error: 1.11e-08
    b1 relative error: 1.67e-08
    b2 relative error: 7.99e-07
    b3 relative error: 1.73e-10
    beta1 relative error: 6.65e-09
    beta2 relative error: 3.48e-09
    gamma1 relative error: 8.80e-09
    gamma2 relative error: 5.28e-09

# Batchnorm for deep networks
Run the following to train a six-layer network on a subset of 1000 training examples both with and without batch normalization.

```python
np.random.seed(231)
# Try training a very deep net with batchnorm
hidden_dims = [100, 100, 100, 100, 100]

num_train = 1000
small_data = {
  'X_train': data['X_train'][:num_train],
  'y_train': data['y_train'][:num_train],
  'X_val': data['X_val'],
  'y_val': data['y_val'],
}

weight_scale = 2e-2
bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization='batchnorm')
model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)

print('Solver with batch norm:')
bn_solver = Solver(bn_model, small_data,
                num_epochs=10, batch_size=50,
                update_rule='adam',
                optim_config={
                  'learning_rate': 1e-3,
                },
                verbose=True,print_every=20)
bn_solver.train()

print('\nSolver without batch norm:')
solver = Solver(model, small_data,
                num_epochs=10, batch_size=50,
                update_rule='adam',
                optim_config={
                  'learning_rate': 1e-3,
                },
                verbose=True, print_every=20)
solver.train()
```

    Solver with batch norm:
    (Iteration 1 / 200) loss: 2.340975
    (Epoch 0 / 10) train acc: 0.107000; val_acc: 0.115000
    (Epoch 1 / 10) train acc: 0.314000; val_acc: 0.266000
    (Iteration 21 / 200) loss: 2.039365
    (Epoch 2 / 10) train acc: 0.389000; val_acc: 0.277000
    (Iteration 41 / 200) loss: 2.036710
    (Epoch 3 / 10) train acc: 0.496000; val_acc: 0.317000
    (Iteration 61 / 200) loss: 1.769536
    (Epoch 4 / 10) train acc: 0.530000; val_acc: 0.320000
    (Iteration 81 / 200) loss: 1.265761
    (Epoch 5 / 10) train acc: 0.594000; val_acc: 0.319000
    (Iteration 101 / 200) loss: 1.261353
    (Epoch 6 / 10) train acc: 0.655000; val_acc: 0.319000
    (Iteration 121 / 200) loss: 1.102829
    (Epoch 7 / 10) train acc: 0.683000; val_acc: 0.319000
    (Iteration 141 / 200) loss: 1.217487
    (Epoch 8 / 10) train acc: 0.696000; val_acc: 0.308000
    (Iteration 161 / 200) loss: 0.794534
    (Epoch 9 / 10) train acc: 0.791000; val_acc: 0.326000
    (Iteration 181 / 200) loss: 0.785690
    (Epoch 10 / 10) train acc: 0.729000; val_acc: 0.328000

    Solver without batch norm:
    (Iteration 1 / 200) loss: 2.302332
    (Epoch 0 / 10) train acc: 0.129000; val_acc: 0.131000
    (Epoch 1 / 10) train acc: 0.283000; val_acc: 0.250000
    (Iteration 21 / 200) loss: 2.041970
    (Epoch 2 / 10) train acc: 0.316000; val_acc: 0.277000
    (Iteration 41 / 200) loss: 1.900473
    (Epoch 3 / 10) train acc: 0.373000; val_acc: 0.282000
    (Iteration 61 / 200) loss: 1.713156
    (Epoch 4 / 10) train acc: 0.390000; val_acc: 0.310000
    (Iteration 81 / 200) loss: 1.662209
    (Epoch 5 / 10) train acc: 0.434000; val_acc: 0.300000
    (Iteration 101 / 200) loss: 1.696062
    (Epoch 6 / 10) train acc: 0.536000; val_acc: 0.346000
    (Iteration 121 / 200) loss: 1.550785
    (Epoch 7 / 10) train acc: 0.530000; val_acc: 0.310000
    (Iteration 141 / 200) loss: 1.436308
    (Epoch 8 / 10) train acc: 0.622000; val_acc: 0.342000
    (Iteration 161 / 200) loss: 1.000868
    (Epoch 9 / 10) train acc: 0.654000; val_acc: 0.328000
    (Iteration 181 / 200) loss: 0.925455
    (Epoch 10 / 10) train acc: 0.726000; val_acc: 0.335000

Run the following to visualize the results from two networks trained above. You should find that using batch normalization helps the network to converge much faster.

```python
def plot_training_history(title, label, baseline, bn_solvers, plot_fn, bl_marker='.', bn_marker='.', labels=None):
    """utility function for plotting training history"""
    plt.title(title)
    plt.xlabel(label)
    bn_plots = [plot_fn(bn_solver) for bn_solver in bn_solvers]
    bl_plot = plot_fn(baseline)
    num_bn = len(bn_plots)
    for i in range(num_bn):
        label='with_norm'
        if labels is not None:
            label += str(labels[i])
        plt.plot(bn_plots[i], bn_marker, label=label)
    label='baseline'
    if labels is not None:
        label += str(labels[0])
    plt.plot(bl_plot, bl_marker, label=label)
    plt.legend(loc='lower center', ncol=num_bn+1)

plt.subplot(3, 1, 1)
plot_training_history('Training loss','Iteration', solver, [bn_solver], \
                      lambda x: x.loss_history, bl_marker='o', bn_marker='o')
plt.subplot(3, 1, 2)
plot_training_history('Training accuracy','Epoch', solver, [bn_solver], \
                      lambda x: x.train_acc_history, bl_marker='-o', bn_marker='-o')
plt.subplot(3, 1, 3)
plot_training_history('Validation accuracy','Epoch', solver, [bn_solver], \
                      lambda x: x.val_acc_history, bl_marker='-o', bn_marker='-o')

plt.gcf().set_size_inches(15, 15)
plt.show()
```

# Batch normalization and initialization
We will now run a small experiment to study the interaction of batch normalization and weight initialization.

The first cell will train 8-layer networks both with and without batch normalization using different scales for weight initialization. The second layer will plot training accuracy, validation set accuracy, and training loss as a function of the weight initialization scale.

```python
np.random.seed(231)
# Try training a very deep net with batchnorm
hidden_dims = [50, 50, 50, 50, 50, 50, 50]
num_train = 1000
small_data = {
  'X_train': data['X_train'][:num_train],
  'y_train': data['y_train'][:num_train],
  'X_val': data['X_val'],
  'y_val': data['y_val'],
}

bn_solvers_ws = {}
solvers_ws = {}
weight_scales = np.logspace(-4, 0, num=20)
for i, weight_scale in enumerate(weight_scales):
  print('Running weight scale %d / %d' % (i + 1, len(weight_scales)))
  bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization='batchnorm')
  model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)

  bn_solver = Solver(bn_model, small_data,
                  num_epochs=10, batch_size=50,
                  update_rule='adam',
                  optim_config={
                    'learning_rate': 1e-3,
                  },
                  verbose=False, print_every=200)
  bn_solver.train()
  bn_solvers_ws[weight_scale] = bn_solver

  solver = Solver(model, small_data,
                  num_epochs=10, batch_size=50,
                  update_rule='adam',
                  optim_config={
                    'learning_rate': 1e-3,
                  },
                  verbose=False, print_every=200)
  solver.train()
  solvers_ws[weight_scale] = solver
```

    Running weight scale 1 / 20
    Running weight scale 2 / 20
    Running weight scale 3 / 20
    Running weight scale 4 / 20
    Running weight scale 5 / 20
    Running weight scale 6 / 20
    Running weight scale 7 / 20
    Running weight scale 8 / 20
    Running weight scale 9 / 20
    Running weight scale 10 / 20
    Running weight scale 11 / 20
    Running weight scale 12 / 20
    Running weight scale 13 / 20
    Running weight scale 14 / 20
    Running weight scale 15 / 20
    Running weight scale 16 / 20
    Running weight scale 17 / 20
    Running weight scale 18 / 20
    Running weight scale 19 / 20
    Running weight scale 20 / 20

```python
# Plot results of weight scale experiment
best_train_accs, bn_best_train_accs = [], []
best_val_accs, bn_best_val_accs = [], []
final_train_loss, bn_final_train_loss = [], []

for ws in weight_scales:
  best_train_accs.append(max(solvers_ws[ws].train_acc_history))
  bn_best_train_accs.append(max(bn_solvers_ws[ws].train_acc_history))

  best_val_accs.append(max(solvers_ws[ws].val_acc_history))
  bn_best_val_accs.append(max(bn_solvers_ws[ws].val_acc_history))

  final_train_loss.append(np.mean(solvers_ws[ws].loss_history[-100:]))
  bn_final_train_loss.append(np.mean(bn_solvers_ws[ws].loss_history[-100:]))

plt.subplot(3, 1, 1)
plt.title('Best val accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best val accuracy')
plt.semilogx(weight_scales, best_val_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_val_accs, '-o', label='batchnorm')
plt.legend(ncol=2, loc='lower right')

plt.subplot(3, 1, 2)
plt.title('Best train accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best training accuracy')
plt.semilogx(weight_scales, best_train_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_train_accs, '-o', label='batchnorm')
plt.legend()

plt.subplot(3, 1, 3)
plt.title('Final training loss vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Final training loss')
plt.semilogx(weight_scales, final_train_loss, '-o', label='baseline')
plt.semilogx(weight_scales, bn_final_train_loss, '-o', label='batchnorm')
plt.legend()
plt.gca().set_ylim(1.0, 3.5)

plt.gcf().set_size_inches(15, 15)
plt.show()
```

## Inline Question 1:
Describe the results of this experiment. How does the scale of weight initialization affect models with/without batch normalization differently, and why?

## Answer:
[FILL THIS IN]

# Batch normalization and batch size
We will now run a small experiment to study the interaction of batch normalization and batch size.

The first cell will train 6-layer networks both with and without batch normalization using different batch sizes. The second layer will plot training accuracy and validation set accuracy over time.

```python
def run_batchsize_experiments(normalization_mode):
    np.random.seed(231)
    # Try training a very deep net with batchnorm
    hidden_dims = [100, 100, 100, 100, 100]
    num_train = 1000
    small_data = {
      'X_train': data['X_train'][:num_train],
      'y_train': data['y_train'][:num_train],
      'X_val': data['X_val'],
      'y_val': data['y_val'],
    }
    n_epochs=10
    weight_scale = 2e-2
    batch_sizes = [5,10,50]
    lr = 10**(-3.5)
    solver_bsize = batch_sizes[0]

    print('No normalization: batch size = ',solver_bsize)
    model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)
    solver = Solver(model, small_data,
                    num_epochs=n_epochs, batch_size=solver_bsize,
                    update_rule='adam',
                    optim_config={
                      'learning_rate': lr,
                    },
                    verbose=False)
    solver.train()

    bn_solvers = []
    for i in range(len(batch_sizes)):
        b_size=batch_sizes[i]
        print('Normalization: batch size = ',b_size)
        bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=normalization_mode)
        bn_solver = Solver(bn_model, small_data,
                        num_epochs=n_epochs, batch_size=b_size,
                        update_rule='adam',
                        optim_config={
                          'learning_rate': lr,
                        },
                        verbose=False)
        bn_solver.train()
        bn_solvers.append(bn_solver)

    return bn_solvers, solver, batch_sizes

batch_sizes = [5,10,50]
bn_solvers_bsize, solver_bsize, batch_sizes = run_batchsize_experiments('batchnorm')
```

    No normalization: batch size =  5
    Normalization: batch size =  5
    Normalization: batch size =  10
    Normalization: batch size =  50

```python
plt.subplot(2, 1, 1)
plot_training_history('Training accuracy (Batch Normalization)','Epoch', solver_bsize, bn_solvers_bsize, \
                      lambda x: x.train_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)
plt.subplot(2, 1, 2)
plot_training_history('Validation accuracy (Batch Normalization)','Epoch', solver_bsize, bn_solvers_bsize, \
                      lambda x: x.val_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)

plt.gcf().set_size_inches(15, 10)
plt.show()
```

## Inline Question 2:
Describe the results of this experiment. What does this imply about the relationship between batch normalization and batch size? Why is this relationship observed?

## Answer:
[FILL THIS IN]

# Layer Normalization
Batch normalization has proved to be effective in making networks easier to train, but the dependency on batch size makes it less useful in complex networks which have a cap on the input batch size due to hardware limitations.

Several alternatives to batch normalization have been proposed to mitigate this problem; one such technique is Layer Normalization [2]. Instead of normalizing over the batch, we normalize over the features. In other words, when using Layer Normalization, each feature vector corresponding to a single datapoint is normalized based on the sum of all terms within that feature vector.

[2] [Ba, Jimmy Lei, Jamie Ryan Kiros, and Geoffrey E. Hinton. "Layer Normalization." stat 1050 (2016): 21.](https://arxiv.org/pdf/1607.06450.pdf)

## Inline Question 3:
Which of these data preprocessing steps is analogous to batch normalization, and which is analogous to layer normalization?

1. Scaling each image in the dataset, so that the RGB channels for each row of pixels within an image sums up to 1.
2. Scaling each image in the dataset, so that the RGB channels for all pixels within an image sums up to 1.
3. Subtracting the mean image of the dataset from each image in the dataset.
4. Setting all RGB values to either 0 or 1 depending on a given threshold.

## Answer:
[FILL THIS IN]

# Layer Normalization: Implementation

Now you'll implement layer normalization. This step should be relatively straightforward, as conceptually the implementation is almost identical to that of batch normalization. One significant difference though is that for layer normalization, we do not keep track of the moving moments, and the testing phase is identical to the training phase, where the mean and variance are directly calculated per datapoint.

Here's what you need to do:

* In `cs231n/layers.py`, implement the forward pass for layer normalization in the function `layernorm_backward`.

Run the cell below to check your results.
* In `cs231n/layers.py`, implement the backward pass for layer normalization in the function `layernorm_backward`.

Run the second cell below to check your results.
* Modify `cs231n/classifiers/fc_net.py` to add layer normalization to the `FullyConnectedNet`. When the `normalization` flag is set to `"layernorm"` in the constructor, you should insert a layer normalization layer before each ReLU nonlinearity.

Run the third cell below to run the batch size experiment on layer normalization.

```python
# Check the training-time forward pass by checking means and variances
# of features both before and after layer normalization

# Simulate the forward pass for a two-layer network
np.random.seed(231)
N, D1, D2, D3 =4, 50, 60, 3
X = np.random.randn(N, D1)
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)
a = np.maximum(0, X.dot(W1)).dot(W2)

print('Before layer normalization:')
print_mean_std(a,axis=1)

gamma = np.ones(D3)
beta = np.zeros(D3)
# Means should be close to zero and stds close to one
print('After layer normalization (gamma=1, beta=0)')
a_norm, _ = layernorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=1)

gamma = np.asarray([3.0,3.0,3.0])
beta = np.asarray([5.0,5.0,5.0])
# Now means should be close to beta and stds close to gamma
print('After layer normalization (gamma=', gamma, ', beta=', beta, ')')
a_norm, _ = layernorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=1)
```

    Before layer normalization:
      means:  [-59.06673243 -47.60782686 -43.31137368 -26.40991744]
      stds:   [10.07429373 28.39478981 35.28360729  4.01831507]

    After layer normalization (gamma=1, beta=0)
      means:  [ 4.81096644e-16 -7.40148683e-17  2.22044605e-16 -5.92118946e-16]
      stds:   [0.99999995 0.99999999 1.         0.99999969]

    After layer normalization (gamma= [3. 3. 3.] , beta= [5. 5. 5.] )
      means:  [5. 5. 5. 5.]
      stds:   [2.99999985 2.99999998 2.99999999 2.99999907]

```python
# Gradient check batchnorm backward pass
np.random.seed(231)
N, D = 4, 5
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

ln_param = {}
fx = lambda x: layernorm_forward(x, gamma, beta, ln_param)[0]
fg = lambda a: layernorm_forward(x, a, beta, ln_param)[0]
fb = lambda b: layernorm_forward(x, gamma, b, ln_param)[0]

dx_num = eval_numerical_gradient_array(fx, x, dout)
da_num = eval_numerical_gradient_array(fg, gamma.copy(), dout)
db_num = eval_numerical_gradient_array(fb, beta.copy(), dout)

_, cache = layernorm_forward(x, gamma, beta, ln_param)
dx, dgamma, dbeta = layernorm_backward(dout, cache)

#You should expect to see relative errors between 1e-12 and 1e-8
print('dx error: ', rel_error(dx_num, dx))
print('dgamma error: ', rel_error(da_num, dgamma))
print('dbeta error: ', rel_error(db_num, dbeta))
```

    dx error:  1.4336154662306233e-09
    dgamma error:  4.519489546032799e-12
    dbeta error:  2.276445013433725e-12

# Layer Normalization and batch size

We will now run the previous batch size experiment with layer normalization instead of batch normalization. Compared to the previous experiment, you should see a markedly smaller influence of batch size on the training history!

```python
ln_solvers_bsize, solver_bsize, batch_sizes = run_batchsize_experiments('layernorm')

plt.subplot(2, 1, 1)
plot_training_history('Training accuracy (Layer Normalization)','Epoch', solver_bsize, ln_solvers_bsize, \
                      lambda x: x.train_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)
plt.subplot(2, 1, 2)
plot_training_history('Validation accuracy (Layer Normalization)','Epoch', solver_bsize, ln_solvers_bsize, \
                      lambda x: x.val_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)

plt.gcf().set_size_inches(15, 10)
plt.show()
```

## Inline Question 4:
When is layer normalization likely to not work well, and why?

1. Using it in a very deep network
2. Having a very small dimension of features
3. Having a high regularization term

## Answer:
[FILL THIS IN]

```python

```


 INF285/ILI285 Computación Científica

 Tarea N°1, v1.00

## Instrucciones

* La tarea es individual.
* Las consultas sobre las tareas se deben realizar por medio de la plataforma Aula.
* La tarea debe ser realizada en `Jupyter Notebook` (`Python3`).
* Se evaluará la correcta utilización de librerias `NumPy`, `SciPy`, entre otras, así como la correcta implementación de algoritmos de forma vectorizada.
*  **El archivo de entrega debe denominarse ROL-tarea-numero.ipynb**. _De no respetarse este formato existirá un descuento de **50 puntos**_
* La fecha de entrega es el jueves 30 de Abril a las **18:00 hrs**.  Se aceptarán entregas hasta las 19:00 hrs sin descuento en caso de existir algun problema, posteriormente existirá un descuento lineal hasta las 20:00 hrs del mismo día.
* Las tareas que sean entregadas antes del jueves a mediodía recibirán una bonificación de 10 puntos
* Debe citar cualquier código ajeno utilizado (incluso si proviene de los Jupyter Notebooks del curso).

## Introducción

En esta primera tarea de INF/ILI-285, versión 2020-1, estudiaremos la importancia de los primeros temas estudiados en el curso, los cuales son: Representación de Punto Flotante, Pérdida de Significancia, Errores de Cancelación y Búsqueda de Ceros. El desarrollo de cada uno de esos temas se presenta en una serie de preguntas, donde ustedes deben ir decidiendo, pregunta a pregunta, como cada uno de los temas se aplica. En general, los temas no se analizan por separado, sino de manera acoplada. Es muy importante que cada uno de los problemas sea analizado teóricamente primero, para luego poner su conocimiento en acción. Cada problema puede ser desarrollado de diversas formas, sin embargo, es muy importante determinar al final si el camino elegido resuelve la pregunta presentada.

## Problemas

### 1. Simulador (50 ptos)

Dada la familia de polinomios de grado 3 con la forma:

\begin{equation}
f(x)=x^3 - A\,x^2 + A\,x - 1
\end{equation}

Se pide implementar un algoritmo que reciba como parámetros el valor de $A$ (con $|A|>>1$) y la cantidad de bits que tiene en la mantisa un computador ficticio, el manejo del exponente debe considerarse de las misma forma que lo maneja _double precision_.
Esta implementación debe calcular las raíces de $f$ sin perdida de significancia con la cantidad de bits disponibles para la mantisa.
Para obtener las raíces de $f$ usted debe encontrar de forma algebraica sus raíces y luego proponer un algoritmo basado en las _fórmulas_ obtenidas.

Considere que en ese computador ficticio cuenta con las operaciones matemáticas necesarias para obtener las raíces.  Considere el siguiente ejemplo:

```python
# Alg. Base
a = 9819824.624837
b = 148736.523476
c = a+b

# Alg. con Representación de Punto Flotante de 'bits_mant' bits en la mantisa.
am = f_new_rep(9819824.624837,bits_mant) # Aproximar el input en la nueva representación.
bm = f_new_rep(148736.523476,bits_mant) # Aproximar el input en la nueva representación.
cm = f_new_rep(m,exp,am+bm) # Aproximar el output de la suma y cada operación en la nueva representación.
```

```
"""
input
x         : (double) valor a evaluar
bits_mant : (int) cantidad de bits de la mantisa
output
eval      : (double) resultado obtenido
"""
def f_new_rep(x,bits_mant):
  # Algoritmo de representación de punto flotante modificada.
  result={}
  return  result

def f_find_roots(A,bits_mant):
  # Algoritmo para encontrar raíces de f(x).
  return x_roots

```

### Polinomios de Legendre (50 puntos)
Dada la función compuesta $f$
\begin{equation}
f_{n,m}(x) = L_n(C_m(x)),
\end{equation}
donde $L_n$ es conocido como el polinomio de Legendre de grado $n$ definido de la siguiente forma:
\begin{equation}
L_{n}(x)=\frac{1}{2^{n}} \sum_{k=0}^{n}\left(\begin{array}{l}
n \\
k
\end{array}\right)^{2}(x-1)^{n-k}(x+1)^{k},
\end{equation}
y $C_m$ es el polinomio de Chebyshev
\begin{equation}
C_m(x) = \cos(m \cdot \arccos(x)).
\end{equation}

Utilizando el método de Bisección y Punto fijo se pide obtener la raíz de $f$ más cercana a $0.5$ dado un valor de $m$ y $n$

*Hint: Las raíces de Legendre son conocidas*

```
"""
input
n: (int) grado del polinomio de Legendre
m: (int) grado del polinomio de Chebyshev
tol: (double) tolerancia para la detención del algoritmo
output
root: (double) raiz obtenida
"""
def f_Biseccion(n, m, tol):
    ##
    return root

def f_FPI(n, m, tol):
    return root
```

###  Determinantes (20 puntos)

Dada una matriz de dimensiones $ n \times n$ de la forma:
\begin{equation}
A
=
\begin{pmatrix}
a_{1,1} & a_{1,2}  & \dots & a_{1,n} \\
a_{2,1} & a_{2,2}  & \dots & a_{2,n} \\
\vdots &  \vdots   & \ddots & \vdots \\
a_{n,1} & a_{n,2}  & \dots & a_{n,n}
\end{pmatrix}
=
\begin{pmatrix}
\mathbf{r}_1 \\
\mathbf{r}_2 \\
\vdots \\
\mathbf{r}_n \\
\end{pmatrix},
\end{equation}
donde $\mathbf{r}_k$ representa la $k$-ésima fila de la matriz $A$.
Considere la siguiente matriz $C_{i,j}(\alpha)$,
\begin{equation}
C_{i,j}(\alpha)
=
\begin{pmatrix}
\mathbf{r}_1 \\
\vdots \\
\mathbf{r}_i\,(1-\alpha)+\mathbf{r}_j\,\alpha \\
\vdots \\
\mathbf{r}_j\,(1-\alpha)+\mathbf{r}_i\,\alpha \\
\vdots \\
\mathbf{r}_n \\
\end{pmatrix},
\end{equation}
de lo cual sabemos que $C_{i,j}(0)=A$ y que $C_{i,j}(1)$ es la matriz $A$ donde se intercambiaron las filas $i$ y $j$, es decir:
\begin{equation}
C_{i,j}(1)
=
\begin{pmatrix}
\mathbf{r}_1 \\
\vdots \\
\mathbf{r}_j \\
\vdots \\
\mathbf{r}_i \\
\vdots \\
\mathbf{r}_n \\
\end{pmatrix}.
\end{equation}
De las relaciones anteriores podemos concluir que el determinante de la matriz $A$, denominado $D=\det(A)$, es igual al determinante de $C_{i,j}(0)$, es decir $\det(C_{i,j}(0))=\det(A)=D$.
Por el otro lado, el determinante de $C_{i,j}(1)$ es $-D$, dado que es el intercambio de las filas $i$ y $j$ de la matriz $A$.
Por lo cual podemos concluir que $-D\leq \det(C_{i,j}(\alpha))\leq D$.

Utilizando el método de Bisección debe encontrar el valor de $\alpha$ con $p$ decimales de precisión que permitan que, dado los índices de las filas $i$, $j$, y $i\neq j$, el determinante de la matriz sea igual a $d$, donde $d\in[-D,D]$.

Para esto se debe implementar una función que reciba la matriz $A$, las filas $i$ y $j$, y $p$; y retorne $\widehat{\alpha}$ tal que $\det(C_{i,j}(\widehat{\alpha}))=d$.

```
"""
input
A: (array n x n) matriz
i: (int) índice de la fila "i".
j: (int) índice de la fila "j".
p: (int) cantidad de decimales de precision
d: (double) valor requerido del determinante de $C_{i,j}(\alpha)$, i.e. $\det(C_{i,j}(\widehat{\alpha}))=d$.
output
alpha_hat: (double) alpha_hat tal que det(C_{i,j}(alpha_hat))=d.
"""
def find_alpha_hat(A,i,j,p,d):
  # Algoritmo a desarrollar, note que usted debe definir el intervalo a utilizar en el método de la bisección.
  return alpha_hat
```

# Referencias

Si corresponde


## Histograms of Oriented Gradients (HOG)

As we saw with the ORB algorithm, we can use keypoints in images to do keypoint-based matching to detect objects in images. These type of algorithms work great when you want to detect objects that have a lot of consistent internal features that are not affected by the background. For example, these algorithms work well for facial detection because faces have a lot of consistent internal features that don’t get affected by the image background, such as the eyes, nose, and mouth. However, these type of algorithms don’t work so well when attempting to do more general object recognition, say for example, pedestrian detection in images. The reason is that people don’t have consistent internal features, like faces do, because the body shape and style of every person is different (see Fig. 1). This means that every person is going to have a different set of internal features, and so we need something that can more generally describe a person.

Fig. 1. - Pedestrians.

One option is to try to detect pedestrians by their contours instead.  Detecting objects in images by their contours (boundaries) is very challenging because we have to deal with the difficulties brought about by the contrast between the background and the foreground. For example, suppose you wanted to detect a pedestrian in an image that is walking in front of a white building and she is wearing a white coat and black pants (see Fig. 2). We can see in Fig. 2, that since the background of the image is mostly white, the black pants are going to have a very high contrast, but the coat, since it is white as well, is going to have very low contrast. In this case, detecting the edges of pants is going to be easy but detecting the edges of the coat is going to be very difficult. This is where **HOG** comes in. HOG stands for **Histograms of Oriented Gradients** and it was first introduced by Navneet Dalal and Bill Triggs in 2005.

Fig. 2. - High and Low Contrast.

The HOG algorithm works by creating histograms of the distribution of gradient orientations in an image and then normalizing them in a very special way. This special normalization is what makes HOG so effective at detecting the edges of objects even in cases where the contrast is very low. These normalized histograms are put together into a feature vector, known as the HOG descriptor, that can be used to train a machine learning algorithm, such as a Support Vector Machine (SVM), to detect objects in images based on their boundaries (edges). Due to its great success and reliability, HOG has become one of the most widely used algorithms in computer vison for object detection.

In this notebook, you will learn:

* How the HOG algorithm works
* How to use OpenCV to create a HOG descriptor
* How to visualize the HOG descriptor.

# The HOG Algorithm

As its name suggests, the HOG algorithm, is based on creating histograms from the orientation of image gradients. The HOG algorithm is implemented in a series of steps:

1. Given the image of particular object, set a detection window (region of interest) that covers the entire object in the image (see Fig. 3).

2. Calculate the magnitude and direction of the gradient for each individual pixel in the detection window.

3. Divide the detection window into connected *cells* of pixels, with all cells being of the same size (see Fig. 3). The size of the cells is a free parameter and it is usually chosen so as to match the scale of the features that want to be detected. For example, in a 64 x 128 pixel detection window, square cells 6 to 8 pixels wide are suitable for detecting human limbs.

4. Create a Histogram for each cell, by first grouping the gradient directions of all pixels in each cell into a particular number of orientation (angular) bins; and then adding up the gradient magnitudes of the gradients in each angular bin (see Fig. 3). The number of bins in the histogram is a free parameter and it is usually set to 9 angular bins.

5. Group adjacent cells into *blocks* (see Fig. 3). The number of cells in each block is a free parameter and all blocks must be of the same size. The distance between each block (known as the stride) is a free parameter but it is usually set to half the block size, in which case you will get overlapping blocks (*see video below*).  The HOG algorithm has been shown empirically to work better with overlapping blocks.

6. Use the cells contained within each block to normalize the cell histograms in that block (see Fig. 3). If you have overlapping blocks this means that most cells will be normalized with respect to different blocks (*see video below*). Therefore, the same cell may have several different normalizations.

7. Collect all the normalized histograms from all the blocks into a single feature vector called the HOG descriptor.

8. Use the resulting HOG descriptors from many images of the same type of object to train a machine learning algorithm, such as an SVM, to detect those type of objects in images. For example, you could use the HOG descriptors from many images of pedestrians to train an SVM to detect pedestrians in images. The training is done with both positive a negative examples of the object you want detect in the image.

9. Once the SVM has been trained, a sliding window approach is used to try to detect and locate objects in images.  Detecting an object in the image entails finding the part of the image that looks similar to the HOG pattern learned by the SVM.

Fig. 3. - HOG Diagram.

Vid. 1. - HOG Animation.

# Why The HOG Algorithm Works

As we learned above, HOG creates histograms by adding the magnitude of the gradients in particular orientations in localized portions of the image called *cells*.  By doing this we guarantee that stronger gradients will contribute more to the magnitude of their respective angular bin, while the effects of weak and randomly oriented gradients resulting from noise are minimized. In this manner the histograms tell us the dominant gradient orientation of each cell.

### Dealing with contrast

Now, the magnitude of the dominant orientation can vary widely due to variations in local illumination and the contrast between the background and the foreground.

To account for the background-foreground contrast differences, the HOG algorithm tries to detect edges locally.  In order to do this, it defines groups of cells, called **blocks**, and normalizes the histograms using this local group of cells. By normalizing locally, the HOG algorithm can detect the edges in each block very reliably; this is called **block normalization**.

In addition to using block normalization, the HOG algorithm also uses overlapping blocks to increase its performance. By using overlapping blocks, each cell contributes several independent components to the final HOG descriptor, where each component corresponds to a cell being normalized with respect to a different block. This may seem redundant but, it has been shown empirically that by normalizing each cell several times with respect to different local blocks, the performance of the HOG algorithm increases dramatically.

### Loading Images and Importing Resources

The first step in building our HOG descriptor is to load the required packages into Python and to load our image.

We start by using OpenCV to load an image of a triangle tile. Since, the `cv2.imread()` function loads images as BGR we will convert our image to RGB so we can display it with the correct colors. As usual we will convert our BGR image to Gray Scale for analysis.

```python
import cv2
import numpy as np
import matplotlib.pyplot as plt

# Set the default figure size
plt.rcParams['figure.figsize'] = [17.0, 7.0]

# Load the image
image = cv2.imread('./images/triangle_tile.jpeg')

# Convert the original image to RGB
original_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

# Convert the original image to gray scale
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# Print the shape of the original and gray scale images
print('The original image has shape: ', original_image.shape)
print('The gray scale image has shape: ', gray_image.shape)

# Display the images
plt.subplot(121)
plt.imshow(original_image)
plt.title('Original Image')
plt.subplot(122)
plt.imshow(gray_image, cmap='gray')
plt.title('Gray Scale Image')
plt.show()
```

    The original image has shape:  (250, 250, 3)
    The gray scale image has shape:  (250, 250)

# Creating The HOG Descriptor

We will be using OpenCV’s `HOGDescriptor` class to create the HOG descriptor. The parameters of the HOG descriptor are setup using the `HOGDescriptor()` function. The parameters of the `HOGDescriptor()` function and their default values are given below:

`cv2.HOGDescriptor(win_size = (64, 128),
                  block_size = (16, 16),
                  block_stride = (8, 8),
                  cell_size = (8, 8),
                  nbins = 9,
                  win_sigma = DEFAULT_WIN_SIGMA,
                  threshold_L2hys = 0.2,
                  gamma_correction = true,
                  nlevels = DEFAULT_NLEVELS)`

Parameters:

* **win_size** – *Size*
Size of detection window in pixels (*width, height*). Defines the region of interest. Must be an integer multiple of cell size.

* **block_size** – *Size*
Block size in pixels (*width, height*). Defines how many cells are in each block. Must be an integer multiple of cell size and it must be smaller than the detection window. The smaller the block the finer detail you will get.

* **block_stride** – *Size*
Block stride in pixels (*horizontal, vertical*). It must be an integer multiple of cell size. The `block_stride` defines the distance between adjecent blocks, for example, 8 pixels horizontally and 8 pixels vertically. Longer `block_strides` makes the algorithm run faster (because less blocks are evaluated) but the algorithm may not perform as well.

* **cell_size** – *Size*
Cell size in pixels (*width, height*). Determines the size fo your cell. The smaller the cell the finer detail you will get.

* **nbins** – *int*
Number of bins for the histograms. Determines the number of angular bins used to make the histograms. With more bins you capture more gradient directions. HOG uses unsigned gradients, so the angular bins will have values between 0 and 180 degrees.

* **win_sigma** – *double*
Gaussian smoothing window parameter. The performance of the HOG algorithm can be improved by smoothing the pixels near the edges of the blocks by applying a Gaussian spatial window to each pixel before computing the histograms.

* **threshold_L2hys** – *double*
L2-Hys (Lowe-style clipped L2 norm) normalization method shrinkage. The L2-Hys method is used to normalize the blocks and it consists of an L2-norm followed by clipping and a renormalization. The clipping limits the maximum value of the descriptor vector for each block to have the value of the given threshold (0.2 by default). After the clipping the descriptor vector is renormalized as described in *IJCV*, 60(2):91-110, 2004.

* **gamma_correction** – *bool*
Flag to specify whether the gamma correction preprocessing is required or not. Performing gamma correction slightly increases the performance of the HOG algorithm.

* **nlevels** – *int*
Maximum number of detection window increases.

As we can see, the  `cv2.HOGDescriptor()`function supports a wide range of parameters. The first few arguments (`block_size, block_stride, cell_size`, and `nbins`) are probably the ones you are most likely to change. The other parameters can be safely left at their default values and you will get good results.

In the code below, we will use the `cv2.HOGDescriptor()`function to set the cell size, block size, block stride, and the number of bins for the histograms of the HOG descriptor. We will then use `.compute(image)`method to compute the HOG descriptor (feature vector) for the given `image`.

```python
# Specify the parameters for our HOG descriptor

# Cell Size in pixels (width, height). Must be smaller than the size of the detection window
# and must be chosen so that the resulting Block Size is smaller than the detection window.
cell_size = (6, 6)

# Number of cells per block in each direction (x, y). Must be chosen so that the resulting
# Block Size is smaller than the detection window
num_cells_per_block = (2, 2)

# Block Size in pixels (width, height). Must be an integer multiple of Cell Size.
# The Block Size must be smaller than the detection window
block_size = (num_cells_per_block[0] * cell_size[0],
              num_cells_per_block[1] * cell_size[1])

# Calculate the number of cells that fit in our image in the x and y directions
x_cells = gray_image.shape[1] // cell_size[0]
y_cells = gray_image.shape[0] // cell_size[1]

# Horizontal distance between blocks in units of Cell Size. Must be an integer and it must
# be set such that (x_cells - num_cells_per_block[0]) / h_stride = integer.
h_stride = 1

# Vertical distance between blocks in units of Cell Size. Must be an integer and it must
# be set such that (y_cells - num_cells_per_block[1]) / v_stride = integer.
v_stride = 1

# Block Stride in pixels (horizantal, vertical). Must be an integer multiple of Cell Size
block_stride = (cell_size[0] * h_stride, cell_size[1] * v_stride)

# Number of gradient orientation bins
num_bins = 9

# Specify the size of the detection window (Region of Interest) in pixels (width, height).
# It must be an integer multiple of Cell Size and it must cover the entire image. Because
# the detection window must be an integer multiple of cell size, depending on the size of
# your cells, the resulting detection window might be slightly smaller than the image.
# This is perfectly ok.
win_size = (x_cells * cell_size[0] , y_cells * cell_size[1])

# Print the shape of the gray scale image for reference
print('\nThe gray scale image has shape: ', gray_image.shape)
print()

# Print the parameters of our HOG descriptor
print('HOG Descriptor Parameters:\n')
print('Window Size:', win_size)
print('Cell Size:', cell_size)
print('Block Size:', block_size)
print('Block Stride:', block_stride)
print('Number of Bins:', num_bins)
print()

# Set the parameters of the HOG descriptor using the variables defined above
hog = cv2.HOGDescriptor(win_size, block_size, block_stride, cell_size, num_bins)

# Compute the HOG Descriptor for the gray scale image
hog_descriptor = hog.compute(gray_image)
```

    The gray scale image has shape:  (250, 250)

    HOG Descriptor Parameters:

    Window Size: (246, 246)
    Cell Size: (6, 6)
    Block Size: (12, 12)
    Block Stride: (6, 6)
    Number of Bins: 9

# Number of Elements In The HOG Descriptor

The resulting HOG Descriptor (feature vector), contains the normalized histograms from all cells from all blocks in the detection window concatenated in one long vector. Therefore, the size of the HOG feature vector will be given by the total number of blocks in the detection window, multiplied by the number of cells per block, times the number of orientation bins:

\begin{equation}
\mbox{total_elements} = (\mbox{total_number_of_blocks})\mbox{ } \times \mbox{ } (\mbox{number_cells_per_block})\mbox{ } \times \mbox{ } (\mbox{number_of_bins})
\end{equation}

If we don’t have overlapping blocks (*i.e.* the `block_stride`equals the `block_size`), the total number of blocks can be easily calculated by dividing the size of the detection window by the block size. However, in the general case we have to take into account the fact that we have overlapping blocks. To find the total number of blocks in the general case (*i.e.* for any `block_stride` and `block_size`), we can use the formula given below:

\begin{equation}
\mbox{Total}_i = \left( \frac{\mbox{block_size}_i}{\mbox{block_stride}_i} \right)\left( \frac{\mbox{window_size}_i}{\mbox{block_size}_i} \right) - \left [\left( \frac{\mbox{block_size}_i}{\mbox{block_stride}_i} \right) - 1 \right]; \mbox{  for  } i = x,y
\end{equation}

Where Total$_x$, is the total number of blocks along the width of the detection window, and Total$_y$, is the total number of blocks along the height of the detection window. This formula for Total$_x$ and Total$_y$, takes into account the extra blocks that result from overlapping. After calculating Total$_x$ and Total$_y$, we can get the total number of blocks in the detection window by multiplying Total$_x$  $\times$ Total$_y$. The above formula can be simplified considerably because the `block_size`, `block_stride`, and `window_size`are all defined in terms of the `cell_size`. By making all the appropriate substitutions and cancelations the above formula reduces to:

\begin{equation}
\mbox{Total}_i = \left(\frac{\mbox{cells}_i - \mbox{num_cells_per_block}_i}{N_i}\right) + 1\mbox{  }; \mbox{  for  } i = x,y
\end{equation}

Where cells$_x$ is the total number of cells along the width of the detection window, and cells$_y$, is the total number of cells along the height of the detection window. And $N_x$ is the horizontal block stride in units of `cell_size` and $N_y$ is the vertical block stride in units of `cell_size`.

Let's calculate what the number of elements for the HOG feature vector should be and check that it matches the shape of the HOG Descriptor calculated above.

```python
# Calculate the total number of blocks along the width of the detection window
tot_bx = np.uint32(((x_cells - num_cells_per_block[0]) / h_stride) + 1)

# Calculate the total number of blocks along the height of the detection window
tot_by = np.uint32(((y_cells - num_cells_per_block[1]) / v_stride) + 1)

# Calculate the total number of elements in the feature vector
tot_els = (tot_bx) * (tot_by) * num_cells_per_block[0] * num_cells_per_block[1] * num_bins

# Print the total number of elements the HOG feature vector should have
print('\nThe total number of elements in the HOG Feature Vector should be: ',
      tot_bx, 'x',
      tot_by, 'x',
      num_cells_per_block[0], 'x',
      num_cells_per_block[1], 'x',
      num_bins, '=',
      tot_els)

# Print the shape of the HOG Descriptor to see that it matches the above
print('\nThe HOG Descriptor has shape:', hog_descriptor.shape)
```

    The total number of elements in the HOG Feature Vector should be:  40 x 40 x 2 x 2 x 9 = 57600

    The HOG Descriptor has shape: (57600, 1)

# Visualizing The HOG Descriptor

We can visualize the HOG Descriptor by plotting the histogram associated with each cell as a collection of vectors. To do this, we will plot each bin in the histogram as a single vector whose magnitude is given by the height of the bin and its orientation is given by the angular bin that its associated with. Since any given cell might have multiple histograms associated with it, due to the overlapping blocks, we will choose to average all the histograms for each cell to produce a single histogram for each cell.

OpenCV has no easy way to visualize the HOG Descriptor, so we have to do some manipulation first in order to visualize it. We will start by reshaping the HOG Descriptor in order to make our calculations easier.  We will then compute the average histogram of each cell and finally we will convert the histogram bins into vectors. Once we have the vectors, we plot the corresponding vectors for each cell in an image.

The code below produces an interactive plot so that you can interact with the figure. The figure contains:
* the grayscale image,
* the HOG Descriptor (feature vector),
* a zoomed-in portion of the HOG Descriptor, and
* the histogram of the selected cell.

**You can click anywhere on the gray scale image or the HOG Descriptor image to select a particular cell**. Once you click on either image a *magenta* rectangle will appear showing the cell you selected. The Zoom Window will show you a zoomed in version of the HOG descriptor around the selected cell; and the histogram plot will show you the corresponding histogram for the selected cell. The interactive window also has buttons at the bottom that allow for other functionality, such as panning, and giving you the option to save the figure if desired. The home button returns the figure to its default value.

**NOTE**: If you are running this notebook in the Udacity workspace, there is around a 2 second lag in the interactive plot. This means that if you click in the image to zoom in, it will take about 2 seconds for the plot to refresh.

```python
%matplotlib notebook

import copy
import matplotlib.patches as patches

# Set the default figure size
plt.rcParams['figure.figsize'] = [9.8, 9]

# Reshape the feature vector to [blocks_y, blocks_x, num_cells_per_block_x, num_cells_per_block_y, num_bins].
# The blocks_x and blocks_y will be transposed so that the first index (blocks_y) referes to the row number
# and the second index to the column number. This will be useful later when we plot the feature vector, so
# that the feature vector indexing matches the image indexing.
hog_descriptor_reshaped = hog_descriptor.reshape(tot_bx,
                                                 tot_by,
                                                 num_cells_per_block[0],
                                                 num_cells_per_block[1],
                                                 num_bins).transpose((1, 0, 2, 3, 4))

# Print the shape of the feature vector for reference
print('The feature vector has shape:', hog_descriptor.shape)

# Print the reshaped feature vector
print('The reshaped feature vector has shape:', hog_descriptor_reshaped.shape)

# Create an array that will hold the average gradients for each cell
ave_grad = np.zeros((y_cells, x_cells, num_bins))

# Print the shape of the ave_grad array for reference
print('The average gradient array has shape: ', ave_grad.shape)

# Create an array that will count the number of histograms per cell
hist_counter = np.zeros((y_cells, x_cells, 1))

# Add up all the histograms for each cell and count the number of histograms per cell
for i in range (num_cells_per_block[0]):
    for j in range(num_cells_per_block[1]):
        ave_grad[i:tot_by + i,
                 j:tot_bx + j] += hog_descriptor_reshaped[:, :, i, j, :]

        hist_counter[i:tot_by + i,
                     j:tot_bx + j] += 1

# Calculate the average gradient for each cell
ave_grad /= hist_counter

# Calculate the total number of vectors we have in all the cells.
len_vecs = ave_grad.shape[0] * ave_grad.shape[1] * ave_grad.shape[2]

# Create an array that has num_bins equally spaced between 0 and 180 degress in radians.
deg = np.linspace(0, np.pi, num_bins, endpoint = False)

# Each cell will have a histogram with num_bins. For each cell, plot each bin as a vector (with its magnitude
# equal to the height of the bin in the histogram, and its angle corresponding to the bin in the histogram).
# To do this, create rank 1 arrays that will hold the (x,y)-coordinate of all the vectors in all the cells in the
# image. Also, create the rank 1 arrays that will hold all the (U,V)-components of all the vectors in all the
# cells in the image. Create the arrays that will hold all the vector positons and components.
U = np.zeros((len_vecs))
V = np.zeros((len_vecs))
X = np.zeros((len_vecs))
Y = np.zeros((len_vecs))

# Set the counter to zero
counter = 0

# Use the cosine and sine functions to calculate the vector components (U,V) from their maginitudes. Remember the
# cosine and sine functions take angles in radians. Calculate the vector positions and magnitudes from the
# average gradient array
for i in range(ave_grad.shape[0]):
    for j in range(ave_grad.shape[1]):
        for k in range(ave_grad.shape[2]):
            U[counter] = ave_grad[i,j,k] * np.cos(deg[k])
            V[counter] = ave_grad[i,j,k] * np.sin(deg[k])

            X[counter] = (cell_size[0] / 2) + (cell_size[0] * i)
            Y[counter] = (cell_size[1] / 2) + (cell_size[1] * j)

            counter = counter + 1

# Create the bins in degress to plot our histogram.
angle_axis = np.linspace(0, 180, num_bins, endpoint = False)
angle_axis += ((angle_axis[1] - angle_axis[0]) / 2)

# Create a figure with 4 subplots arranged in 2 x 2
fig, ((a,b),(c,d)) = plt.subplots(2,2)

# Set the title of each subplot
a.set(title = 'Gray Scale Image\n(Click to Zoom)')
b.set(title = 'HOG Descriptor\n(Click to Zoom)')
c.set(title = 'Zoom Window', xlim = (0, 18), ylim = (0, 18), autoscale_on = False)
d.set(title = 'Histogram of Gradients')

# Plot the gray scale image
a.imshow(gray_image, cmap = 'gray')
a.set_aspect(aspect = 1)

# Plot the feature vector (HOG Descriptor)
b.quiver(Y, X, U, V, color = 'white', headwidth = 0, headlength = 0, scale_units = 'inches', scale = 5)
b.invert_yaxis()
b.set_aspect(aspect = 1)
b.set_facecolor('black')

# Define function for interactive zoom
def onpress(event):

    #Unless the left mouse button is pressed do nothing
    if event.button != 1:
        return

    # Only accept clicks for subplots a and b
    if event.inaxes in [a, b]:

        # Get mouse click coordinates
        x, y = event.xdata, event.ydata

        # Select the cell closest to the mouse click coordinates
        cell_num_x = np.uint32(x / cell_size[0])
        cell_num_y = np.uint32(y / cell_size[1])

        # Set the edge coordinates of the rectangle patch
        edgex = x - (x % cell_size[0])
        edgey = y - (y % cell_size[1])

        # Create a rectangle patch that matches the the cell selected above
        rect = patches.Rectangle((edgex, edgey),
                                  cell_size[0], cell_size[1],
                                  linewidth = 1,
                                  edgecolor = 'magenta',
                                  facecolor='none')

        # A single patch can only be used in a single plot. Create copies
        # of the patch to use in the other subplots
        rect2 = copy.copy(rect)
        rect3 = copy.copy(rect)

        # Update all subplots
        a.clear()
        a.set(title = 'Gray Scale Image\n(Click to Zoom)')
        a.imshow(gray_image, cmap = 'gray')
        a.set_aspect(aspect = 1)
        a.add_patch(rect)

        b.clear()
        b.set(title = 'HOG Descriptor\n(Click to Zoom)')
        b.quiver(Y, X, U, V, color = 'white', headwidth = 0, headlength = 0, scale_units = 'inches', scale = 5)
        b.invert_yaxis()
        b.set_aspect(aspect = 1)
        b.set_facecolor('black')
        b.add_patch(rect2)

        c.clear()
        c.set(title = 'Zoom Window')
        c.quiver(Y, X, U, V, color = 'white', headwidth = 0, headlength = 0, scale_units = 'inches', scale = 1)
        c.set_xlim(edgex - cell_size[0], edgex + (2 * cell_size[0]))
        c.set_ylim(edgey - cell_size[1], edgey + (2 * cell_size[1]))
        c.invert_yaxis()
        c.set_aspect(aspect = 1)
        c.set_facecolor('black')
        c.add_patch(rect3)

        d.clear()
        d.set(title = 'Histogram of Gradients')
        d.grid()
        d.set_xlim(0, 180)
        d.set_xticks(angle_axis)
        d.set_xlabel('Angle')
        d.bar(angle_axis,
              ave_grad[cell_num_y, cell_num_x, :],
              180 // num_bins,
              align = 'center',
              alpha = 0.5,
              linewidth = 1.2,
              edgecolor = 'k')

        fig.canvas.draw()

# Create a connection between the figure and the mouse click
fig.canvas.mpl_connect('button_press_event', onpress)
plt.show()
```

    The feature vector has shape: (57600, 1)
    The reshaped feature vector has shape: (40, 40, 2, 2, 9)
    The average gradient array has shape:  (41, 41, 9)

    <IPython.core.display.Javascript object>

```python

```

# Understanding The Histograms

Let's take a look at a couple of snapshots of the above figure to see if the histograms for the selected cell make sense. Let's start looking at a cell that is inside a triangle and not near an edge:

Fig. 4. - Histograms Inside a Triangle.

In this case, since the triangle is nearly all of the same color there shouldn't be any dominant gradient in the selected cell. As we can clearly see in the Zoom Window and the histogram, this is indeed the case. We have many gradients but none of them clearly dominates over the other.

Now let’s take a look at a cell that is near a horizontal edge:

Fig. 5. - Histograms Near a Horizontal Edge.

Remember that edges are areas of an image where the intensity changes abruptly. In these cases, we will have a high intensity gradient in some particular direction. This is exactly what we see in the corresponding histogram and Zoom Window for the selected cell.  In the Zoom Window, we can see that the dominant gradient is pointing up, almost at 90 degrees, since that’s the direction in which there is a sharp change in intensity. Therefore, we should expect to see the 90-degree bin in the histogram to dominate strongly over the others. This is in fact what we see.

Now let’s take a look at a cell that is near a vertical edge:

Fig. 6. - Histograms Near a Vertical Edge.

In this case we expect the dominant gradient in the cell to be horizontal, close to 180 degrees, since that’s the direction in which there is a sharp change in intensity. Therefore, we should expect to see the 170-degree bin in the histogram to dominate strongly over the others. This is what we see in the histogram but we also see that there is another dominant gradient in the cell, namely the one in the 10-degree bin. The reason for this, is because the HOG algorithm is using unsigned gradients, which means 0 degrees and 180 degrees are considered the same. Therefore, when the histograms are being created, angles between 160 and 180 degrees, contribute proportionally to both the 10-degree bin and the 170-degree bin. This results in there being two dominant gradients in the cell near the vertical edge instead of just one.

To conclude let’s take a look at a cell that is near a diagonal edge.

Fig. 7. - Histograms Near a Diagonal Edge.

To understand what we are seeing, let’s first remember that gradients have an *x*-component, and a *y*-component, just like vectors. Therefore, the resulting orientation of a gradient is going to be given by the vector sum of its components. For this reason, on vertical edges the gradients are horizontal, because they only have an x-component, as we saw in Figure 4. While on horizontal edges the gradients are vertical, because they only have a y-component, as we saw in Figure 3. Consequently, on diagonal edges, the gradients are also going to be diagonal because both the *x* and *y* components are non-zero.  Since the diagonal edges in the image are close to 45 degrees, we should expect to see a dominant gradient orientation in the 50-degree bin. This is in fact what we see in the histogram but, just like in Figure 4., we see there are two dominant gradients instead of just one. The reason for this is that when the histograms are being created, angles that are near the boundaries of bins, contribute proportionally to the adjacent bins.  For example, a gradient with an angle of 40 degrees, is right in the middle of the 30-degree and 50-degree bin. Therefore, the magnitude of the gradient is split evenly into the 30-degree and 50-degree bin. This results in there being two dominant gradients in the cell near the diagonal edge instead of just one.

Now that you know how HOG is implemented, in the workspace you will find a notebook named *Examples*. In there, you will be able set your own paramters for the HOG descriptor for various images. Have fun!

```python

```