Find similar light curves using kernel distances

```python
#http://elephant.readthedocs.io/en/latest/_modules/elephant/statistics.html#fftkernel
#https://pdfs.semanticscholar.org/d516/a1cf9110963bf50898d3918bae30cbce61a4.pdf
import numpy as np
%matplotlib inline
import matplotlib.pylab as plt
from astropy.io import fits
from tqdm import tqdm_notebook
import pickle

def get_light_curve(file_name, no_phase=True):
    with fits.open(file_name) as hdul:
        events = hdul[1].data
    times = np.sort(events['time'])
    if no_phase:
        times = times - times[0]
    return times

def kde_spike(times, sigma=1, resolution=1):
    times_grid = np.arange(times[0], times[-1], step=resolution)
    smooth_times = np.zeros(shape=times_grid.shape)
    for center in times:
        min_idx = np.argmin((times_grid - center + 5*sigma)**2)
        max_idx = np.argmin((times_grid - center - 5*sigma)**2)
        smooth_times[min_idx:max_idx] += np.exp(-0.5*(times_grid[min_idx:max_idx] - center)**2/sigma**2)
    #smooth_times = smooth_times*np.sqrt(2.0*np.pi)*sigma*len(times)
    return times_grid, smooth_times

times = get_light_curve('/home/phuijse/Data/XRAY/cut_evt2.fits')
tgrid, smoothed_lc = kde_spike(times, sigma=50, resolution=1)
fig = plt.figure(figsize=(13, 3))
ax = fig.add_subplot(2, 1, 1)
ax.vlines(times, ymin=0.0, ymax=1.0, alpha=0.1)
ax.set_ylabel('Spike train')
ax = fig.add_subplot(2, 1, 2)
ax.plot(tgrid, smoothed_lc)
plt.tight_layout()
ax.set_ylabel('Smoothed')
ax.set_xlabel('Time [days]');
```

```python
from os import listdir
from os.path import join

# Light curves
lc_path = "/home/phuijse/Data/XRAY/CSCVAR/"
file_list = [file for file in sorted(listdir(lc_path)) if file[-13:] == '_regevt3.fits']
```

## Schoenberg kernel

A spike train is a sequence of ordered spike times $\mathcal{S}^{(i)} = \{t_m \in [0, T]: m \in 0, \ldots, M\}$ which can be viewed as a realization of a stochastic point process defined by its conditional intesity function $ \lambda(t|H_t^{(i)})$ with $t\in [0,T]$ and $H_t^{(i)}$ the history of the process $i$ up to $t$.

The Schoeberg kernel is a universal binless kernel that compares the conditional intensity functions
\begin{equation}
\kappa(\lambda_i, \lambda_j) = \exp \left ( -\alpha \int [\lambda(t|H_t^{(i)}) - \lambda(t|H_t^{(j)})]^2 \,dt \right) ,
\end{equation}

with parameter $\alpha > 0$. In this case we approximate the conditional intensity function as
\begin{equation}
\widehat \lambda(t) = \sum_{m=1}^M g(t-t_m),
\end{equation}
*i.e.* the spike times convolved with a smoothing kernel (e.g. Laplacian, Gaussian).

If the smoothing function takes this simple form $g(t)= \frac{1}{\mathcal{T}}  (U(t) - U(t-\mathcal{T}))$, then the integral reduces to $\sum_m (t_m^{(i)} - t_m^{(j)})^2$

1. https://www.frontiersin.org/articles/10.3389/fnins.2018.00194/full
1. https://arxiv.org/pdf/1302.5964.pdf
1. http://www.sci.utah.edu/~arpaiva/pubs/2010b.pdf

```python
tau = 10.0
overlap = 10
T = 5e+3 # times_template[-1]

times_template = get_light_curve('/home/phuijse/Data/XRAY/cut_evt2.fits')
K_template = np.sum(np.exp(-np.absolute(times_template[times_template < T, np.newaxis] \
                                        - times_template[times_template < T, np.newaxis].T)/tau))/(2*tau)
```

```python
distances = list()
lc_index = list()
for k in tqdm_notebook(range(len(file_list)), 'LC'):
    times = get_light_curve(join(lc_path, file_list[k]))
    time_span = times[-1]  # Assume template is shorter than lc
    N_windows = int(overlap*(time_span - T)/T)
    if N_windows < 1:
        continue
    lc_index.append(k)
    K_cross = np.zeros(shape=(N_windows,))
    K_sig = np.zeros(shape=(N_windows,))
    K_time = np.zeros(shape=(N_windows,))
    K_points = np.zeros(shape=(N_windows,))
    for i in range(N_windows):
        K_time[i] = i*T/overlap
        init_point = np.argmin((times - K_time[i])**2)
        end_point = np.argmin((times - K_time[i] - T)**2)
        window_t = times[init_point:end_point] - times[init_point]
        K_points[i] = len(window_t)
        K_sig[i] = np.sum(np.exp(-np.absolute(window_t[:, np.newaxis] - window_t[:, np.newaxis].T)/tau))/(2*tau)
        K_cross[i] = np.sum(np.exp(-np.absolute(times_template[times_template < T, np.newaxis] \
                                                 - window_t[:, np.newaxis].T)/tau))/(2*tau)
    distances.append(np.stack((K_time, K_points, K_sig, K_cross)))

pickle.dump([lc_index, file_list, K_template, distances], open("xray_distances.p", "wb"), protocol=2)
```

```python
lc_index, file_list, K_template, distances = pickle.load(open("xray_distances.p", "rb"))
```

```python
min_distance = np.zeros(shape=(len(distances),))

def dist_function(idx):
    K_time, K_points, K_sig, K_cross = distances[idx]
    valid_idx = np.where(K_points > 50)[0]
    d= -np.log(K_cross[valid_idx]) + 0.5*np.log(K_sig[valid_idx]) + 0.5*np.log(K_template)
    #d = -K_cross[valid_idx]/K_points[valid_idx] + 0.5*K_sig[valid_idx]/K_points[valid_idx] + 0.5*K_template/110
    #d = -2*K_cross[valid_idx]/(110*K_points[valid_idx]) + K_sig[valid_idx]/K_points[valid_idx]**2 +K_template/110**2
    #d = -2*K_cross[valid_idx] + K_sig[valid_idx] +K_template
    return K_time[valid_idx], d

for i in range(len(distances)):
    _, D = dist_function(i)
    min_distance[i] = np.amin(D)

plt.plot(min_distance, '-o')
```

```python
file_idx = 27
print(file_idx)
K_time, D = dist_function(file_idx)
best = np.argmin(D)
plt.plot(K_time, D)
plt.plot(K_time[best], D[best], 'ro')
plt.ylabel('Distance metric between spike trains')

tgrid, smoothed_lc = kde_spike(times_template, sigma=50, resolution=1)
fig = plt.figure(figsize=(13, 6))
ax = fig.add_subplot(4, 1, 1)
ax.vlines(times_template, ymin=0.0, ymax=1.0, alpha=0.1)
ax.set_ylabel('Spike train')
ax = fig.add_subplot(4, 1, 2)
ax.plot(tgrid, smoothed_lc)
plt.plot([times_template[0], times_template[0] + T],
         [np.amin(smoothed_lc), np.amin(smoothed_lc)], 'r', linewidth=4)

times = get_light_curve(join(lc_path, file_list[lc_index[file_idx]]))
selection = (times > K_time[best]-5*T) & (times < K_time[best]+5*T)
tgrid, smoothed_lc = kde_spike(times[selection], sigma=50, resolution=10)
ax = fig.add_subplot(4, 1, 3)
ax.vlines(times[selection], ymin=0.0, ymax=1.0, alpha=0.01)
ax.set_ylabel('Spike train')
ax = fig.add_subplot(4, 1, 4)
ax.plot(tgrid, smoothed_lc)
plt.plot([K_time[best], K_time[best] + T],
         [np.amin(smoothed_lc), np.amin(smoothed_lc)], 'r', linewidth=4)
plt.tight_layout()
ax.set_ylabel('Smoothed')
ax.set_xlabel('Time [days]');
```
s = 0.01
G = np.exp(-0.5*D/s**2)
pca = PCA(n_components=2)
D_red = pca.fit_transform(G)
plt.scatter(D_red[:, 0], D_red[:, 1])
for i in range(M+1):
    plt.annotate(str(i), (D_red[i, 0], D_red[i, 1]))

```python

```


```python
%autosave 0
%matplotlib notebook
```

    Autosave disabled

```python
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sympy import Matrix, init_printing
init_printing()
```

```python
data = np.array([
    [172, 58, 63],
    [180, 80, 85],
    [175, 60, 70],
    [162, 50, 65],
    [167, 63, 70],
    [151, 45, 59],
    [188, 70, 90],
    [170, 70, 70],
    [166, 61, 66],
])
```

```python
avg = np.array([np.mean(data[:, 0]), np.mean(data[:, 1]), np.mean(data[:, 2])])
avg
```

    array([170.11111111,  61.88888889,  70.88888889])

```python
cov = np.cov(data.T)
Matrix(cov)
```

$$\left[\begin{matrix}112.861111111111 & 90.7638888888889 & 95.0138888888889\\90.7638888888889 & 113.361111111111 & 88.7361111111111\\95.0138888888889 & 88.7361111111111 & 103.611111111111\end{matrix}\right]$$

```python
# 固有値, 固有ベクトル
ev, evec = np.linalg.eig(cov)
evec = evec.T
Matrix(evec)
```

$$\left[\begin{matrix}-0.58904357698053 & -0.576998859387866 & -0.565773789321389\\-0.634318860412278 & -0.103630316639308 & 0.766094211437801\\0.500666803116854 & -0.810143859902881 & 0.304958486549253\end{matrix}\right]$$

```python
Matrix(ev)
```

$$\left[\begin{matrix}293.029493323743\\12.9370765367566\\23.8667634728339\end{matrix}\right]$$

```python
# 寄与率
crate = ev / np.sum(ev)
Matrix(crate)
```

$$\left[\begin{matrix}0.888416856969407\\0.0392230718648507\\0.0723600711657421\end{matrix}\right]$$

```python
# 累積寄与率
(crate[0], crate[0] + crate[2], crate[0] + crate[1] + crate[2])
```

```python
# 描画のために寄与率順にソートする
evec = np.array([evec[0], evec[2], evec[1]])
Matrix(evec)
```

$$\left[\begin{matrix}-0.58904357698053 & -0.576998859387866 & -0.565773789321389\\0.500666803116854 & -0.810143859902881 & 0.304958486549253\\-0.634318860412278 & -0.103630316639308 & 0.766094211437801\end{matrix}\right]$$

```python
# それぞれの内積はほぼ0
evec[0].dot(evec[1]), evec[0].dot(evec[2]), evec[1].dot(evec[2])
```

```python
fig = plt.figure()
ax = fig.gca(projection='3d', adjustable='box')
ax.axis('equal')
ax.set_xlabel('Height')
ax.set_ylabel('Weight')
ax.set_zlabel('Waist circumference')

distance = np.sqrt(((data.max(axis=0) - data.min(axis=0)) ** 2).sum())
right = distance / 2
left = -right

xs0 = np.array([evec[0][0] * left, evec[0][0] * right]) + avg[0]
ys0 = np.array([evec[0][1] * left, evec[0][1] * right]) + avg[1]
zs0 = np.array([evec[0][2] * left, evec[0][2] * right]) + avg[2]

xs1 = np.array([evec[1][0] * left, evec[1][0] * right]) + avg[0]
ys1 = np.array([evec[1][1] * left, evec[1][1] * right]) + avg[1]
zs1 = np.array([evec[1][2] * left, evec[1][2] * right]) + avg[2]

xs2 = np.array([evec[2][0] * left, evec[2][0] * right]) + avg[0]
ys2 = np.array([evec[2][1] * left, evec[2][1] * right]) + avg[1]
zs2 = np.array([evec[2][2] * left, evec[2][2] * right]) + avg[2]

ax.plot(xs0, ys0, zs0, 'r')
ax.plot(xs1, ys1, zs1, 'g')
ax.plot(xs2, ys2, zs2, 'b')

ax.scatter(data[:,0], data[:,1], data[:,2])
```

    <IPython.core.display.Javascript object>

    <mpl_toolkits.mplot3d.art3d.Path3DCollection at 0x7f25a847d198>

```python

```


```python
%matplotlib inline
```

Introduction to PyTorch
***********************

Introduction to Torch's tensor library
======================================

All of deep learning is computations on tensors, which are
generalizations of a matrix that can be indexed in more than 2
dimensions. We will see exactly what this means in-depth later. First,
lets look what we can do with tensors.

```python
# Author: Robert Guthrie

import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

torch.manual_seed(1)
```

Creating Tensors
~~~~~~~~~~~~~~~~

Tensors can be created from Python lists with the torch.Tensor()
function.

```python
# Create a torch.Tensor object with the given data.  It is a 1D vector
V_data = [1., 2., 3.]
V = torch.Tensor(V_data)
print(V)

# Creates a matrix
M_data = [[1., 2., 3.], [4., 5., 6]]
M = torch.Tensor(M_data)
print(M)

# Create a 3D tensor of size 2x2x2.
T_data = [[[1., 2.], [3., 4.]],
          [[5., 6.], [7., 8.]]]
T = torch.Tensor(T_data)
print(T)
```

What is a 3D tensor anyway? Think about it like this. If you have a
vector, indexing into the vector gives you a scalar. If you have a
matrix, indexing into the matrix gives you a vector. If you have a 3D
tensor, then indexing into the tensor gives you a matrix!

A note on terminology:
when I say "tensor" in this tutorial, it refers
to any torch.Tensor object. Matrices and vectors are special cases of
torch.Tensors, where their dimension is 1 and 2 respectively. When I am
talking about 3D tensors, I will explicitly use the term "3D tensor".

```python
# Index into V and get a scalar
print(V[0])

# Index into M and get a vector
print(M[0])

# Index into T and get a matrix
print(T[0])
```

You can also create tensors of other datatypes. The default, as you can
see, is Float. To create a tensor of integer types, try
torch.LongTensor(). Check the documentation for more data types, but
Float and Long will be the most common.

You can create a tensor with random data and the supplied dimensionality
with torch.randn()

```python
x = torch.randn((3, 4, 5))
print(x)
```

Operations with Tensors
~~~~~~~~~~~~~~~~~~~~~~~

You can operate on tensors in the ways you would expect.

```python
x = torch.Tensor([1., 2., 3.])
y = torch.Tensor([4., 5., 6.])
z = x + y
print(z)
```

See `the documentation <http://pytorch.org/docs/torch.html>`__ for a
complete list of the massive number of operations available to you. They
expand beyond just mathematical operations.

One helpful operation that we will make use of later is concatenation.

```python
# By default, it concatenates along the first axis (concatenates rows)
x_1 = torch.randn(2, 5)
y_1 = torch.randn(3, 5)
z_1 = torch.cat([x_1, y_1])
print(z_1)

# Concatenate columns:
x_2 = torch.randn(2, 3)
y_2 = torch.randn(2, 5)
# second arg specifies which axis to concat along
z_2 = torch.cat([x_2, y_2], 1)
print(z_2)

# If your tensors are not compatible, torch will complain.  Uncomment to see the error
# torch.cat([x_1, x_2])
```

Reshaping Tensors
~~~~~~~~~~~~~~~~~

Use the .view() method to reshape a tensor. This method receives heavy
use, because many neural network components expect their inputs to have
a certain shape. Often you will need to reshape before passing your data
to the component.

```python
x = torch.randn(2, 3, 4)
print(x)
print(x.view(2, 12))  # Reshape to 2 rows, 12 columns
# Same as above.  If one of the dimensions is -1, its size can be inferred
print(x.view(2, -1))
```

Computation Graphs and Automatic Differentiation
================================================

The concept of a computation graph is essential to efficient deep
learning programming, because it allows you to not have to write the
back propagation gradients yourself. A computation graph is simply a
specification of how your data is combined to give you the output. Since
the graph totally specifies what parameters were involved with which
operations, it contains enough information to compute derivatives. This
probably sounds vague, so lets see what is going on using the
fundamental class of Pytorch: autograd.Variable.

First, think from a programmers perspective. What is stored in the
torch.Tensor objects we were creating above? Obviously the data and the
shape, and maybe a few other things. But when we added two tensors
together, we got an output tensor. All this output tensor knows is its
data and shape. It has no idea that it was the sum of two other tensors
(it could have been read in from a file, it could be the result of some
other operation, etc.)

The Variable class keeps track of how it was created. Lets see it in
action.

```python
# Variables wrap tensor objects
x = autograd.Variable(torch.Tensor([1., 2., 3]), requires_grad=True)
# You can access the data with the .data attribute
print(x.data)

# You can also do all the same operations you did with tensors with Variables.
y = autograd.Variable(torch.Tensor([4., 5., 6]), requires_grad=True)
z = x + y
print(z.data)

# BUT z knows something extra.
print(z.grad_fn)
```

So Variables know what created them. z knows that it wasn't read in from
a file, it wasn't the result of a multiplication or exponential or
whatever. And if you keep following z.grad_fn, you will find yourself at
x and y.

But how does that help us compute a gradient?

```python
# Lets sum up all the entries in z
s = z.sum()
print(s)
print(s.grad_fn)
```

So now, what is the derivative of this sum with respect to the first
component of x? In math, we want

\begin{align}\frac{\partial s}{\partial x_0}\end{align}

Well, s knows that it was created as a sum of the tensor z. z knows
that it was the sum x + y. So

\begin{align}s = \overbrace{x_0 + y_0}^\text{$z_0$} + \overbrace{x_1 + y_1}^\text{$z_1$} + \overbrace{x_2 + y_2}^\text{$z_2$}\end{align}

And so s contains enough information to determine that the derivative
we want is 1!

Of course this glosses over the challenge of how to actually compute
that derivative. The point here is that s is carrying along enough
information that it is possible to compute it. In reality, the
developers of Pytorch program the sum() and + operations to know how to
compute their gradients, and run the back propagation algorithm. An
in-depth discussion of that algorithm is beyond the scope of this
tutorial.

Lets have Pytorch compute the gradient, and see that we were right:
(note if you run this block multiple times, the gradient will increment.
That is because Pytorch *accumulates* the gradient into the .grad
property, since for many models this is very convenient.)

```python
# calling .backward() on any variable will run backprop, starting from it.
s.backward()
print(x.grad)
```

Understanding what is going on in the block below is crucial for being a
successful programmer in deep learning.

```python
x = torch.randn((2, 2))
y = torch.randn((2, 2))
z = x + y  # These are Tensor types, and backprop would not be possible

var_x = autograd.Variable(x, requires_grad=True)
var_y = autograd.Variable(y, requires_grad=True)
# var_z contains enough information to compute gradients, as we saw above
var_z = var_x + var_y
print(var_z.grad_fn)

var_z_data = var_z.data  # Get the wrapped Tensor object out of var_z...
# Re-wrap the tensor in a new variable
new_var_z = autograd.Variable(var_z_data)

# ... does new_var_z have information to backprop to x and y?
# NO!
print(new_var_z.grad_fn)
# And how could it?  We yanked the tensor out of var_z (that is
# what var_z.data is).  This tensor doesn't know anything about
# how it was computed.  We pass it into new_var_z, and this is all the
# information new_var_z gets.  If var_z_data doesn't know how it was
# computed, theres no way new_var_z will.
# In essence, we have broken the variable away from its past history
```

Here is the basic, extremely important rule for computing with
autograd.Variables (note this is more general than Pytorch. There is an
equivalent object in every major deep learning toolkit):

**If you want the error from your loss function to backpropagate to a
component of your network, you MUST NOT break the Variable chain from
that component to your loss Variable. If you do, the loss will have no
idea your component exists, and its parameters can't be updated.**

I say this in bold, because this error can creep up on you in very
subtle ways (I will show some such ways below), and it will not cause
your code to crash or complain, so you must be careful.


## Lista de Exercício 03 e 04 - Fundamentos de Matemática para Computação.

# Universidade Federal do ABC - UFABC
## Centro de Matemática, Computação e Cognição - CMCC

## Disciplina:  Fundamentos de Matemática para a Computação -  CCM-007

Responsável: Prof. Dr. Saul Leite

Estudantes: Bruno Aristimunha.

Santo André, 07 de Março de 2019

### Primeira Parte do Curso

#### Objetivos

O objetivo dessa lista é explorar e aplicar os conceitos de sala de aula sobre **Interpolação Polinomial**.

Para alcançar esse objetivo a resolução dos exercícios é necessária.

Bibliotecas empregadas para resolução da Lista 03 e 04

```
import matplotlib.pylab as plt

import sympy as sy

import numpy as np

x = sy.Symbol('x')
plt.style.use("ggplot")
```

### Funções desenvolvidas para Lista 03
___

```
def coeficientes_newton(x_i,f_x):
  #entrada: pares (x, y)
  #saida: coeficientes
  d = f_x.copy()

  for k in range(1,len(f_x)):
    for i in range(len(f_x), k, -1):
      d[i-1] = (d[i-1]-d[i-2]) / (x_i[i-1] - x_i[i-(k+1)])
  return d
```

```
def funcao_polinomial(x_i, f_x):
    coeficientes = coeficientes_newton(x_i, f_x)

    n = len(x_i) - 1 # grau do polinomio
    p = coeficientes[n]

    for k in range(1,n+1):
        p = coeficientes[n-k] + (x -x_i[n-k])*p
    return p

```

___

# Exercício 01.
___

### a. Determinar um valor aproximado para $\sqrt{1.12}$ usando o polinômio de interpolação de Newton sobre três pontos. (escolha pontos próximos a $x=1.12$)

___

```
x_e1 = [1,1.1,1.15,1.25,1.30]
f_e1 = [1.000, 1.049, 1.072, 1.118, 1.140]
```

```
x_e1[:3]
p_e1 = funcao_polinomial(x_e1[:3].copy(), f_e1[:3].copy())
```

```
p_e1 = lambda x: (-0.199999999999964*x + 0.70999999999996)*(x - 1) + 1.0

```

O valor estimado para $\sqrt{1.12}$ é:

```
p_e1(1.12)
```

    1.0583200000000001

Esse valor está com a precisão de $10^{-5}$

```
np.abs(p_e1(1.12) - np.sqrt(1.12))
```

    1.947557416381862e-05

___

### b. Calcular um limitante superior para o erro de aproximação para valores próximos dos pontos de interpolação escolhidos (note que não são igualmente espaçado)

___

Seja $f(x) = \sqrt{x}$, $f'(x) =\frac{1}{2} x^{-\frac{1}{2}}$, $f^{''}(x) = -\frac{1}{4}x^{-\frac{3}{2}}$, $f^{(3)}(x) =\frac{3}{8}x^{\frac{5}{8}}$

Seja a função de erro para o polinômio de grau 3.

>$|E_n(x)| = (x-x_0)(x-x_1)(x-x_2) \frac{f^{n+1}(\xi)}{(n+1)!}$

>$|E_n(x)| \leq |(x-1)(x-1.1)(x-1.15) |\cdot \left| \frac{3}{8\cdot 3!} \right|$

>$|E_n(x)| \leq \max|(x-1)(x-1.1)(x-1.15) |\cdot  \left| \frac{1}{16} \right|$

Estudando o polinômio temos que:

>$(x-1)(x-1.1)(x-1.15) = -1.265 + 3.515 x - 3.25 x^2 + x^3$

>$\frac{d(-1.265 + 3.515 x - 3.25 x^2 + x^3)}{dx} = 3.515 - 6.5 x + 3 x^2$

>$3.515 - 6.5 x + 3 x^2 =  0 => r_1 = 1.03924, r_2 = 1.12743$

Verificando computacionalmente temos que módulo que maximiza a função é:

```
f_linha = lambda x: -1.265 + 3.515*x - 3.25*x**2 + x**3
np.abs(f_linha(1.03924)), np.abs(f_linha(1.12743))
```

    (0.00026407647302506554, 7.889128859273065e-05)

O valor que maximiza a função é: 0.00026407647302506554, logo, o limitante superior é:

```
(0.00026407647302506554/16)
```

    1.6504779564066596e-05

___
# Exercício 02.

Mostre que a interpolação de um polinômio de grau $n$ sobre $n+k$ pontos, $k\geq1$, é exata.(Dica: use a fórmula para o erro de interpolação)
___

Da forma do erro temos que a $(n+1)$-derivada  de um polinômio de grau n será 0. Dessa forma, o erro será exato.

___
# Exercício 03

Determine o número de pontos necessários para aproximar a função $f(x) = xe^{3x}$ no intervalo $[0, 0.4]$ com erro inferior a $10^{−7}$ usando pontos igualmente espaçados. (Dica: mostre, usando indução, que $f^{(k)} = (k + 3x)3^{k−1} e^{3x}$, k ≥ 1.)
___

## **Caso base: **
$k  =  1$

$\begin{align}
f_1(x) & = xe^{3x} \\
f'_1(x)   & = e^{3x} + 3x  e^{3x}\\
\\
\text{Exatamente como nossa fórmula nos diz.}\\
\\
f^{(1)}& = (k + 3x)3^{k−1} e^{3x}\\
& = (1 + 3x)3^{1-1} e^{3x}\\
& = (1 + 3x)1\cdot e^{3x}\\
f^{(1)}& = e^{3x} +3x\cdot e^{3x}
\end{align}$

Passo base é verdadeiro :)

## **Para hipótese de indução**

Assumimos que nosso caso $k-1$ é verdade. Ou seja:

$\begin{align}
f_{(k-1)}(x) & = (k-1+3x)3^{k-2} e^{3x}\\
\end{align}$

## **Passo de indução**

$\begin{align}
f_{k-1}& =(k-1+3x)3^{k-2} e^{3x} \\
\text{Ao derivarmos temos então:}\\
f'_{(k-1)}(x)& = (k + 3x)\quad3^{k-1} e^{3 x} \Box \\
\end{align}$

Com a nossa $f^{n+1}$ temos:

> $10^{-7} \leq \frac{h^{n+1}}{4(n+1)} \cdot \max_{t\in[0,0.4]} | f^{n+1}(t) |$

> $10^{-7} \leq \frac{({\frac{0.4}{n}})^{n+1}}{4(n+1)} \cdot(n+1+3\cdot 0.4) 3^{n} e^{3\cdot 0.4} $

```
import numpy as np

condicao = True
n=1
while condicao:

  print(n,"|",(.830029* 0.4**(n + 1)* (3**n)* (1/n)**(n + 1) *(n + 2.2))/(n + 1))

  condicao = (10**(-7) <= ((0.4/n)**(n+1))/(4*(n+1)) * (n +1 + 3* 0.4)*(3**(n))*(np.e**(3*0.4)))

  n=n +1
```

    1 | 0.6374622720000002
    2 | 0.08366692320000002
    3 | 0.009207788373333333
    4 | 0.0008336811276000002
    5 | 6.344840482652165e-05
    6 | 4.148563992380952e-06
    7 | 2.3732063415057555e-07
    8 | 1.205455007460938e-08

Dessa forma, nós observamos que o número de pontos necesários é 9. Isso ocorre, pois, temos o erro para a função de grau $n+1$.

___

# Exercício 04.

Para gerar uma aproximação para a parte superior do pato, os pontos ilustrados no desenho foram selecionados e são apresentados na tabela que segue.

    x
    0.9
    1.3
    1.9
    2.1
    2.6
    3.0
    3.9
    4.4
    4.7
    5.0
    6.0
    7.0
    8.0
    9.2
    10.5
    11.3
    11.6
    12.0
    12.6
    13.0
    13.3

    f(x)
    1.3
    1.5
    1.85
    2.1
    2.6
    2.7
    2.4
    2.15
    2.05
    2.1
    2.25
    2.3
    2.25
    1.95
    1.4
    0.9
    0.7
    0.6
    0.5
    0.4
    0.25

Implemente em Python o polinômio interpolador de Newton para aproximar o desenho da parte superior do pato.

---

```
import numpy as np
import matplotlib.pylab as plt
```

```
x_e4 = [0.9,1.3,1.9,2.1,2.6,3.0,3.9,4.4,4.7,5.0,6.0,7.0,8.0,9.2,10.5,11.3,11.6,12.0,12.6,13.0,13.3]
f_e4 = [1.3,1.5,1.85,2.1,2.6,2.7,2.4,2.15,2.05,2.1,2.25,2.3,2.25,1.95,1.4,0.9,0.7,0.6,0.5,0.4,0.25]
x_e5 = np.arange(np.min(x_e4),np.max(x_e4),0.1)
```

```
funcao_polinomial(x_e4.copy(),f_e4.copy())
```

    (x - 0.9)*((x - 1.3)*((x - 1.9)*((x - 2.1)*((x - 2.6)*((x - 3.0)*((x - 3.9)*((x - 4.4)*((x - 4.7)*((x - 5.0)*((x - 6.0)*((x - 7.0)*((x - 8.0)*((x - 9.2)*((x - 10.5)*((x - 11.3)*((x - 11.6)*((x - 12.0)*((-3.074530780108e-11*x + 6.51542950918716e-10)*(x - 12.6) - 1.98371331929768e-9) + 1.46954842089078e-8) - 9.86036108443615e-8) + 6.08595004676425e-7) - 3.45353446933275e-6) + 1.57981716791882e-5) - 4.29023399481625e-5) - 6.34107592999165e-6) + 0.000572931763659381) - 0.00185867502994866) - 0.00254815041559561) + 0.0387469060492263) - 0.18391194861783) + 0.566835125658653) - 0.906324068088771) + 0.624999999999998) + 0.0833333333333337) + 0.5) + 1.3

```
p = lambda x: (x - 0.9)*((x - 1.3)*((x - 1.9)*((x - 2.1)*((x - 2.6)*((x - 3.0)*((x - 3.9)*((x - 4.4)*((x - 4.7)*((x - 5.0)*((x - 6.0)*((x - 7.0)*((x - 8.0)*((x - 9.2)*((x - 10.5)*((x - 11.3)*((x - 11.6)*((x - 12.0)*((-3.074530780108e-11*x + 6.51542950918716e-10)*(x - 12.6) - 1.98371331929768e-9) + 1.46954842089078e-8) - 9.86036108443615e-8) + 6.08595004676425e-7) - 3.45353446933275e-6) + 1.57981716791882e-5) - 4.29023399481625e-5) - 6.34107592999165e-6) + 0.000572931763659381) - 0.00185867502994866) - 0.00254815041559561) + 0.0387469060492263) - 0.18391194861783) + 0.566835125658653) - 0.906324068088771) + 0.624999999999998) + 0.0833333333333337) + 0.5) + 1.3
```

```
fig, ax = plt.subplots(figsize=(20,10))

ax.plot(x_e4, f_e4, 'o', c='r',label='pontos originais')
#ax.plot(x_e4, list(map(p,x_e4)), c='g', marker='+', label='interpolacao por Newton')
ax.plot(x_e5, list(map(p,x_e5)), c='b', marker='+', label='interpolando com mais pontos')

ax.legend(loc='lower left')

plt.show()
```

Observamos no comportamento da função o fenômeno de Runge. Em suma, estamos empregando uma função de complexidade muito acima da necessária para descrever o nosso conjunto de dados. Uma das formas para contornar a situação consiste em amostrar o dataset. A técnica escolhida pode ser diversa. Nesse caso, optamos por escolher 1 ponto a cada 4 pontos.

```
x_04_01 = x_e4[:5]
f_04_01 = f_e4[:5]

x_04_02 = x_e4[4:9]
f_04_02 = f_e4[4:9]

x_04_03 = x_e4[8:13]
f_04_03 = f_e4[8:13]

x_04_04 = x_e4[12:17]
f_04_04 = f_e4[12:17]

x_04_05 = x_e4[16:21]
f_04_05 = f_e4[16:21]

p_04_01 = funcao_polinomial(x_04_01,f_04_01)
p_04_02 = funcao_polinomial(x_04_02,f_04_02)
p_04_03 = funcao_polinomial(x_04_03,f_04_03)
p_04_04 = funcao_polinomial(x_04_04,f_04_04)
p_04_05 = funcao_polinomial(x_04_05,f_04_05)
```

```
print(p_04_01)
print(p_04_02)
print(p_04_03)
print(p_04_04)
print(p_04_05)
```

    (x - 0.9)*((x - 1.3)*((-0.906324068088771*x + 2.52828054298642)*(x - 1.9) + 0.0833333333333337) + 0.5) + 1.3
    (x - 2.6)*((x - 3.0)*((0.00448897507720958*x + 0.163398692810462)*(x - 3.9) - 0.448717948717949) + 0.25) + 2.6
    (x - 4.7)*((x - 5.0)*((0.00489848315935264*x - 0.0504543765413322)*(x - 6.0) - 0.0128205128205137) + 0.166666666666668) + 2.05
    (x - 8.0)*((x - 9.2)*((0.00901105067771695*x - 0.10998338081671)*(x - 10.5) - 0.0692307692307691) - 0.25) + 2.25
    (x - 11.6)*((x - 12.0)*((-0.053867700926524*x + 0.581232492997194)*(x - 12.6) + 0.083333333333333) - 0.25) + 0.7

```
p_04_01 = lambda x: (x - 0.9)*((x - 1.3)*((-0.906324068088771*x + 2.52828054298642)*(x - 1.9) + 0.0833333333333337) + 0.5) + 1.3
p_04_02 = lambda x: (x - 2.6)*((x - 3.0)*((0.00448897507720958*x + 0.163398692810462)*(x - 3.9) - 0.448717948717949) + 0.25) + 2.6
p_04_03 = lambda x: (x - 4.7)*((x - 5.0)*((0.00489848315935264*x - 0.0504543765413322)*(x - 6.0) - 0.0128205128205137) + 0.166666666666668) + 2.05
p_04_04 = lambda x: (x - 8.0)*((x - 9.2)*((0.00901105067771695*x - 0.10998338081671)*(x - 10.5) - 0.0692307692307691) - 0.25) + 2.25
p_04_05 = lambda x: (x - 11.6)*((x - 12.0)*((-0.053867700926524*x + 0.581232492997194)*(x - 12.6) + 0.083333333333333) - 0.25) + 0.725
```

```
fig, ax = plt.subplots(figsize=(20,10))

ax.plot(x_e4, f_e4, 'o', c='r',label='pontos originais')
ax.plot(x_04_01, list(map(p_04_01,x_04_01)), c='b', marker='+',
        label='1 divisao')
ax.plot(x_04_02, list(map(p_04_02,x_04_02)), c='g', marker='+',
        label='2 divisao')
ax.plot(x_04_03, list(map(p_04_03,x_04_03)), c='y', marker='+',
        label='3 divisao')
ax.plot(x_04_04, list(map(p_04_04,x_04_04)), c='turquoise', marker='+',
        label='4 divisao')
ax.plot(x_04_05, list(map(p_04_05,x_04_05)), c='azure', marker='+',
        label='5 divisao')

ax.legend(loc='lower left')
```

___
## Exercício 5

Refazer o exercício **E4** da aula passada (do pato) usando splines
___

```
from scipy.interpolate import interp1d
from scipy.interpolate import CubicSpline
```

```

cs = CubicSpline(x=x_e4, y=f_e4)

fig, ax = plt.subplots(figsize=(15,10))
ax.plot(x_e4, f_e4, 'o', label='Dados')
ax.plot(x_e5, list(map(p,x_e5)), c='g', marker='+', label='Interpolação Newton')
ax.plot(x_e5, cs(x_e5), label="Splines Cúbicos", c='r')

ax.legend(loc='lower left')
```

```
fig, ax = plt.subplots(figsize=(15,7.5))
ax.plot(x_e4, f_e4, 'o', label='Dados')
ax.plot(x_e5, cs(x_e5), label="Splines Cúbicos")
for i in range(np.shape(cs.c)[0]):
  ax.plot(x_e5, cs(x_e5,i), label="S^"+str(i))
ax.legend(loc='best', ncol=2)
```


# Batch Normalization
One way to make deep networks easier to train is to use more sophisticated optimization procedures such as SGD+momentum, RMSProp, or Adam. Another strategy is to change the architecture of the network to make it easier to train.
One idea along these lines is batch normalization which was proposed by [1] in 2015.

The idea is relatively straightforward. Machine learning methods tend to work better when their input data consists of uncorrelated features with zero mean and unit variance. When training a neural network, we can preprocess the data before feeding it to the network to explicitly decorrelate its features; this will ensure that the first layer of the network sees data that follows a nice distribution. However, even if we preprocess the input data, the activations at deeper layers of the network will likely no longer be decorrelated and will no longer have zero mean or unit variance since they are output from earlier layers in the network. Even worse, during the training process the distribution of features at each layer of the network will shift as the weights of each layer are updated.

The authors of [1] hypothesize that the shifting distribution of features inside deep neural networks may make training deep networks more difficult. To overcome this problem, [1] proposes to insert batch normalization layers into the network. At training time, a batch normalization layer uses a minibatch of data to estimate the mean and standard deviation of each feature. These estimated means and standard deviations are then used to center and normalize the features of the minibatch. A running average of these means and standard deviations is kept during training, and at test time these running averages are used to center and normalize features.

It is possible that this normalization strategy could reduce the representational power of the network, since it may sometimes be optimal for certain layers to have features that are not zero-mean or unit variance. To this end, the batch normalization layer includes learnable shift and scale parameters for each feature dimension.

[1] [Sergey Ioffe and Christian Szegedy, "Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shift", ICML 2015.](https://arxiv.org/abs/1502.03167)

```python
# As usual, a bit of setup
import time
import numpy as np
import matplotlib.pyplot as plt
from ie590.classifiers.fc_net import *
from ie590.data_utils import get_CIFAR10_data
from ie590.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array
from ie590.solver import Solver
# import pixiedust

%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

def rel_error(x, y):
    """ returns relative error """
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

def print_mean_std(x,axis=0):
    print('  means: ', x.mean(axis=axis))
    print('  stds:  ', x.std(axis=axis))
    print()
```

    Pixiedust database opened successfully

    Pixiedust version 1.1.17

```python
# Load the (preprocessed) CIFAR10 data.
data = get_CIFAR10_data()
for k, v in data.items():
  print('%s: ' % k, v.shape)
```

    X_train:  (49000, 3, 32, 32)
    y_train:  (49000,)
    X_val:  (1000, 3, 32, 32)
    y_val:  (1000,)
    X_test:  (1000, 3, 32, 32)
    y_test:  (1000,)

## Batch normalization: forward
In the file `ie590/layers.py`, implement the batch normalization forward pass in the function `batchnorm_forward`. Once you have done so, run the following to test your implementation.

Referencing the paper linked to above in [1] may be helpful!

```python
# %pixie_debugger

# Check the training-time forward pass by checking means and variances
# of features both before and after batch normalization

# Simulate the forward pass for a two-layer network
np.random.seed(220)
N, D1, D2, D3 = 200, 50, 60, 3
X = np.random.randn(N, D1)
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)
a = np.maximum(0, X.dot(W1)).dot(W2)

print('Before batch normalization:')
print_mean_std(a,axis=0)

gamma = np.ones((D3,))
beta = np.zeros((D3,))
# Means should be close to zero and stds close to one
print('After batch normalization (gamma=1, beta=0)')
a_norm, _ = batchnorm_forward(a, gamma, beta, {'mode': 'train', 'flag': None})
print_mean_std(a_norm,axis=0)

gamma = np.asarray([1.0, 2.0, 3.0])
beta = np.asarray([11.0, 12.0, 13.0])
# Now means should be close to beta and stds close to gamma
print('After batch normalization (gamma=', gamma, ', beta=', beta, ')')
a_norm, _ = batchnorm_forward(a, gamma, beta, {'mode': 'train','flag': None})
print_mean_std(a_norm,axis=0)
```

    Before batch normalization:
      means:  [28.57337215 27.08029233  1.62528447]
      stds:   [34.85744517 25.9706331  29.6313792 ]

    After batch normalization (gamma=1, beta=0)
      means:  [ 2.24820162e-16  3.88578059e-16 -4.44089210e-18]
      stds:   [1.         0.99999999 0.99999999]

    After batch normalization (gamma= [1. 2. 3.] , beta= [11. 12. 13.] )
      means:  [11. 12. 13.]
      stds:   [1.         1.99999999 2.99999998]

```python
# Check the test-time forward pass by running the training-time
# forward pass many times to warm up the running averages, and then
# checking the means and variances of activations after a test-time
# forward pass.

np.random.seed(220)
N, D1, D2, D3 = 200, 50, 60, 3
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)

bn_param = {'mode': 'train', 'flag': None}
gamma = np.ones(D3)
beta = np.zeros(D3)

for t in range(50):
  X = np.random.randn(N, D1)
  a = np.maximum(0, X.dot(W1)).dot(W2)
  batchnorm_forward(a, gamma, beta, bn_param)

bn_param['mode'] = 'test'
X = np.random.randn(N, D1)
a = np.maximum(0, X.dot(W1)).dot(W2)
a_norm, _ = batchnorm_forward(a, gamma, beta, bn_param)

# Means should be close to zero and stds close to one, but will be
# noisier than training-time forward passes.
print('After batch normalization (test-time):')
print_mean_std(a_norm,axis=0)
```

    After batch normalization (test-time):
      means:  [-0.00462469  0.06873645  0.09765526]
      stds:   [0.96218861 1.04444084 0.9853543 ]

## Batch normalization: backward
Now implement the backward pass for batch normalization in the function `batchnorm_backward`.

To derive the backward pass you should write out the computation graph for batch normalization and backprop through each of the intermediate nodes. Some intermediates may have multiple outgoing branches; make sure to sum gradients across these branches in the backward pass.

Once you have finished, run the following to numerically check your backward pass.

```python
# Gradient check batchnorm backward pass
np.random.seed(220)
N, D = 4, 5
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

bn_param = {'mode': 'train', 'flag': None}
fx = lambda x: batchnorm_forward(x, gamma, beta, bn_param)[0]
fg = lambda a: batchnorm_forward(x, a, beta, bn_param)[0]
fb = lambda b: batchnorm_forward(x, gamma, b, bn_param)[0]

dx_num = eval_numerical_gradient_array(fx, x, dout)
da_num = eval_numerical_gradient_array(fg, gamma.copy(), dout)
db_num = eval_numerical_gradient_array(fb, beta.copy(), dout)

_, cache = batchnorm_forward(x, gamma, beta, bn_param)
dx, dgamma, dbeta = batchnorm_backward(dout, cache)
#You should expect to see relative errors between 1e-13 and 1e-8
print('dx error: ', rel_error(dx_num, dx))
print('dgamma error: ', rel_error(da_num, dgamma))
print('dbeta error: ', rel_error(db_num, dbeta))
```

    dx error:  5.028689560610235e-09
    dgamma error:  2.2733064981744968e-11
    dbeta error:  3.276531879937969e-12

## Batch normalization: alternative backward
In class we talked about two different implementations for the sigmoid backward pass. One strategy is to write out a computation graph composed of simple operations and backprop through all intermediate values. Another strategy is to work out the derivatives on paper. For example, you can derive a very simple formula for the sigmoid function's backward pass by simplifying gradients on paper.

Surprisingly, it turns out that you can do a similar simplification for the batch normalization backward pass too!

In the forward pass, given a set of inputs $X=\begin{bmatrix}x_1\\x_2\\...\\x_N\end{bmatrix}$,

we first calculate the mean $\mu$ and variance $v$.
With $\mu$ and $v$ calculated, we can calculate the standard deviation $\sigma$  and normalized data $Y$.
The equations and graph illustration below describe the computation ($y_i$ is the i-th element of the vector $Y$).

\begin{align}
& \mu=\frac{1}{N}\sum_{k=1}^N x_k  &  v=\frac{1}{N}\sum_{k=1}^N (x_k-\mu)^2 \\
& \sigma=\sqrt{v+\epsilon}         &  y_i=\frac{x_i-\mu}{\sigma}
\end{align}

The meat of our problem during backpropagation is to compute $\frac{\partial L}{\partial X}$, given the upstream gradient we receive, $\frac{\partial L}{\partial Y}.$ To do this, recall the chain rule in calculus gives us $\frac{\partial L}{\partial X} = \frac{\partial L}{\partial Y} \cdot \frac{\partial Y}{\partial X}$.

The unknown/hart part is $\frac{\partial Y}{\partial X}$. We can find this by first deriving step-by-step our local gradients at
$\frac{\partial v}{\partial X}$, $\frac{\partial \mu}{\partial X}$,
$\frac{\partial \sigma}{\partial v}$,
$\frac{\partial Y}{\partial \sigma}$, and $\frac{\partial Y}{\partial \mu}$,
and then use the chain rule to compose these gradients (which appear in the form of vectors!) appropriately to compute $\frac{\partial Y}{\partial X}$.

If it's challenging to directly reason about the gradients over $X$ and $Y$ which require matrix multiplication, try reasoning about the gradients in terms of individual elements $x_i$ and $y_i$ first: in that case, you will need to come up with the derivations for $\frac{\partial L}{\partial x_i}$, by relying on the Chain Rule to first calculate the intermediate $\frac{\partial \mu}{\partial x_i}, \frac{\partial v}{\partial x_i}, \frac{\partial \sigma}{\partial x_i},$ then assemble these pieces to calculate $\frac{\partial y_i}{\partial x_i}$.

You should make sure each of the intermediary gradient derivations are all as simplified as possible, for ease of implementation.

After doing so, implement the simplified batch normalization backward pass in the function `batchnorm_backward_alt` and compare the two implementations by running the following. Your two implementations should compute nearly identical results, but the alternative implementation should be a bit faster.

```python
# %%pixie_debugger
np.random.seed(220)
N, D = 1000, 5000
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

bn_param = {'mode': 'train', 'flag': None}
out, cache = batchnorm_forward(x, gamma, beta, bn_param)

t1 = time.time()
dx1, dgamma1, dbeta1 = batchnorm_backward(dout, cache)
t2 = time.time()
dx2, dgamma2, dbeta2 = batchnorm_backward_alt(dout, cache)
t3 = time.time()

print('dx difference: ', rel_error(dx1, dx2))
print('dgamma difference: ', rel_error(dgamma1, dgamma2))
print('dbeta difference: ', rel_error(dbeta1, dbeta2))
print('speedup: %.2fx' % ((t2 - t1) / (t3 - t2)))
```

    dx difference:  2.919489291497593e-10
    dgamma difference:  0.0
    dbeta difference:  0.0
    speedup: 1.56x

## Fully Connected Nets with Batch Normalization
Now that you have a working implementation for batch normalization, go back to your `FullyConnectedNet` in the file `ie590/classifiers/fc_net.py`. Modify your implementation to add batch normalization.

Concretely, when the `normalization` flag is set to `"batchnorm"` in the constructor, you should insert a batch normalization layer before each ReLU nonlinearity. The outputs from the last layer of the network should not be normalized. Once you are done, run the following to gradient-check your implementation.

HINT: You might find it useful to define an additional helper layer similar to those in the file `ie590/layer_utils.py`. If you decide to do so, do it in the file `ie590/classifiers/fc_net.py`.

```python
np.random.seed(220)
N, D, H1, H2, C = 2, 15, 20, 30, 10
X = np.random.randn(N, D)
y = np.random.randint(C, size=(N,))

# You should expect losses between 1e-4~1e-10 for W,
# losses between 1e-08~1e-10 for b,
# and losses between 1e-08~1e-09 for beta and gammas.
for reg in [0, 3.14]:
  print('Running check with reg = ', reg)
  model = FullyConnectedNet([H1, H2], input_dim=D, num_classes=C,
                            reg=reg, weight_scale=5e-2, dtype=np.float64,
                            normalization='batchnorm')

  loss, grads = model.loss(X, y)
  print('Initial loss: ', loss)

  for name in sorted(grads):
    f = lambda _: model.loss(X, y)[0]
    grad_num = eval_numerical_gradient(f, model.params[name], verbose=False, h=1e-5)
    print('%s relative error: %.2e' % (name, rel_error(grad_num, grads[name])))
  if reg == 0: print()
```

    Running check with reg =  0
    Initial loss:  2.6098865752882405
    W1 relative error: 5.64e-05
    W2 relative error: 3.80e-06
    W3 relative error: 1.30e-09
    b1 relative error: 8.88e-06
    b2 relative error: 1.25e-08
    b3 relative error: 8.26e-11
    beta1 relative error: 8.40e-09
    beta2 relative error: 3.62e-09
    gamma1 relative error: 8.66e-09
    gamma2 relative error: 8.98e-09

    Running check with reg =  3.14
    Initial loss:  7.008456540025971
    W1 relative error: 2.14e-05
    W2 relative error: 6.13e-06
    W3 relative error: 2.16e-08
    b1 relative error: 5.82e-06
    b2 relative error: 1.83e-07
    b3 relative error: 3.12e-10
    beta1 relative error: 4.22e-09
    beta2 relative error: 1.21e-08
    gamma1 relative error: 4.53e-09
    gamma2 relative error: 1.60e-08

# Batchnorm for deep networks
Run the following to train a six-layer network on a subset of 1000 training examples both with and without batch normalization.

```python
# %%pixie_debugger
np.random.seed(220)
# Try training a very deep net with batchnorm
hidden_dims = [100, 100, 100, 100, 100]

num_train = 1000
small_data = {
  'X_train': data['X_train'][:num_train],
  'y_train': data['y_train'][:num_train],
  'X_val': data['X_val'],
  'y_val': data['y_val'],
}

weight_scale = 2e-2
bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization='batchnorm')
model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)

print('Solver with batch norm:')
bn_solver = Solver(bn_model, small_data,
                num_epochs=10, batch_size=50,
                update_rule='adam',
                optim_config={
                  'learning_rate': 1e-3,
                },
                verbose=True,print_every=20)
bn_solver.train()

print('\nSolver without batch norm:')
solver = Solver(model, small_data,
                num_epochs=10, batch_size=50,
                update_rule='adam',
                optim_config={
                  'learning_rate': 1e-3,
                },
                verbose=True, print_every=20)
solver.train()
```

    Solver with batch norm:
    (Iteration 1 / 200) loss: 2.315319
    (Epoch 0 / 10) train acc: 0.124000; val_acc: 0.147000
    (Epoch 1 / 10) train acc: 0.301000; val_acc: 0.246000
    (Iteration 21 / 200) loss: 2.048442
    (Epoch 2 / 10) train acc: 0.404000; val_acc: 0.286000
    (Iteration 41 / 200) loss: 1.782234
    (Epoch 3 / 10) train acc: 0.452000; val_acc: 0.295000
    (Iteration 61 / 200) loss: 1.455887
    (Epoch 4 / 10) train acc: 0.538000; val_acc: 0.288000
    (Iteration 81 / 200) loss: 1.303186
    (Epoch 5 / 10) train acc: 0.618000; val_acc: 0.320000
    (Iteration 101 / 200) loss: 1.324725
    (Epoch 6 / 10) train acc: 0.648000; val_acc: 0.295000
    (Iteration 121 / 200) loss: 0.986068
    (Epoch 7 / 10) train acc: 0.673000; val_acc: 0.322000
    (Iteration 141 / 200) loss: 1.123718
    (Epoch 8 / 10) train acc: 0.723000; val_acc: 0.318000
    (Iteration 161 / 200) loss: 0.785963
    (Epoch 9 / 10) train acc: 0.754000; val_acc: 0.308000
    (Iteration 181 / 200) loss: 0.983303
    (Epoch 10 / 10) train acc: 0.799000; val_acc: 0.309000

    Solver without batch norm:
    (Iteration 1 / 200) loss: 2.302857
    (Epoch 0 / 10) train acc: 0.132000; val_acc: 0.144000
    (Epoch 1 / 10) train acc: 0.191000; val_acc: 0.179000
    (Iteration 21 / 200) loss: 2.030069
    (Epoch 2 / 10) train acc: 0.298000; val_acc: 0.254000
    (Iteration 41 / 200) loss: 1.907353
    (Epoch 3 / 10) train acc: 0.329000; val_acc: 0.257000
    (Iteration 61 / 200) loss: 1.711589
    (Epoch 4 / 10) train acc: 0.402000; val_acc: 0.300000
    (Iteration 81 / 200) loss: 1.645850
    (Epoch 5 / 10) train acc: 0.436000; val_acc: 0.314000
    (Iteration 101 / 200) loss: 1.639541
    (Epoch 6 / 10) train acc: 0.497000; val_acc: 0.308000
    (Iteration 121 / 200) loss: 1.471674
    (Epoch 7 / 10) train acc: 0.571000; val_acc: 0.330000
    (Iteration 141 / 200) loss: 1.113221
    (Epoch 8 / 10) train acc: 0.584000; val_acc: 0.322000
    (Iteration 161 / 200) loss: 1.362420
    (Epoch 9 / 10) train acc: 0.645000; val_acc: 0.330000
    (Iteration 181 / 200) loss: 0.930322
    (Epoch 10 / 10) train acc: 0.679000; val_acc: 0.325000

Run the following to visualize the results from two networks trained above. You should find that using batch normalization helps the network to converge much faster.

```python
def plot_training_history(title, label, baseline, bn_solvers, plot_fn, bl_marker='.', bn_marker='.', labels=None):
    """utility function for plotting training history"""
    plt.title(title)
    plt.xlabel(label)
    bn_plots = [plot_fn(bn_solver) for bn_solver in bn_solvers]
    bl_plot = plot_fn(baseline)
    num_bn = len(bn_plots)
    for i in range(num_bn):
        label='with_norm'
        if labels is not None:
            label += str(labels[i])
        plt.plot(bn_plots[i], bn_marker, label=label)
    label='baseline'
    if labels is not None:
        label += str(labels[0])
    plt.plot(bl_plot, bl_marker, label=label)
    plt.legend(loc='lower center', ncol=num_bn+1)

plt.subplot(3, 1, 1)
plot_training_history('Training loss','Iteration', solver, [bn_solver], \
                      lambda x: x.loss_history, bl_marker='o', bn_marker='o')
plt.subplot(3, 1, 2)
plot_training_history('Training accuracy','Epoch', solver, [bn_solver], \
                      lambda x: x.train_acc_history, bl_marker='-o', bn_marker='-o')
plt.subplot(3, 1, 3)
plot_training_history('Validation accuracy','Epoch', solver, [bn_solver], \
                      lambda x: x.val_acc_history, bl_marker='-o', bn_marker='-o')

plt.gcf().set_size_inches(15, 15)
plt.show()
```

# Batch normalization and initialization
We will now run a small experiment to study the interaction of batch normalization and weight initialization.

The first cell will train 8-layer networks both with and without batch normalization using different scales for weight initialization. The second layer will plot training accuracy, validation set accuracy, and training loss as a function of the weight initialization scale.

```python
np.random.seed(220)
# Try training a very deep net with batchnorm
hidden_dims = [50, 50, 50, 50, 50, 50, 50]
num_train = 1000
small_data = {
  'X_train': data['X_train'][:num_train],
  'y_train': data['y_train'][:num_train],
  'X_val': data['X_val'],
  'y_val': data['y_val'],
}

bn_solvers_ws = {}
solvers_ws = {}
weight_scales = np.logspace(-4, 0, num=20)
for i, weight_scale in enumerate(weight_scales):
  print('Running weight scale %d / %d' % (i + 1, len(weight_scales)))
  bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization='batchnorm')
  model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)

  bn_solver = Solver(bn_model, small_data,
                  num_epochs=10, batch_size=50,
                  update_rule='adam',
                  optim_config={
                    'learning_rate': 1e-3,
                  },
                  verbose=False, print_every=200)
  bn_solver.train()
  bn_solvers_ws[weight_scale] = bn_solver

  solver = Solver(model, small_data,
                  num_epochs=10, batch_size=50,
                  update_rule='adam',
                  optim_config={
                    'learning_rate': 1e-3,
                  },
                  verbose=False, print_every=200)
  solver.train()
  solvers_ws[weight_scale] = solver
```

    Running weight scale 1 / 20
    Running weight scale 2 / 20
    Running weight scale 3 / 20
    Running weight scale 4 / 20
    Running weight scale 5 / 20
    Running weight scale 6 / 20
    Running weight scale 7 / 20
    Running weight scale 8 / 20
    Running weight scale 9 / 20
    Running weight scale 10 / 20
    Running weight scale 11 / 20
    Running weight scale 12 / 20
    Running weight scale 13 / 20
    Running weight scale 14 / 20
    Running weight scale 15 / 20
    Running weight scale 16 / 20
    Running weight scale 17 / 20
    Running weight scale 18 / 20
    Running weight scale 19 / 20
    Running weight scale 20 / 20

```python
# Plot results of weight scale experiment
best_train_accs, bn_best_train_accs = [], []
best_val_accs, bn_best_val_accs = [], []
final_train_loss, bn_final_train_loss = [], []

for ws in weight_scales:
  best_train_accs.append(max(solvers_ws[ws].train_acc_history))
  bn_best_train_accs.append(max(bn_solvers_ws[ws].train_acc_history))

  best_val_accs.append(max(solvers_ws[ws].val_acc_history))
  bn_best_val_accs.append(max(bn_solvers_ws[ws].val_acc_history))

  final_train_loss.append(np.mean(solvers_ws[ws].loss_history[-100:]))
  bn_final_train_loss.append(np.mean(bn_solvers_ws[ws].loss_history[-100:]))

plt.subplot(3, 1, 1)
plt.title('Best val accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best val accuracy')
plt.semilogx(weight_scales, best_val_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_val_accs, '-o', label='batchnorm')
plt.legend(ncol=2, loc='lower right')

plt.subplot(3, 1, 2)
plt.title('Best train accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best training accuracy')
plt.semilogx(weight_scales, best_train_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_train_accs, '-o', label='batchnorm')
plt.legend()

plt.subplot(3, 1, 3)
plt.title('Final training loss vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Final training loss')
plt.semilogx(weight_scales, final_train_loss, '-o', label='baseline')
plt.semilogx(weight_scales, bn_final_train_loss, '-o', label='batchnorm')
plt.legend()
plt.gca().set_ylim(1.0, 3.5)

plt.gcf().set_size_inches(15, 15)
plt.show()
```

## Inline Question 1:
Describe the results of this experiment. How does the scale of weight initialization affect models with/without batch normalization differently, and why?

## Answer:

As evident from First graph, without batchnorm, the model is very sensitive to weight scale initialization. Also, we can see overfitting without batchnorm on comparing graph 1 and graph 2.

From graph 3, we see the problem of vanishing gradients without batch norm which is a common issue with larger weight scale initialization.

# Batch normalization and batch size
We will now run a small experiment to study the interaction of batch normalization and batch size.

The first cell will train 6-layer networks both with and without batch normalization using different batch sizes. The second layer will plot training accuracy and validation set accuracy over time.

```python
def run_batchsize_experiments(normalization_mode):
    np.random.seed(220)
    # Try training a very deep net with batchnorm
    hidden_dims = [100, 100, 100, 100, 100]
    num_train = 1000
    small_data = {
      'X_train': data['X_train'][:num_train],
      'y_train': data['y_train'][:num_train],
      'X_val': data['X_val'],
      'y_val': data['y_val'],
    }
    n_epochs=10
    weight_scale = 2e-2
    batch_sizes = [5,10,50]
    lr = 10**(-3.5)
    solver_bsize = batch_sizes[0]

    print('No normalization: batch size = ',solver_bsize)
    model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=None)
    solver = Solver(model, small_data,
                    num_epochs=n_epochs, batch_size=solver_bsize,
                    update_rule='adam',
                    optim_config={
                      'learning_rate': lr,
                    },
                    verbose=True, print_every=1000)
    solver.train()

    bn_solvers = []
    for i in range(len(batch_sizes)):
        b_size=batch_sizes[i]
        print('Normalization: batch size = ',b_size)
        bn_model = FullyConnectedNet(hidden_dims, weight_scale=weight_scale, normalization=normalization_mode)
        bn_solver = Solver(bn_model, small_data,
                        num_epochs=n_epochs, batch_size=b_size,
                        update_rule='adam',
                        optim_config={
                          'learning_rate': lr,
                        },
                        verbose=True, print_every=1000)
        bn_solver.train()
        bn_solvers.append(bn_solver)

    return bn_solvers, solver, batch_sizes

batch_sizes = [5,10,50]
bn_solvers_bsize, solver_bsize, batch_sizes = run_batchsize_experiments('batchnorm')
```

    No normalization: batch size =  5
    (Iteration 1 / 2000) loss: 2.300857
    (Epoch 0 / 10) train acc: 0.131000; val_acc: 0.130000
    (Epoch 1 / 10) train acc: 0.242000; val_acc: 0.228000
    (Epoch 2 / 10) train acc: 0.318000; val_acc: 0.270000
    (Epoch 3 / 10) train acc: 0.374000; val_acc: 0.313000
    (Epoch 4 / 10) train acc: 0.398000; val_acc: 0.265000
    (Epoch 5 / 10) train acc: 0.460000; val_acc: 0.298000
    (Iteration 1001 / 2000) loss: 1.691890
    (Epoch 6 / 10) train acc: 0.490000; val_acc: 0.306000
    (Epoch 7 / 10) train acc: 0.534000; val_acc: 0.317000
    (Epoch 8 / 10) train acc: 0.579000; val_acc: 0.322000
    (Epoch 9 / 10) train acc: 0.637000; val_acc: 0.316000
    (Epoch 10 / 10) train acc: 0.675000; val_acc: 0.325000
    Normalization: batch size =  5
    (Iteration 1 / 2000) loss: 2.326095
    (Epoch 0 / 10) train acc: 0.095000; val_acc: 0.098000
    (Epoch 1 / 10) train acc: 0.250000; val_acc: 0.226000
    (Epoch 2 / 10) train acc: 0.323000; val_acc: 0.279000
    (Epoch 3 / 10) train acc: 0.314000; val_acc: 0.239000
    (Epoch 4 / 10) train acc: 0.344000; val_acc: 0.292000
    (Epoch 5 / 10) train acc: 0.361000; val_acc: 0.297000
    (Iteration 1001 / 2000) loss: 1.898464
    (Epoch 6 / 10) train acc: 0.379000; val_acc: 0.273000
    (Epoch 7 / 10) train acc: 0.408000; val_acc: 0.320000
    (Epoch 8 / 10) train acc: 0.405000; val_acc: 0.278000
    (Epoch 9 / 10) train acc: 0.394000; val_acc: 0.277000
    (Epoch 10 / 10) train acc: 0.420000; val_acc: 0.289000
    Normalization: batch size =  10
    (Iteration 1 / 1000) loss: 2.357343
    (Epoch 0 / 10) train acc: 0.103000; val_acc: 0.096000
    (Epoch 1 / 10) train acc: 0.320000; val_acc: 0.246000
    (Epoch 2 / 10) train acc: 0.365000; val_acc: 0.289000
    (Epoch 3 / 10) train acc: 0.433000; val_acc: 0.313000
    (Epoch 4 / 10) train acc: 0.445000; val_acc: 0.320000
    (Epoch 5 / 10) train acc: 0.457000; val_acc: 0.324000
    (Epoch 6 / 10) train acc: 0.525000; val_acc: 0.306000
    (Epoch 7 / 10) train acc: 0.578000; val_acc: 0.330000
    (Epoch 8 / 10) train acc: 0.581000; val_acc: 0.321000
    (Epoch 9 / 10) train acc: 0.630000; val_acc: 0.338000
    (Epoch 10 / 10) train acc: 0.636000; val_acc: 0.310000
    Normalization: batch size =  50
    (Iteration 1 / 200) loss: 2.347591
    (Epoch 0 / 10) train acc: 0.115000; val_acc: 0.111000
    (Epoch 1 / 10) train acc: 0.288000; val_acc: 0.230000
    (Epoch 2 / 10) train acc: 0.418000; val_acc: 0.301000
    (Epoch 3 / 10) train acc: 0.482000; val_acc: 0.317000
    (Epoch 4 / 10) train acc: 0.520000; val_acc: 0.323000
    (Epoch 5 / 10) train acc: 0.607000; val_acc: 0.338000
    (Epoch 6 / 10) train acc: 0.664000; val_acc: 0.349000
    (Epoch 7 / 10) train acc: 0.729000; val_acc: 0.335000
    (Epoch 8 / 10) train acc: 0.786000; val_acc: 0.349000
    (Epoch 9 / 10) train acc: 0.798000; val_acc: 0.345000
    (Epoch 10 / 10) train acc: 0.867000; val_acc: 0.318000

```python
plt.subplot(2, 1, 1)
plot_training_history('Training accuracy (Batch Normalization)','Epoch', solver_bsize, bn_solvers_bsize, \
                      lambda x: x.train_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)
plt.subplot(2, 1, 2)
plot_training_history('Validation accuracy (Batch Normalization)','Epoch', solver_bsize, bn_solvers_bsize, \
                      lambda x: x.val_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)

plt.gcf().set_size_inches(15, 10)
plt.show()
```

## Inline Question 2:
Describe the results of this experiment. What does this imply about the relationship between batch normalization and batch size? Why is this relationship observed?

## Answer:

We are trying to estimate mean and variance for whole dataset based on a subset of the dataset. Hence a larger subset will statistically be a best approximation of the population compared to a smaller dataset. Law of large numbers.

# Layer Normalization
Batch normalization has proved to be effective in making networks easier to train, but the dependency on batch size makes it less useful in complex networks which have a cap on the input batch size due to hardware limitations.

Several alternatives to batch normalization have been proposed to mitigate this problem; one such technique is Layer Normalization [2]. Instead of normalizing over the batch, we normalize over the features. In other words, when using Layer Normalization, each feature vector corresponding to a single datapoint is normalized based on the sum of all terms within that feature vector.

[2] [Ba, Jimmy Lei, Jamie Ryan Kiros, and Geoffrey E. Hinton. "Layer Normalization." stat 1050 (2016): 21.](https://arxiv.org/pdf/1607.06450.pdf)

## Inline Question 3:
Which of these data preprocessing steps is analogous to batch normalization, and which is analogous to layer normalization?

1. Scaling each image in the dataset, so that the RGBD channels for each row of pixels within an image sums up to 1.
2. Scaling each image in the dataset, so that the RGBD channels for all pixels within an image sums up to 1.
3. Subtracting the mean image of the dataset from each image in the dataset.
4. Setting all RGBD values to either 0 or 1 depending on a given threshold.

## Answer:

(3) is similar to Batch Normalization

(2) is similar to Layernormalization

# Layer Normalization: Implementation

Now you'll implement layer normalization. This step should be relatively straightforward, as conceptually the implementation is almost identical to that of batch normalization. One significant difference though is that for layer normalization, we do not keep track of the moving moments, and the testing phase is identical to the training phase, where the mean and variance are directly calculated per datapoint.

Here's what you need to do:

* In `ie590/layers.py`, implement the forward pass for layer normalization in the function `layernorm_backward`.

Run the cell below to check your results.
* In `ie590/layers.py`, implement the backward pass for layer normalization in the function `layernorm_backward`.

Run the second cell below to check your results.
* Modify `ie590/classifiers/fc_net.py` to add layer normalization to the `FullyConnectedNet`. When the `normalization` flag is set to `"layernorm"` in the constructor, you should insert a layer normalization layer before each ReLU nonlinearity.

Run the third cell below to run the batch size experiment on layer normalization.

```python
# Check the training-time forward pass by checking means and variances
# of features both before and after layer normalization

# Simulate the forward pass for a two-layer network
np.random.seed(220)
N, D1, D2, D3 =4, 50, 60, 3
X = np.random.randn(N, D1)
W1 = np.random.randn(D1, D2)
W2 = np.random.randn(D2, D3)
a = np.maximum(0, X.dot(W1)).dot(W2)

print('Before layer normalization:')
print_mean_std(a,axis=1)

gamma = np.ones(D3)
beta = np.zeros(D3)
# Means should be close to zero and stds close to one
print('After layer normalization (gamma=1, beta=0)')
a_norm, _ = layernorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=1)

gamma = np.asarray([3.0,3.0,3.0])
beta = np.asarray([5.0,5.0,5.0])
# Now means should be close to beta and stds close to gamma
print('After layer normalization (gamma=', gamma, ', beta=', beta, ')')
a_norm, _ = layernorm_forward(a, gamma, beta, {'mode': 'train'})
print_mean_std(a_norm,axis=1)
```

    Before layer normalization:
      means:  [34.05612999 30.32922803 38.20844387 77.27961435]
      stds:   [ 7.7622382  32.9930656  19.4537999  38.93389575]

    After layer normalization (gamma=1, beta=0)
      means:  [ 4.81096644e-16 -7.51713506e-17 -4.62592927e-17  1.11022302e-16]
      stds:   [0.99999992 1.         0.99999999 1.        ]

    After layer normalization (gamma= [3. 3. 3.] , beta= [5. 5. 5.] )
      means:  [5. 5. 5. 5.]
      stds:   [2.99999975 2.99999999 2.99999996 2.99999999]

```python
# %%pixie_debugger
# Gradient check batchnorm backward pass
np.random.seed(220)
N, D = 4, 5
x = 5 * np.random.randn(N, D) + 12
gamma = np.random.randn(D)
beta = np.random.randn(D)
dout = np.random.randn(N, D)

ln_param = {}
fx = lambda x: layernorm_forward(x, gamma, beta, ln_param)[0]
fg = lambda a: layernorm_forward(x, a, beta, ln_param)[0]
fb = lambda b: layernorm_forward(x, gamma, b, ln_param)[0]

dx_num = eval_numerical_gradient_array(fx, x, dout)
da_num = eval_numerical_gradient_array(fg, gamma.copy(), dout)
db_num = eval_numerical_gradient_array(fb, beta.copy(), dout)

_, cache = layernorm_forward(x, gamma, beta, ln_param)

dx, dgamma, dbeta = layernorm_backward(dout, cache)

#You should expect to see relative errors between 1e-12 and 1e-8
print('dx error: ', rel_error(dx_num, dx))
print('dgamma error: ', rel_error(da_num, dgamma))
print('dbeta error: ', rel_error(db_num, dbeta))
```

    dx error:  2.1492647163467308e-09
    dgamma error:  5.918194116037158e-12
    dbeta error:  3.276531879937969e-12

# Layer Normalization and batch size

We will now run the previous batch size experiment with layer normalization instead of batch normalization. Compared to the previous experiment, you should see a markedly smaller influence of batch size on the training history!

```python
ln_solvers_bsize, solver_bsize, batch_sizes = run_batchsize_experiments('layernorm')

plt.subplot(2, 1, 1)
plot_training_history('Training accuracy (Layer Normalization)','Epoch', solver_bsize, ln_solvers_bsize, \
                      lambda x: x.train_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)
plt.subplot(2, 1, 2)
plot_training_history('Validation accuracy (Layer Normalization)','Epoch', solver_bsize, ln_solvers_bsize, \
                      lambda x: x.val_acc_history, bl_marker='-^', bn_marker='-o', labels=batch_sizes)

plt.gcf().set_size_inches(15, 10)
plt.show()
```

## Inline Question 4:
When is layer normalization likely to not work well, and why?

1. Using it in a very deep network
2. Having a very small dimension of features
3. Having a high regularization term

## Answer:

It won't work well with deeper layers as it may led to values being to close to each other in deeper layers which usually leads to no improvement in accuracy.


# **MAC023 - Mecânica das Estruturas**

# ME-02 - Segunda Avaliação de Conhecimentos

Alunos:

Brian Luis Coimbra Maia

Mathews Edwirds Gomes Almeida

# Condições Gerais

Esta avaliação tem como objetivo avaliar os conhecimentos adquiridos na primeira parte da disciplina de Mecânica das Estruturas.

---

As condicões abaixo devem ser observadas:

1. Serão formadas equipes e cada uma delas com no mínimo **2** e no máximo **3** integrantes.

2. A avaliação será realizada por meio da entrega de uma cópia deste notebook com as soluções desenvolvidas até a data estipulada de entrega.

3. Da entrega da avaliação.
  * Os documentos necessários para a entrega do trabalho são (1) os códigos  desenvolvidos pela equipe e (2) vídeo com a descrição da solução.
  * A equipe deve usar este modelo de notebook para desenvolver os códigos.
  * Os códigos podem ser desenvolvidos combinado a linguagem LaTeX e computação simbólica ou computação numérica quando necessário.
  * Os gráficos necessários para a apresentação da solução devem estar embutidos no notebook.

4. Da distribuição das questões.
  * A quantidade de questões será a mesma para cada grupo.
  * Serão atribuídas as mesmas questões para todos os grupos.
  * A pontuacão referente a cada questão será igualitária e o valor total da avaliação será 100 pontos.

5. As equipes devem ser formadas até às **23:59 horas o dia 18/01/2022** por meio do preenchimento da planilha [[MAC023] Formação das Equipes](https://docs.google.com/spreadsheets/d/1Dlftymao970nnrE4mu958iP8nMqKqSuhHiiLH91BKpQ/edit#gid=153704268).

6. A formação das equipes pode ser acompanhada arquivo indicado acima. Cada equipe será indentificada por uma letra em ordem alfabética seguida do número 2 (A2, B2, C2, e assim por diante). O arquivo está aberto para edição e pode ser alterado pelos alunos até a data estipulada.

7. Equipes formadas após a data estabelecida para a formação das equipes terão a nota da avaliação multiplicada por um coeficiente de **0.80**.

8. A equipe deve indicar no arquivo de indicação de equipes um responsável pela entrega do projeto.
  * Somente o responsável pela entrega deve fazer o upload do arquivo na plataforma

9. A entrega dos projetos deve ocorrer até às **23:59 do dia 04/02/2021** na plataforma da disciplina pelo responsável pela entrega.
  * Caso a entrega seja feita por outro integrante diferente daquele indicado pela pela equipe a avaliação será desconsiderada e não será corrigida até que a a condição de entrega seja satisfeita.

10. Quaisquer dúvidas ou esclarecimentos devem ser encaminhadas pela sala de aula virtual.

## (Q1) Encontre  para a estrutura abaixo: (a) a deflexão máxima e (b) o ângulo máximo em relação a horizontal.

---

Dado que, uma simples viga AB é sujeita a uma carga distribuída de intensidade $w(x) = w_0 * sin(\frac{Πx}{L})$, onde $w_0$ é a intensidade máxima da carga.

Vamos integrar sucessivamente a equação de w(x) a fim de encontrar as devidas equações de deflexão, *slope*, momento, cortante etc.

$EIv^{(iv)} = -w_0 * sin(\frac{πx}{L}) \\
EIv''' = w_0 * \frac{L}{π} * cos(\frac{πx}{L}) + c1 \\
EIv'' = w_0 * (\frac{L}{π})^2 * sin(\frac{πx}{L}) + c1*x + c2 \\
EIv' = - w_0 * (\frac{L}{π})^3 * cos(\frac{πx}{L}) + c1*(\frac{x^2}{2}) + c2*x + c3 \\
EIv = - w_0 * (\frac{L}{π})^4 * sin(\frac{πx}{L}) + c1*(\frac{x^3}{6}) + c2*(\frac{x^2}{2}) + c3*x + c4$ \\

Igualando as constantes de integração à zero:

$C1 = C2 = C3 = C4 = 0 ⇒ \\
EIv = - w_0 * (\frac{L}{π})^4 * sin(\frac{πx}{L})$

a) Assim, a deflexão máxima foi encontrada pela integração do esforço cortante e equações de carga:

$v = - \frac{w_o * L^4}{π^4*EI} * sin(\frac{πx}{L})$

ou, considerando que a deflexão máxima ocorre no centro da viga ($\frac{L}{2}$), temos:

$v = - \frac{w_o * L^4}{π^4*EI} * sin(\frac{πx}{L}) ⇒ \\
v(\frac{L}{2}) = - \frac{w_o * L^4}{π^4*EI} * sin(\frac{π*L}{L*2}) ⇒ \\
v(\frac{L}{2}) = - \frac{w_o * L^4}{π^4*EI} * sin(\frac{π}{2}) ⇒ \\
v(\frac{L}{2}) = - \frac{w_o * L^4}{π^4*EI} * 1
$

Assim, temos a deflexão máxima:

$δ_{max} = v(\frac{L}{2}) = + \frac{w_0 * L^4}{π^4 * EI}$

b) Para encontrar o angulo máximo de rotação em relação ao eixo horizontal, basta substituir os devidos valores da viga na equação da deflexão (encontrada logo acima) para os apoios A($0$) e B($L$):

Para o apoio A: \\
$θ_A (0) = v' = - \frac{w_0 * L^3}{π^3 * EI} * cos(\frac{π0}{L}) ⇒ \\
θ_A (0) = - \frac{w_0 * L^3}{π^3 * EI} * cos(0) ⇒ \\
θ_A (0) = - \frac{w_0 * L^3}{π^3 * EI} * 1
$

Para o apoio B:

$θ_B (L) = v' = - \frac{w_0 * L^3}{π^3 * EI} * cos(\frac{πL}{L}) ⇒ \\
θ_B (L) = v' = - \frac{w_0 * L^3}{π^3 * EI} * cos(π) ⇒ \\
θ_B (L) = v' = - \frac{w_0 * L^3}{π^3 * EI} * (-1) ⇒ \\
θ_B (L) = v' = + \frac{w_0 * L^3}{π^3 * EI}$

## (Q2) Determina a razão $P/Q$ para que  (a) o ângulo em $C$ seja zero e (b) a deflexão em $B$ seja nula
---

Primeiro determinamos o momento fletor e as derivadas parciais para o seguimento BC:

$ M_{BC} = - Px ,\textrm{ sendo }(0 ≤ x ≤ L) $ \\
$\frac{∂M_{BC}}{∂Q} = 0$ \\
$\frac{∂M_{BC}}{∂P} = -x$ \\

Em segundo lugar, determinamos o momento fletor e as derivadas parciais para o seguimento AB:

$ M_{AB} = Q(L - x) - Px ,\textrm{ sendo }(0 ≤ x ≤ L) $ \\
$\frac{∂M_{AB}}{∂Q} = L - x $ \\
$\frac{∂M_{AB}}{∂P} = -x $ \\

a) Primeiro calculamos a deflexão em C:

$δ_C = \frac{1}{EI} \displaystyle \int_{0}^{L} (M_{BC})(\frac{∂M_{BC}}{∂P})dx + \frac{1}{EI}
\displaystyle \int_{0}^{L} (M_{AB})(\frac{∂M_{AB}}{∂P})dx$ \\

$δ_C = \frac{1}{EI} \displaystyle \int_{0}^{L} (-Px)(-x)dx + \frac{1}{EI}
\displaystyle \int_{0}^{L} (Q*(L - x) - Px)(-x)dx$ \\

$δ_C = L^3 * (\frac{2 P}{3 EI} - \frac{Q}{6 EI})  = \frac{L^3 * (4 P - Q)}{6 EI}$

Então derivamos a deflexão para encontrar o angulo em C: \\
$δ_C' = θ_C ⇒ \frac{\partial f}{\partial L} \frac{L^3 (4 P - Q)}{6 EI} = \frac{(4 P - Q) L^2}{2 EI}$ \\
$θ_C = \frac{L^2 * (4 P - Q)}{2 EI}$

Sendo assim, para encontrar a razão $\frac{P}{Q}$ em C, precisaremos igualar $θ_C$ a 0, então:

$θ_C = \frac{L^2 * (4 P - Q)}{2 EI} = 0$ \\
$(4 P - Q) = 0$ \\
$4 P = Q$ \\
$\frac{P}{Q} = \frac{1}{4} \textrm{ ou } 0.25$

b) Agora calculamos a deflexão em B:

$δ_B = \frac{1}{EI} \displaystyle \int_{0}^{L} (M_{AB})(\frac{∂M_{AB}}{∂Q})dx$

$δ_B = \frac{1}{EI} \displaystyle \int_{0}^{L} (Q*(L - x) - Px)(L-x)dx$

$δ_B = \frac{L^3 (2Q - P)}{6 EI}$

Sendo assim, para encontrar a razão $\frac{P}{Q}$, vamos igualar a deflexão em B a 0 ($δ_B = 0$), faremos: \\
$δ_B = \frac{L^3 (2Q - P)}{6 EI} = 0$ \\
$(2 Q - P) = 0$ \\
$2Q = P $ \\
$\frac{P}{Q} = 2$

## (Q3) Determine o deslocamento no meio do vão

---

```
# Importação das bibliotecas e inicialização de parâmetros
import sympy as sp
from sympy import N

sp.init_printing(use_unicode=False, wrap_line=False, no_global=True)

x, l, w, E, I, a, b = sp.var('x L w E I a b', real=True, positive=True)

# Separando as equações do momento fletor real por trechos
MR = [(w/8)*(3*l*x - 4*x**2), (w/8)*(l**2-l*x)] #AC e CB
print("Equações do momento fletor real por trechos:")
display(MR)

# Separando as equações do momento fletor virtual por trechos
MV = [x/2, x/2-(x-l/2)]
print("\n")
print("Equações do momento fletor virtual por trechos:")
display(MV)

# Integrando em cada trecho
delta1 = sp.integrate(MR[0]*MV[0]/(2*E*I),(x,0,l/2))
delta2 = sp.integrate(MR[1]*MV[1]/(E*I),(x,l/2,l))

print("\n")
print("O deslocamento no meio do vão é dado pela soma dos resultados das duas integrações:")
delta1+delta2
```

Outra solução pode ser pela separação da figura em dois trechos AC e CB, seguida pela montagem de suas deflexões:

$y_{AC} = - \frac{wx}{384*2*EI}*(9L^3 -24Lx^2 + 16x^3)$ \\

$y_{CB} = - \frac{wx}{384*EI}*(8x^3 -24Lx^2 + 17L^2x - L^3)$ \\

Derivando essas equações, encontramos os *slopes* para os pontos AC, CB:

$Θ_{AC} = - \frac{w}{384*2*EI}*(9L^3 -72Lx^2 + 64x^3)$ \\

$Θ_{CB} = - \frac{wL}{384*EI}*(24x^2 -48Lx + 17L^2)$ \\

Assim, basta substituirmos a posição (L) na equação $Θ_{CB}$, para encontrarmos o deslocamento no meio do vão.

$Θ_{B} (x)= - \frac{wL}{384*EI}*(24x^2 -48Lx + 17L^2)$ \\
$Θ_{B} (L)= \frac{7wL^4}{1536*EI}$ \\

## (Q4) Encontre uma expressão para (a) o deslocamento **vertical** no ponto de aplicação da carga **que está aplicada a um terço do comprimento horizontal a partir do ponto $B$** e (b) o deslocamento horizontal do ponto $C$

---

##A)

```
#-- separando as equações do momento fletor real por trechos
MR=[
    0,                   # trecho AB, origem em A
    w*x*(2/3)/l,         # trecho BW, origem em B
    w*(1/3)*(l-x)/l,     # trecho WC, origem em W
    0,                   # trecho CD, origem em C
]

# Separando as equações do momento fletor virtual por trechos
MV=[
    x,                   # trecho AB, origem em A
    x,                   # trecho BW, origem em B
    x,                   # trecho WC, origem em W
    x,                   # trecho CD, origem em C
]

# Integrando em cada trecho
delta1 = sp.integrate(MR[0]*MV[0]/(E*I),(x,0,l))
delta2 = sp.integrate(MR[1]*MV[1]/(E*I),(x,0,l/3))

print("O deslocamento no ponto de aplicação da carga é dado pela soma dos resultados das integrações envolvendo AB e BW:")
desloc_W = delta1+delta2

N(desloc_W, 4)
```

##B)

```
delta1 = sp.integrate(MR[0]*MV[0]/(E*I),(x,0,l))
delta2 = sp.integrate(MR[1]*MV[1]/(E*I),(x,0,l/3))
delta3 = sp.integrate(MR[2]*MV[2]/(E*I),(x,l/3,l))

print("O deslocamento horizontal do ponto C é dado pela soma dos resultados das integrações envolvendo AB, BW e WC:")
desloc_C = delta1+delta2+delta3

N(desloc_C, 4)
```

## (Q5)  Determine o deslocamento vertical na extremidade livre da viga em balanço. Observe que a barra possui uma variação linear na sua altura que influencia as propriedades geométricas. Modifique a formulação do método da carga unitária para acomodar a variação da altura ao longo do comprimento.

Dados:

tf - tonelada-força

```
print("Primeiro equacionamos os momentos reais de cada uma das barras da figura")
MR = 2*x - 2*l
display(MR)

print("Agora equacionamos os momentos virtuais em cada uma das barra da figura")
MV = x - l
display(MV)

print("Assim temos:")
I = -1.575E-3*x/l + 1.8E-3
display(I)
```

```
delta_B = sp.integrate(MR*MV/(E*I),(x,0,l))
display(delta_B)
```

Formula dos Trapézios (Δx=L/m):

$\int_{x_0}^{x_n} y(x) dx ≈ (\frac{Δx}{2})(y_0+2y_1+2y_2+...+2y_{m-1}+y_m)$

Fórmula de Simpson (Δx=L/2m):

$\int_{x_0}^{x_n} y(x)dx ≈ (\frac{Δx}{3})(y_0+4y_1+2y_2+4y_3+...+4y_{2m-3}+2y_{2m-2}+4y_{2m-1}+y_{2m})$

Fórmula de Simpson (Δx=L/3):

$\int_{x_0}^{x_n} y(x)dx ≈ (\frac{3Δx}{8})(y_0+3y_1+3y_2+2y_3+...+2y_{3m-3}+3y_{3m-2}+3y_{3m-1}+y_{3m})$

```
# Importação das bibliotecas e inicialização de parâmetros
import pandas as pd

# Definindo os dados iniciais
h1, h2, L, b = 0.3, 0.6, 6, 0.1 #m
P = 2       #tf
E = 2100000 #tf/m^2

divisoes = ['6 (Δx=1)', '12 (Δx=1/2)', '18 (Δx=1/3)', '24 (Δx=1/4)']
trapezios = [6.2421, 6.2333, 6.2320, 6.2314]
simpson_l_2 = [6.2365, 6.2310, 6.2307, 6.2306]
simpson_l_3 = [6.2420, 6.2315, 6.2308, 6.2307]

print("A partir dos valores dados, podemos encontrar os deslocamentos verticais dos pontos para diferentes divisões da viga, usando as formulas dos Trápezios (Δx=L/m), Simpson(Δx=L/2m) e Simpson(Δx=l/3m)\n")
d = {'Intervalos': divisoes, 'Trápezios (Δx=L/m)': trapezios, 'Simpson(Δx=L/2m)':simpson_l_2, 'Simpson(Δx=l/3m)':simpson_l_3}
df = pd.DataFrame(data=d)
print(df.to_string(index=False))
```

    A partir dos valores dados, podemos encontrar os deslocamentos verticais dos pontos para diferentes divisões da viga, usando as formulas dos Trápezios (Δx=L/m), Simpson(Δx=L/2m) e Simpson(Δx=l/3m)

     Intervalos  Trápezios (Δx=L/m)  Simpson(Δx=L/2m)  Simpson(Δx=l/3m)
       6 (Δx=1)              6.2421            6.2365            6.2420
    12 (Δx=1/2)              6.2333            6.2310            6.2315
    18 (Δx=1/3)              6.2320            6.2307            6.2308
    24 (Δx=1/4)              6.2314            6.2306            6.2307


---
layout: post
title:  "Bayesian Sample Selection Effects"
desc: "Sample bias and selection effects are the worst. Here's one solution."
date: 2019-07-30
categories: [tutorial]
tags: [bayesian]
loc: 'tutorials/sampleselectionbias/'
permalink: /tutorials/sample_selection
math: true
---

!!!replace

In a perfect world our experiments would capture all the data that exists. This is not a perfect world, and we miss a lot of data. Let's consider one method of accounting for this in a Bayesian formalism - integrating it out.

Let's begin with a motivational dataset.

```python
# Remove
import matplotlib.pyplot as plt
from base import *
from cycler import cycler
# plt.rcParams['axes.prop_cycle'] = (cycler(color=['#56d870', '#f9ee4a', '#44d9ff', '#f95b4a', '#3d9fe2', '#ffa847', '#c4ef7a', '#e195e2', '#ced9ed', '#fff29b']) + cycler(linestyle=['-', '--', ':', '-.', '-', '--', ':', '-.', '-', '--']))
#plt.rcParams['axes.prop_cycle'] = (cycler(color=['#003049', '#D62828', '#F77F00', '#FCBF49', '#EAE2B7']) + cycler(linestyle=['-', '--', ':', '-.', '-']))
```

```python
import matplotlib.pyplot as plt
import numpy as np
n = 10000
alpha = 85
mu, sigma = 100, 10
x = np.random.normal(mu, sigma, size=n)
mask = x > alpha
x_good = x[mask]

fig, ax = plt.subplots(figsize=(7,4))
ax.hist(x_good, label="Observed data", color='#F77F00', alpha=0.3, density=True)
ax.hist(x_good, histtype="step", color='#F77F00', linewidth=1.5, density=True)

xs = np.linspace(70, 140, 1000)
ax.axvline(alpha, ls="--", lw=1.0, label="Instrument cutoff", color='#D62828')
from scipy.stats import norm
ax.plot(xs, norm(mu, sigma).pdf(xs), label="All data", lw=1.8)
leg = ax.legend(frameon=False, loc=1)
for lh in leg.legendHandles:
    lh.set_alpha(1)
ax.set_xlabel("x")
ax.set_ylabel("Probability");
```

!!!main

So it looks like for our example data, we've got some gaussian-like distribution of $x$ observations, but at some point it seems like our instrument is unable to pick up the observations. Maybe its brightness and its too dim! Or maybe something else, who knows! But regardless, we can work with this.

To start at the beginning, let's write out the full formula for our posterior:

$$ P(\theta|d) = \frac{P(d|\theta) P(\theta)}{P(d)} $$

Everything looks good, so let's home in here on the likelihood. Now, what we should also do to make our lives easier if we write out the fact we have some sort of *selection effect* applying to our model. That is, some separate probability that dictates our experiment successfully observes an event which actually happened.

$$ P(d|\theta) \rightarrow P(d|\theta, S), $$

where $S$ in colloquial English represents "we successfully observed the event". Now, we can normally write our selection probability given data or model easily, so we want to get things into a state where we have $P(S\|d,\theta)$. Via some probability manipulation we can twist this around and get the following:

$$ P(d|\theta, S) = \frac{P(S|d,\theta) P(d|\theta)}{P(S|\theta)} $$

So let's break that down. Our likelihood given our model and our selection effects is given by the chance we observed our experimenal data (which is going to be $1$ for deterministic processes given we **have** observed it already) multiplied by our standard likelihood where we ignore selection effects, divided by the chance of observing data in general at the area in parameter space.

Now the denominator here cannot be evaluated in its current state, we need to introduce an integral over all data (denoted $D$) such that we can apply the selection effect on it.

$$ P(d|\theta, S) = \frac{P(S|d,\theta) P(d|\theta)}{\int P(S|D, \theta) P(D|\theta) dD} $$

## The Simplest Gaussian Example

Let's assume we have data samples of some observable $d$. In our model, $d$ is drawn from a normal distribution such that we wish to characterise two model parameters describing said normal - the mean $\mu$ and the standard deviation $\sigma$.

$$ P(d|\mu, \sigma) = \mathcal{N}(d|\mu, \sigma) $$

Now let's imagine the case as described in our data where we can only observe some value when its above a threshold of $\alpha$. Or more formally,

$$P(S|d, \theta) = \mathcal{H}(d-\alpha),$$

where $\mathcal{H}$ is the Heaviside step function:

$$ \mathcal{H}(y)\equiv \begin{cases}
1 \quad {\rm if }\  y \ge 0 \\
0 \quad {\rm otherwise.}
\end{cases} $$

You can see that this selection probability is deterministic, so any data we did observe we observed with a probability of one. This helps simplify the numerator such that $P(S\|d,\theta) = 1$. Which just leaves us the denominator:

$$ \int P(S|D, \theta) P(D|\mu, \sigma) dD = \int \mathcal{H}(d-\alpha) \mathcal{N}(D|\mu, \sigma) dD $$

And because the Heaviside step function sets all $d<\alpha$ to zero, we can simply modify the bounds of the integral and do this analytically:

$$\begin{align}
\int \mathcal{H}(d-\alpha) P(D|\mu, \sigma) dD &= \int_{\alpha}^\infty \mathcal{N}(D|\mu, \sigma)\, dD \\ &= \frac{1}{2} {\rm erfc}\left[ \frac{\alpha - \mu}{\sqrt{2}\sigma} \right]
\end{align}$$

So this means we can throw this denominator back into our full expression for the likelihood:

$$ P(d|\mu, \sigma, S) = \frac{2\mathcal{N}(d|\mu, \sigma)}{ {\rm erfc}\left[ \frac{\alpha - \mu}{\sqrt{2}\sigma} \right]} $$

Finally, we should note this likelihood (and correction) is for one data point. If we had a hundred data points, we'd do this multiplicatively for each $d$.

## Verifying the selection-corrected likelihood

To do this, lets first generate a dataset, create a model, fit it with `emcee` and verify that our estimations of $\mu$ and $\sigma$ are unbiased. First, the dataset.

```python
import numpy as np
np.random.seed(3)
mu, sigma, alpha, num_points = 100, 10, 85, 1000
d_all = np.random.normal(mu, sigma, size=num_points)
d = d_all[d_all > alpha]
```

This is the same code used to generat the plot you saw up the top, just with less datapoints! Let's create our model, one with the sample selection, and one without. Remember, we work in log space for probability, so that you don't get tripped up when reading the code implementation of the math above.

```python
from scipy.stats import norm
from scipy.special import erfc

def uncorrected_likelihood(xs, data):
    mu, sigma = xs
    if sigma < 0:
        return -np.inf
    return norm(mu, sigma).logpdf(data).sum()

def corrected_likelihood(xs, data):
    mu, sigma = xs
    if sigma < 0:
        return -np.inf
    correction = data.size * np.log(0.5 * erfc((alpha - mu)/(np.sqrt(2) * sigma)))
    return uncorrected_likelihood(xs, data) - correction
```

Note here that I've been cheeky and included flat priors and a prior boundary to keep $\sigma$ positive in the likehood, which means I should really call it the posterior, but let's not get bogged down on semantics.

With that, our model is fully defined. We can now try and fit it to the data to see how we go.

### Model Fitting
Let's use my go-to solution, `emcee` to sample our likelihood given our dataset, and `ChainConsumer` to take those samples and turn them into handy plots. If you want more details check out the [Bayesian Linear Regression tutorial](/tutorial/2019/07/27/BayesianLinearRegression.html) for implementation details.

```python
import emcee

ndim = 2
nwalkers = 50
p0 = np.array([95, 0]) + np.random.uniform(low=1, high=10, size=(nwalkers, ndim))
results = {}

functions = [corrected_likelihood, uncorrected_likelihood]
names = ["Corrected Likelihood", "Uncorrected Likelihood"]
for fn, name in zip(functions, names):
    sampler = emcee.EnsembleSampler(nwalkers, ndim, fn, args=[d])
    state = sampler.run_mcmc(p0, 2000)
    chain = sampler.chain[:, 300:, :]
    flat_chain = chain.reshape((-1, ndim))
    results[name] = flat_chain
```

And now to plot these samples:

```python
from chainconsumer import ChainConsumer
c = ChainConsumer()
for name, flat_chain in results.items():
    c.add_chain(flat_chain, parameters=["$\mu$", "$\sigma$"], name=name)
c.configure()
c.configure_truth(color='w') ### REMOVE
c.plotter.plot(truth=[mu, sigma], figsize=2.0);
```

Hopefully you can now see how the correction we applied to the likelihood unbiases its estimations.

You'll notice that the contours don't sit perfectly on the true value, but if we made a hundred realisations of the data and averaged out the contour positions, you'd see they would. [In fact, you can see it right here](https://arxiv.org/abs/1706.03856).

Thinking about the problem in this way allows us to neatly separate out the selection effects and generic likelihood so we can treat them independently. Of course, when you get past the point where analytic approximations to your selection effects aren't good enough, you can expect a good numerical hit where you have to numerically compute the correction.

But one thing that is *correct* about this approach that a lot of other approaches miss (such as adding bias corrections to your data) is that the *correction* is dependent on where you are in parameter space. And this should make sense conceptually - the correction is just answering the question "How efficient are we *in general* given our current model parametrisation". If we've charactered our instrument cannot detect $d < 85$, we expect to lose more events if the population mean is close to $85$ and less events if the population mean is at $200$.