```python
%matplotlib widget
%matplotlib
# check your backend is ipympl, which needs to be installed by yourself manually pip install ipympl
import matplotlib.pyplot as plt
import numpy as np
plt.ion()
```

    Using matplotlib backend: module://ipympl.backend_nbagg

    <matplotlib.pyplot._IonContext at 0x1eac4c06d90>

```python
from matplotlib import pyplot as plt

class LineBuilder:
    def __init__(self, line):
        self.line = line
        self.xs = list(line.get_xdata())
        self.ys = list(line.get_ydata())
        self.cid_mouse_press = line.figure.canvas.mpl_connect('button_press_event', self)
        self.cid_key_press = line.figure.canvas.mpl_connect('key_press_event', self.finish)

    def __call__(self, event):
#         print('click', event)
        if event.inaxes!=self.line.axes: return
        self.xs.append(event.xdata)
        self.ys.append(event.ydata)
        self.line.set_data(self.xs, self.ys)
        self.line.figure.canvas.draw()

    def finish(self, event):
        if event.inaxes!=self.line.axes: return
        if event.key == 'enter':
            self.xs.append(self.xs[0])
            self.ys.append(self.ys[0])
            self.line.set_data(self.xs, self.ys)
            self.line.figure.canvas.draw()
            line.figure.canvas.mpl_disconnect(self.cid_mouse_press)
            line.figure.canvas.mpl_disconnect(self.cid_key_press)

fig, ax = plt.subplots()
ax.set_title('click to build line segments')
line, = ax.plot([], [])  # empty line
linebuilder = LineBuilder(line)
ax.set_xlim(-9, 9)
ax.set_ylim(-9, 9)

plt.show()
```

```python
def uniformly_scattered_points_in_polygon(polygon, ny=40):
    if polygon.xs[-1] != polygon.xs[0]:
        raise ValueError("The polygon is not closed in x coordinate.")
    if polygon.ys[-1] != polygon.ys[0]:
        raise ValueError("The polygon is not closed in y coordinate.")
    poly_edge_num = len(polygon.xs) - 1

    ymin, ymax = min(polygon.ys), max(polygon.ys)
    ygap = ymax - ymin
    dy = ygap / ny
    dx = dy
    yarr = np.linspace(ymin+dy, ymax, num=ny, endpoint=False)

    scattered_points_x = []
    scattered_points_y = []
    def line_of_two_points(x1, y1, x2, y2):
        '''
        return a, b, c of line ax + by + c = 0
        '''
        if abs(y2-y1)> abs(x2-x1): # the line is steeo, let x = k2 * y + b
            k2 = (x2 - x1) / (y2 - y1)
            c = x1 - k2*y1
            return 1, -k2, -c
        else: # let y = k1 * x + b
            k1 = (y2 - y1) / (x2 - x1)
            c = y1 - k1*x1
            return -k1, 1, -c
    for y in yarr:
        intersected_points_x = list()
        for i in range(poly_edge_num):
            if (polygon.ys[i] - y)*(polygon.ys[i+1] - y) < 0: # the segment intersects with y = {y} horizontal line.
                seg_a, seg_b, seg_c = line_of_two_points(
                    polygon.xs[i], polygon.ys[i],
                    polygon.xs[i+1], polygon.ys[i+1])
                intersected_points_x.append( -(seg_b*y+seg_c) / seg_a )
        intersected_points_x.sort()
        if len(intersected_points_x)%2 != 0:
            print(intersected_points_x)
            raise RuntimeError(f"The polygon is strange that we don't have even number of intersected points with y={y} line.")

        for i in range(int(len(intersected_points_x)/2)):
            horizon_seg_xbeg = intersected_points_x[2*i]
            horizon_seg_xend = intersected_points_x[2*i+1]
            scat_x = horizon_seg_xbeg
            while True:
                scattered_points_x.append(scat_x)
                scattered_points_y.append(y)
                scat_x+= dx
                if scat_x > horizon_seg_xend:
                    break

    return scattered_points_x, scattered_points_y

```

```python

```

```python
from sympy import Function, Symbol, symbols, lambdify
from sympy import sin, pi
epsilon, rho_i, theta_i = symbols("\\epsilon, r_i, \\theta_i ", real=True)
rho_ip1, theta_ip1 = symbols("r_{i+1}, \\theta_{i+1}", real=True)

rho_ip1 = rho_ip1.subs(rho_ip1, rho_i + epsilon*sin(theta_i))
theta_ip1 = theta_ip1.subs(theta_ip1, theta_i + rho_ip1)

fixed_points_rho = np.asarray([2*np.pi, 2*np.pi])
fixed_points_theta = np.asarray([0, np.pi])
fixed_points_x = fixed_points_rho * np.cos(fixed_points_theta)
fixed_points_y = fixed_points_rho * np.sin(fixed_points_theta)
fixed_points_scat = ax.scatter(fixed_points_x, fixed_points_y)

init_scat_x, init_scat_y = uniformly_scattered_points_in_polygon(linebuilder)
init_scat_x, init_scat_y = np.asarray(init_scat_x), np.asarray(init_scat_y)
init_scat_rho = np.sqrt( init_scat_x ** 2 + init_scat_y ** 2 )
init_scat_theta = np.arctan2( init_scat_y, init_scat_x )
# scat = ax.scatter(init_scat_x, init_scat_y)

parameter_dict = {epsilon: 0.3}

lamb_rho_ip1 = lambdify([rho_i, theta_i], rho_ip1.subs(parameter_dict))
lamb_theta_ip1 = lambdify([rho_i, theta_i], theta_ip1.subs(parameter_dict))
```

```python
from sympy import Function, Symbol, symbols, lambdify
from sympy import sin, pi
epsilon, rho_i, theta_i = symbols("\\epsilon, r_i, \\theta_i ", real=True)
rho_ip1, theta_ip1 = symbols("r_{i+1}, \\theta_{i+1}", real=True)

rho_ip1 = rho_ip1.subs(rho_ip1, rho_i + epsilon*sin(theta_i))
theta_ip1 = theta_ip1.subs(theta_ip1, theta_i + rho_ip1)

fixed_points_rho = np.asarray([2*np.pi, 2*np.pi])
fixed_points_theta = np.asarray([0, np.pi])
fixed_points_x = fixed_points_rho * np.cos(fixed_points_theta)
fixed_points_y = fixed_points_rho * np.sin(fixed_points_theta)
fixed_points_scat = ax.scatter(fixed_points_x, fixed_points_y)

init_scat_x, init_scat_y = uniformly_scattered_points_in_polygon(linebuilder)
init_scat_x, init_scat_y = np.asarray(init_scat_x), np.asarray(init_scat_y)
init_scat_rho = np.sqrt( init_scat_x ** 2 + init_scat_y ** 2 )
init_scat_theta = np.arctan2( init_scat_y, init_scat_x )
# scat = ax.scatter(init_scat_x, init_scat_y)

parameter_dict = {epsilon: 0.3}

lamb_rho_ip1 = lambdify([rho_i, theta_i], rho_ip1.subs(parameter_dict))
lamb_theta_ip1 = lambdify([rho_i, theta_i], theta_ip1.subs(parameter_dict))
```

```python

```

```python
# mapped_scat_rho = lamb_rho_ip1(init_scat_rho, init_scat_theta)
# mapped_scat_theta = lamb_theta_ip1(init_scat_rho, init_scat_theta)
# mapped_scat_x = mapped_scat_rho * np.cos(mapped_scat_theta)
# mapped_scat_y = mapped_scat_rho * np.sin(mapped_scat_theta)
# scat.remove()
# scat = ax.scatter(mapped_scat_x, mapped_scat_y)
# init_scat_rho = mapped_scat_rho
# init_scat_theta = mapped_scat_theta
```

```python
niter = 50
mapped_scat_rho = np.empty((len(init_scat_rho), niter))
mapped_scat_theta = np.empty((len(init_scat_rho), niter))
mapped_scat_x = np.empty((len(init_scat_rho), niter))
mapped_scat_y = np.empty((len(init_scat_rho), niter))
mapped_scat_rho[:,0] = init_scat_rho
mapped_scat_theta[:,0] = init_scat_theta
mapped_scat_x[:,0] = init_scat_x
mapped_scat_y[:,0] = init_scat_y

scat_list = []
for i in range(niter-1):
    mapped_scat_rho[:,i+1] = lamb_rho_ip1(mapped_scat_rho[:,i], mapped_scat_theta[:,i])
    mapped_scat_theta[:,i+1] = lamb_theta_ip1(mapped_scat_rho[:,i], mapped_scat_theta[:,i])
    mapped_scat_x[:,i+1] = mapped_scat_rho[:,i] * np.cos(mapped_scat_theta[:,i])
    mapped_scat_y[:,i+1] = mapped_scat_rho[:,i] * np.sin(mapped_scat_theta[:,i])
for i in range(len(init_scat_rho)):
    scat_list.append( ax.scatter(mapped_scat_x[i,:], mapped_scat_y[i,:], s=0.02) )

```

```python

```

```python

```

```python
"""
Illustrate the figure and axes enter and leave events by changing the
frame colors on enter and leave
"""
import matplotlib.pyplot as plt

def enter_axes(event):
    print('enter_axes', event.inaxes)
    event.inaxes.patch.set_facecolor('yellow')
    event.canvas.draw()

def leave_axes(event):
    print('leave_axes', event.inaxes)
    event.inaxes.patch.set_facecolor('white')
    event.canvas.draw()

def enter_figure(event):
    print('enter_figure', event.canvas.figure)
    event.canvas.figure.patch.set_facecolor('red')
    event.canvas.draw()

def leave_figure(event):
    print('leave_figure', event.canvas.figure)
    event.canvas.figure.patch.set_facecolor('grey')
    event.canvas.draw()

fig1, axs = plt.subplots(2)
fig1.suptitle('mouse hover over figure or axes to trigger events')

fig1.canvas.mpl_connect('figure_enter_event', enter_figure)
fig1.canvas.mpl_connect('figure_leave_event', leave_figure)
fig1.canvas.mpl_connect('axes_enter_event', enter_axes)
fig1.canvas.mpl_connect('axes_leave_event', leave_axes)

fig2, axs = plt.subplots(2)
fig2.suptitle('mouse hover over figure or axes to trigger events')

fig2.canvas.mpl_connect('figure_enter_event', enter_figure)
fig2.canvas.mpl_connect('figure_leave_event', leave_figure)
fig2.canvas.mpl_connect('axes_enter_event', enter_axes)
fig2.canvas.mpl_connect('axes_leave_event', leave_axes)

plt.show()
```


Universidad Nacional de Córdoba - Facultad de Matemática, Astronomía, Física y Computación

Diplomatura en Ciencia de Datos, Aprendizaje Automático y sus Aplicaciones

 Regresión Linear - Ejemplo

 Análisis y Visualización de Datos - 2019

En este ejemplo veremos cómo implementar una regresión logística para predecir una variable numérica. Volveremos a utilizar el dataset [Human Freedom Index 2018](https://www.cato.org/human-freedom-index-new) de el instituto Cato.

Usaremos la misma versión del conjunto de datos que en el práctico.

En esta notebook vamos a tratar de estimar una función lineal que modele el cambio a través del tiempo de la libertad personal y la económica.

```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy
import pandas
import seaborn
```

    /usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
      import pandas.util.testing as tm

```python
seaborn.set_context(context='talk', font_scale=1.2)
```

```python
BLUE = '#35A7FF'
RED = '#FF5964'
GREEN = '#6BF178'
YELLOW = '#FFE74C'
```

```python
dataset = pandas.read_csv(
    'https://object.cato.org/sites/cato.org/files/human-freedom-index-files/human-freedom-index-2019.csv')
dataset.shape
```

    (1620, 120)

```python
score_cols = [col for col in dataset.columns if 'pf_identity' in col] + [
    'pf_score', # Personal Freedom (score)
    'pf_rank', # Personal Freedom (rank)
    'ef_score', # Economic Freedom (score)
    'ef_rank', # Economic Freedom (rank)
    'hf_score', # Human Freedom (score)
    'hf_rank', # Human Freedom (rank)
]

important_cols = ['year', 'ISO_code', 'countries', 'region'] + score_cols
```

```python
dataset = dataset[important_cols].replace('-', numpy.nan)
for score_col in score_cols:
    dataset[score_col] = pandas.to_numeric(dataset[score_col])
dataset
```

      year
      ISO_code
      countries
      region
      pf_identity_legal
      pf_identity_sex_male
      pf_identity_sex_female
      pf_identity_sex
      pf_identity_divorce
      pf_identity
      pf_score
      pf_rank
      ef_score
      ef_rank
      hf_score
      hf_rank

      0
      2017
      ALB
      Albania
      Eastern Europe
      0.0
      10.0
      10.0
      10.0
      7.5
      5.8
      8.01
      46.0
      7.67
      30.0
      7.84
      38.0

      1
      2017
      DZA
      Algeria
      Middle East & North Africa
      NaN
      0.0
      0.0
      0.0
      0.0
      0.0
      5.20
      146.0
      4.77
      159.0
      4.99
      155.0

      2
      2017
      AGO
      Angola
      Sub-Saharan Africa
      10.0
      0.0
      0.0
      0.0
      5.0
      5.0
      5.98
      121.0
      4.83
      158.0
      5.40
      151.0

      3
      2017
      ARG
      Argentina
      Latin America & the Caribbean
      10.0
      10.0
      10.0
      10.0
      10.0
      10.0
      8.04
      41.0
      5.67
      147.0
      6.86
      77.0

      4
      2017
      ARM
      Armenia
      Caucasus & Central Asia
      7.0
      10.0
      10.0
      10.0
      7.5
      8.2
      7.15
      72.0
      7.70
      27.0
      7.42
      54.0

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      1615
      2008
      AUS
      Australia
      Oceania
      NaN
      10.0
      10.0
      10.0
      10.0
      10.0
      9.29
      7.0
      8.18
      6.0
      8.73
      4.0

      1616
      2008
      DNK
      Denmark
      Western Europe
      NaN
      10.0
      10.0
      10.0
      10.0
      10.0
      9.49
      3.0
      7.98
      9.0
      8.73
      4.0

      1617
      2008
      CHE
      Switzerland
      Western Europe
      NaN
      10.0
      10.0
      10.0
      10.0
      10.0
      9.31
      6.0
      8.35
      4.0
      8.83
      3.0

      1618
      2008
      NZL
      New Zealand
      Oceania
      NaN
      10.0
      10.0
      10.0
      10.0
      10.0
      9.42
      4.0
      8.46
      3.0
      8.94
      2.0

      1619
      2008
      HKG
      Hong Kong
      East Asia
      NaN
      10.0
      10.0
      10.0
      10.0
      10.0
      9.13
      12.0
      9.11
      1.0
      9.12
      1.0

1620 rows × 16 columns

En el práctico habíamos trabajado sobre las variables `ef_score` y `pf_score`, que hacen referencia a los índices de libertad personal y libertad económica de cada páis. Además, sabemos que el dataset incluye una medición del índice anual por país desde 2008 hasta 2016, aunque hay datos faltantes de algunos indicadores.

La motivación de este análisis comienza con este gráfico, que muestra una tendencia decreciente de la libertad personal y una tendencia ascendiente de la libertad económica. La libertad humana o `pf_score` es el promedio de ambos indicadores

```python
plt.figure(figsize=(10,6))
seaborn.lineplot(data=dataset,
                 x='year', y='ef_score',
                 label='Economic Freedom')
seaborn.lineplot(data=dataset,
                 x='year', y='pf_score',
                 label='Personal Freedom')

seaborn.lineplot(data=dataset,
                 x='year', y='hf_score',
                 label='Human Freedom')
plt.legend()
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.ylabel('Index score')
seaborn.despine()
```

Este fenómeno podría estar dado por varios factores:
  * Hay pocos países en los que la libertad personal está decreciendo, pero su libertad económica se mantiene constante.
  * Los países para los cuales sube la libertad económica decrecen en libertad personal.
  * **¿Otras?**

Veamos qué sucede en Argentina. Si graficamos ambas variables, vemos que "van bajando". Formalmente, esto significa que hay la recta que las modela tiene una pendiente negativa.

**¿Y esto, es grave?**

```python
plt.figure(figsize=(15, 8))
seaborn.regplot(data=dataset[(dataset.ISO_code == 'ARG')],
                    x='year', y='ef_score', label='Economic Freedom')
seaborn.regplot(data=dataset[(dataset.ISO_code == 'ARG')],
                    x='year', y='pf_score', label='Personal Freedom')
seaborn.regplot(data=dataset[(dataset.ISO_code == 'ARG')],
                    x='year', y='hf_score', label='Human Freedom')
plt.legend()
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
seaborn.despine()
plt.xlim(2008, 2016)
```

Podemos graficar varios países, pero es difícil comparar visualmente entre tantas variables, qué países "decrecen" más rápido que otros.

```python
countries = ['Argentina', 'Brazil', 'Mexico', 'Bolivia',
             'Uruguay', 'Peru', 'Colombia', 'Venezuela']
g = seaborn.FacetGrid(dataset, col="countries",
                      margin_titles=True, aspect=2, col_wrap=2,
                      col_order=countries)
g.map(seaborn.lineplot, "year", "ef_score", color=BLUE)
g.map(seaborn.lineplot, "year", "pf_score", color=RED)
g.set(xlim=(2008, 2016))

print('Puntajes de libertad para América Latina');
```

Para poder comparar la situación de Argentina con otros países, podemos comparar la pendiente de la recta de la regresión lineal. A partir del gráfico anterior pudimos ver que la mayoría de los países tiene tendencias similares y que se pueden estimar con una recta sin perder generalidad. Esto es posible también, en cierta medida, porque tenemos pocos puntos para estimar.

## Regresión lineal

Queremos ver cuál es el coeficiente que relaciona ambas variables.

Primero: **¿Cuál es la variable dependiente? ¿Cuál la independiente?**

Una vez que hemos determinado eso, lo que queremos encontrar es la función de la siguiente forma:

$$ef = a * year + b$$

Reescribiremos esto como una función $e$ (por economic), cuyo parámetro es el valor $y$ (por year):

$$e(y) = a * y + b$$

Vamos a describir los ejemplos como pares $(x_y, x_e)$, donde $x_y$ denota el `year` y $x_e$ denota `ef_score`.

Para encontrar la recta $e$ que mejor describe los datos, queremos minimizar el error cuadrático medio, definido como:

$$mse = \frac{1}{|X|} \sum_{x \in X} (e(x_y) - x_e)^2 $$

Recordemos que para minimizar una función, una buena opción es comenzar por buscar los puntos estacionarios, donde la derivada se anula. Por suerte, la función $mse$ es convexa, y por lo tanto tiene todos sus puntos estacionarios son minimizadores. El minimizador es el valor de los parámetros $a$ y $b$ que minimizan la función. Ahora, en hemos cambiado nuestras "variables", lo que buscamos es encontrar la función adecuada, por lo tanto lo que cambia son los valores de los parámetros que definen la función.

Primero, notemos que:

$$\frac{\partial}{\partial a}e(y) = x_p$$

$$\frac{\partial}{\partial b}e(y) = 1$$

Con eso, calculamos las derivadas parciales para cada parámetro de la función $mse$.

$$\frac{\partial}{\partial a}mse = \frac{2}{|X|} \sum_{x \in X} (e(x_p) - x_e) \frac{\partial}{\partial a} (e(x_p) - x_e) = \frac{2}{|X|} \sum_{x \in X} (e(x_p) - x_e) e_p $$

$$\frac{\partial}{\partial b}mse = \frac{2}{|X|} \sum_{x \in X} \frac{\partial}{\partial b} e(x_p) - x_e = \frac{2}{|X|} \sum_{x \in X} e(x_p) - x_e $$

A pesar del formulerío, es bastante simple. Sólo reemplazamos $mse$ por su definición, y luego aplicamos un par de reglas como "la derivada de la suma es la suma de las derivadas", la regla de la cadena, o la definición de la derivada de la función cuadrática.

Una vez que tenemos esos valores, tenemos que igualarlos a cero para encontrar los puntos estacionarios.

\begin{align}
 \frac{\partial}{\partial a}mse &= \frac{2}{|X|} \sum_{x \in X} (e(x_y) - x_e) x_y = 0 \\
                                &\Rightarrow a = \frac{\bar{x_y} \bar{x_e} - \overline{x_yx_e}}{(\bar{x_y})^2 - \overline{x_y^2}}
\end{align}

\begin{align}
 \frac{\partial}{\partial b}mse &= \frac{2}{|X|} \sum_{x \in X} e(x_y) - x_e = 0 \\
                                &\Rightarrow b = \bar{x_e} - a \bar{x_y}
\end{align}

Donde $\bar{x}$ es la media del valor para todos los ejemplos. Vamos a confiar en estas fórmulas, pero una demostración de las mismas está en:

https://medium.freecodecamp.org/machine-learning-mean-squared-error-regression-line-c7dde9a26b93

```python
def estimate_params(X_y, X_e):
    """Caculates the value of a using all the examples."""
    num = numpy.mean(X_y)*numpy.mean(X_e) - numpy.mean(numpy.multiply(X_y, X_e))
    denom = numpy.mean(X_y)**2 - numpy.mean(numpy.multiply(X_y, X_y))
    a = num / denom
    b = numpy.mean(X_e) - a * numpy.mean(X_y)
    return a, b
```

```python
# Asumimos que todos los registros que tienen pf_score tienen el año.
a, b = estimate_params(
    dataset[(dataset.ISO_code == 'ARG') &
            (dataset.pf_score.notnull())].year.dropna(),
    dataset[dataset.ISO_code == 'ARG'].pf_score)
a, b
```

    (-0.02048484848483261, 49.32575757572563)

```python
def base_linear_regression(x_y, a):
    return a * x_y
```

```python
def regplot2(data, x, y, reg_func, **reg_func_args):
    """Plots the x, y columns from data and builds a
    line with the regression reg_func."""
    seaborn.scatterplot(data=data, x=x, y=y, color=BLUE)
    minimum = data[x].min()
    maximum = data[x].max()
    plt.plot([minimum, maximum],
             [reg_func(minimum, **reg_func_args),
              reg_func(maximum, **reg_func_args)],
             color=GREEN)
    seaborn.despine()
    plt.show()
```

```python
regplot2(dataset[dataset.ISO_code == 'ARG'],
         x='year', y='pf_score', reg_func=base_linear_regression,
         a=a)
```

Vemos que la recta va en el sentido correcto, pero está demasiado abajo. Esto ocurre porque no hemos usado el término de bias.

Redefinamos entonces la regresión logística

```python
def linear_regression(x_y, a, b):
    return a * x_y + b
```

```python
regplot2(dataset[dataset.ISO_code == 'ARG'],
         x='year', y='pf_score', reg_func=linear_regression,
         a=a, b=b)
```

## Continuamos el análisis

Perfecto! Ahora podemos calcular las pendientes y los biases para todos los años, para regresiones que estimen el `pf_score`.

```python
def build_regressions(data, x_var='year', y_var='pf_score'):
    records = []
    for code in data.ISO_code.unique():
        record = [code, data[data.ISO_code == code].region.values[0],
                  data[data.ISO_code == code].countries.values[0]]
        y_data = data[data.ISO_code == code][y_var].dropna()
        # Comprobamos que hay datos en el intervalo
        if len(y_data) <= 1:
            continue
        x_data = data[(data.ISO_code == code) &
                      (data[y_var].notnull())][x_var].dropna()
        # Estimamos los parámetros
        a, b = estimate_params(x_data, y_data)
        # Calculamos el error cuadrático medio de la regresión lineal estimada
        predictions = numpy.apply_along_axis(
            lambda x: linear_regression(x, a, b), 0, x_data)
        mse = numpy.mean(numpy.power(predictions - y_data, 2))
        record.extend([a, b, mse])
        # Agregamos el registro
        records.append(record)
    return pandas.DataFrame.from_records(
        records, columns=['ISO_code', 'region', 'country',
                          'slope', 'bias', 'mse']
    )
```

```python
pf_regressions = build_regressions(dataset).set_index('ISO_code')
pf_regressions[:10]
```

      region
      country
      slope
      bias
      mse

      ISO_code

      ALB
      Eastern Europe
      Albania
      -0.036909
      82.084545
      0.044086

      DZA
      Middle East & North Africa
      Algeria
      -0.006121
      17.587939
      0.009140

      AGO
      Sub-Saharan Africa
      Angola
      0.087636
      -170.678182
      0.039059

      ARG
      Latin America & the Caribbean
      Argentina
      -0.020485
      49.325758
      0.003178

      ARM
      Caucasus & Central Asia
      Armenia
      -0.016667
      40.756667
      0.008773

      AUS
      Oceania
      Australia
      -0.011576
      32.513212
      0.002516

      AUT
      Western Europe
      Austria
      0.037394
      -66.048303
      0.007005

      AZE
      Caucasus & Central Asia
      Azerbaijan
      -0.105758
      218.958121
      0.060136

      BHS
      Latin America & the Caribbean
      Bahamas
      -0.032909
      74.170545
      0.009934

      BHR
      Middle East & North Africa
      Bahrain
      -0.089515
      186.388242
      0.048762

```python
plt.figure(figsize=(10,6))
seaborn.distplot(
    pf_regressions[pf_regressions.region == 'Latin America & the Caribbean'].slope,
    color=BLUE, label='Latam y Caribe')
seaborn.distplot(pf_regressions.slope, color=RED, label='Global')
plt.xlabel('Slope of linear regression that fits the pf_score of each country')
plt.legend()
seaborn.despine()
```

```python
def plot_regressions(regressions):
    plt.figure(figsize=(10,6))
    colors = seaborn.color_palette("cubehelix", len(regressions))
    for color, (year, row) in zip(colors, regressions.iterrows()):
        minimum, maximum = 2008, 2016
        plt.plot([minimum, maximum],
                 [linear_regression(minimum, row['slope'], row['bias']),
                  linear_regression(maximum, row['slope'], row['bias'])],
                 color=color, label=str(year), linestyle='--')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    seaborn.despine()
```

```python
plot_regressions(pf_regressions.loc[['ARG', 'BRA', 'VEN', 'CAN']])
plt.xlabel('year')
plt.ylabel('pf_score')
plt.ylim(4, 10)
```

### Libertad Económica

```python
ef_regressions = build_regressions(dataset, y_var='ef_score').set_index('ISO_code')
ef_regressions[:10]
```

      region
      country
      slope
      bias
      mse

      ISO_code

      ALB
      Eastern Europe
      Albania
      0.047333
      -87.801333
      0.003397

      DZA
      Middle East & North Africa
      Algeria
      -0.042606
      90.805697
      0.009553

      AGO
      Sub-Saharan Africa
      Angola
      0.015394
      -25.786303
      0.084069

      ARG
      Latin America & the Caribbean
      Argentina
      -0.083273
      173.018364
      0.132188

      ARM
      Caucasus & Central Asia
      Armenia
      0.013333
      -19.165333
      0.003609

      AUS
      Oceania
      Australia
      -0.010909
      30.026545
      0.002294

      AUT
      Western Europe
      Austria
      -0.008485
      24.819758
      0.000690

      AZE
      Caucasus & Central Asia
      Azerbaijan
      0.042242
      -78.787879
      0.007204

      BHS
      Latin America & the Caribbean
      Bahamas
      -0.026667
      61.038667
      0.003569

      BHR
      Middle East & North Africa
      Bahrain
      -0.010364
      28.097818
      0.009243

```python
plot_regressions(ef_regressions.loc[['ARG', 'BRA', 'VEN', 'CAN']])
plt.xlabel('year')
plt.ylabel('ef_score')
```

```python
plt.figure(figsize=(10,6))
seaborn.distplot(
    ef_regressions[ef_regressions.region == 'Latin America & the Caribbean'].slope,
    color=BLUE, label='Latam y Caribe')
seaborn.distplot(ef_regressions.slope, color=RED, label='Global')
plt.xlabel('Slope of linear regression that fits the ef_score of each country')
plt.legend()
seaborn.despine()
```

## Análisis conjunto

**¿Cuáles es el 10% de países en los que la libertad humana disminuye más rápidamente?**

```python
quantil = pf_regressions.slope.quantile(0.1)
pf_regressions[pf_regressions.slope < quantil].country
```

    ISO_code
    BTN               Bhutan
    BRA               Brazil
    BRN    Brunei Darussalam
    BDI              Burundi
    EGY                Egypt
    GMB          Gambia, The
    MUS            Mauritius
    NPL                Nepal
    NER                Niger
    RWA               Rwanda
    SYR                Syria
    TJK           Tajikistan
    THA             Thailand
    TUR               Turkey
    UKR              Ukraine
    VEN            Venezuela
    YEM          Yemen, Rep.
    Name: country, dtype: object

**¿Cuáles es el 10% de países en los que la libertad económica disminuye más rápidamente?**

```python
quantil = ef_regressions.slope.quantile(0.1)
ef_regressions[ef_regressions.slope < quantil].country
```

    ISO_code
    ARG            Argentina
    BRA               Brazil
    BRN    Brunei Darussalam
    EGY                Egypt
    FJI                 Fiji
    GHA                Ghana
    IRQ                 Iraq
    KWT               Kuwait
    LBR              Liberia
    LBY                Libya
    PNG      Pap. New Guinea
    SLE         Sierra Leone
    SDN                Sudan
    SYR                Syria
    TUN              Tunisia
    VEN            Venezuela
    ZMB               Zambia
    Name: country, dtype: object

**¿Cuáles son los paises en los que la libertad económica aumenta pero la libertad personal disminuye (rápidamente)?**

```python
all_countries = dataset.ISO_code.unique()
codes = []
for code in all_countries:
    if (code in ef_regressions.index and code in pf_regressions.index and
        ef_regressions.loc[code].slope > 0.02 and
        pf_regressions.loc[code].slope < -0.02):
        codes.append(code)
ef_regressions.loc[codes].country
```

    ISO_code
    ALB        Albania
    AZE     Azerbaijan
    BGR       Bulgaria
    BDI        Burundi
    KHM       Cambodia
    CPV     Cape Verde
    CHN          China
    GMB    Gambia, The
    GTM      Guatemala
    GIN         Guinea
    ISL        Iceland
    IDN      Indonesia
    KAZ     Kazakhstan
    LAO           Laos
    MYS       Malaysia
    MLT          Malta
    MEX         Mexico
    MAR        Morocco
    MMR        Myanmar
    NPL          Nepal
    NER          Niger
    PRY       Paraguay
    PHL    Philippines
    RUS         Russia
    RWA         Rwanda
    ESP          Spain
    TZA       Tanzania
    VNM        Vietnam
    Name: country, dtype: object

# Errores

Calculamos el mse pero nunca lo usamos. Veamos cómo son los países para los que la regresión linear no produce una buena aproximación

```python
pf_regressions.mse.sort_values()[-10:]
```

    ISO_code
    TLS    0.106220
    SYC    0.107328
    TUR    0.126490
    CAF    0.142989
    VEN    0.145224
    COD    0.165123
    LBY    0.182368
    GNB    0.194737
    YEM    0.195064
    BDI    0.206774
    Name: mse, dtype: float64

```python
plt.figure(figsize=(10,6))
countries = ['BDI', 'YEM', 'GNB', 'LBY']
seaborn.lineplot(data=dataset[dataset.ISO_code.isin(countries)], x='year', y='hf_score',
                 hue='countries')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
seaborn.despine()
```

Claramente se ve que estas funciones no podían ser estimadas satisfactoriamente con una recta, pero aún así, la tendencia general (descendiente o ascendiente) habría sido aproximada

```python

```


# Tutorial 3: Deep linear neural networks
**Week 1, Day 2: Linear Deep Learning**

**By Neuromatch Academy**

__Content creators:__ Saeed Salehi, Spiros Chavlis, Andrew Saxe

__Content reviewers:__ Polina Turishcheva, Antoine De Comite

__Content editors:__ Anoop Kulkarni

__Production editors:__ Khalid Almubarak, Spiros Chavlis

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives

* Deep linear neural networks
* Learning dynamics and singular value decomposition
* Representational Similarity Analysis
* Illusory correlations & ethics

```python
# @title Tutorial slides

# @markdown These are the slides for the videos in this tutorial

# @markdown If you want to locally download the slides, click [here](https://osf.io/bncr8/download)
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/bncr8/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

---
# Setup

This a GPU-Free tutorial!

```python
# @title Install dependencies
!pip install git+https://github.com/NeuromatchAcademy/evaltools --quiet

from evaltools.airtable import AirtableForm
```

```python
# Imports
import math
import torch
import matplotlib

import numpy as np
import matplotlib.pyplot as plt

import torch.nn as nn
import torch.optim as optim
```

```python
# @title Figure settings

from matplotlib import gridspec
from ipywidgets import interact, IntSlider, FloatSlider, fixed
from ipywidgets import FloatLogSlider, Layout, VBox
from ipywidgets import interactive_output
from mpl_toolkits.axes_grid1 import make_axes_locatable

import warnings
warnings.filterwarnings("ignore")

%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/content-creation/main/nma.mplstyle")
```

```python
# @title Plotting functions

def plot_x_y_hier_data(im1, im2, subplot_ratio=[1, 2]):
  fig = plt.figure(figsize=(12, 5))
  gs = gridspec.GridSpec(1, 2, width_ratios=subplot_ratio)
  ax0 = plt.subplot(gs[0])
  ax1 = plt.subplot(gs[1])
  ax0.imshow(im1, cmap="cool")
  ax1.imshow(im2, cmap="cool")
  # plt.suptitle("The whole dataset as imshow plot", y=1.02)
  ax0.set_title("Labels of all samples")
  ax1.set_title("Features of all samples")
  ax0.set_axis_off()
  ax1.set_axis_off()
  plt.tight_layout()
  plt.show()

def plot_x_y_hier_one(im1, im2, subplot_ratio=[1, 2]):
  fig = plt.figure(figsize=(12, 1))
  gs = gridspec.GridSpec(1, 2, width_ratios=subplot_ratio)
  ax0 = plt.subplot(gs[0])
  ax1 = plt.subplot(gs[1])
  ax0.imshow(im1, cmap="cool")
  ax1.imshow(im2, cmap="cool")
  ax0.set_title("Labels of a single sample")
  ax1.set_title("Features of a single sample")
  ax0.set_axis_off()
  ax1.set_axis_off()
  plt.tight_layout()
  plt.show()

def plot_tree_data(label_list = None, feature_array = None, new_feature = None):
  cmap = matplotlib.colors.ListedColormap(['cyan', 'magenta'])
  n_features = 10
  n_labels = 8
  im1 = np.eye(n_labels)
  if feature_array is None:
    im2 = np.array([[1, 1, 1, 1, 1, 1, 1, 1],
                      [0, 0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 1, 1, 1, 1],
                      [1, 1, 1, 1, 0, 0, 0, 0],
                      [0, 0, 0, 0, 0, 0, 1, 1],
                      [0, 0, 1, 1, 0, 0, 0, 0],
                      [1, 1, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 1, 1, 0, 0],
                      [0, 1, 1, 1, 0, 0, 0, 0],
                      [0, 0, 0, 0, 1, 1, 0, 1]]).T
    im2[im2 == 0] = -1
    feature_list = ['can_grow',
                    'is_mammal',
                    'has_leaves',
                    'can_move',
                    'has_trunk',
                    'can_fly',
                    'can_swim',
                    'has_stem',
                    'is_warmblooded',
                    'can_flower']
  else:
    im2 = feature_array
  if label_list is None:
    label_list = ['Goldfish', 'Tuna', 'Robin', 'Canary',
                  'Rose', 'Daisy', 'Pine', 'Oak']
  fig = plt.figure(figsize=(12, 7))
  gs = gridspec.GridSpec(1, 2, width_ratios=[1, 1.35])
  ax1 = plt.subplot(gs[0])
  ax2 = plt.subplot(gs[1])
  ax1.imshow(im1, cmap=cmap)
  if feature_array is None:
    implt = ax2.imshow(im2, cmap=cmap, vmin=-1.0, vmax=1.0)
  else:
    implt = ax2.imshow(im2[:, -n_features:], cmap=cmap, vmin=-1.0, vmax=1.0)
  divider = make_axes_locatable(ax2)
  cax = divider.append_axes("right", size="5%", pad=0.1)
  cbar = plt.colorbar(implt, cax=cax, ticks=[-0.5, 0.5])
  cbar.ax.set_yticklabels(['no', 'yes'])
  ax1.set_title("Labels")
  ax1.set_yticks(ticks=np.arange(n_labels))
  ax1.set_yticklabels(labels=label_list)
  ax1.set_xticks(ticks=np.arange(n_labels))
  ax1.set_xticklabels(labels=label_list, rotation='vertical')
  ax2.set_title("{} random Features".format(n_features))
  ax2.set_yticks(ticks=np.arange(n_labels))
  ax2.set_yticklabels(labels=label_list)
  if feature_array is None:
    ax2.set_xticks(ticks=np.arange(n_features))
    ax2.set_xticklabels(labels=feature_list, rotation='vertical')
  else:
    ax2.set_xticks(ticks=[n_features-1])
    ax2.set_xticklabels(labels=[new_feature], rotation='vertical')
  plt.tight_layout()
  plt.show()

def plot_loss(loss_array, title="Training loss (Mean Squared Error)", c="r"):
  plt.figure(figsize=(10, 5))
  plt.plot(loss_array, color=c)
  plt.xlabel("Epoch")
  plt.ylabel("MSE")
  plt.title(title)
  plt.show()

def plot_loss_sv(loss_array, sv_array):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("Set1", n_sing_values)

  _, (plot1, plot2) = plt.subplots(2, 1, sharex=True, figsize=(10, 10))
  plot1.set_title("Training loss (Mean Squared Error)")
  plot1.plot(loss_array, color='r')

  plot2.set_title("Evolution of singular values (modes)")
  for i in range(n_sing_values):
    plot2.plot(sv_array[:, i], c=cmap(i))
  plot2.set_xlabel("Epoch")
  plt.show()

def plot_loss_sv_twin(loss_array, sv_array):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(10, 5))
  ax1 = plt.gca()
  ax1.set_title("Learning Dynamics")
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel("Mean Squared Error", c='r')
  ax1.tick_params(axis='y', labelcolor='r')
  ax1.plot(loss_array, color='r')

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values (modes)", c='b')
  ax2.tick_params(axis='y', labelcolor='b')
  for i in range(n_sing_values):
    ax2.plot(sv_array[:, i], c=cmap(i))

  fig.tight_layout()
  plt.show()

def plot_ills_sv_twin(ill_array, sv_array, ill_label):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(10, 5))
  ax1 = plt.gca()
  ax1.set_title("Network training and the Illusory Correlations")
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel(ill_label, c='r')
  ax1.tick_params(axis='y', labelcolor='r')
  ax1.plot(ill_array, color='r', linewidth=3)
  ax1.set_ylim(-1.05, 1.05)
  # ax1.set_yticks([-1, 0, 1])
  # ax1.set_yticklabels(['False', 'Not sure', 'True'])

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values (modes)", c='b')
  ax2.tick_params(axis='y', labelcolor='b')
  for i in range(n_sing_values):
    ax2.plot(sv_array[:, i], c=cmap(i))

  fig.tight_layout()
  plt.show()

def plot_loss_sv_rsm(loss_array, sv_array, rsm_array, i_ep):
  n_ep = loss_array.shape[0]
  rsm_array = rsm_array / np.max(rsm_array)
  sv_array = sv_array / np.max(sv_array)

  n_sing_values = sv_array.shape[1]
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(14, 5))
  gs = gridspec.GridSpec(1, 2, width_ratios=[5, 3])

  ax0 = plt.subplot(gs[1])
  ax0.yaxis.tick_right()
  implot = ax0.imshow(rsm_array[i_ep], cmap="Purples", vmin=0.0, vmax=1.0)
  divider = make_axes_locatable(ax0)
  cax = divider.append_axes("right", size="5%", pad=0.9)
  cbar = plt.colorbar(implot, cax=cax, ticks=[])
  cbar.ax.set_ylabel('Similarity', fontsize=12)
  ax0.set_title("RSM at epoch {}".format(i_ep), fontsize=16)
  # ax0.set_axis_off()
  ax0.set_yticks(ticks=np.arange(n_sing_values))
  ax0.set_yticklabels(labels=item_names)
  # ax0.set_xticks([])
  ax0.set_xticks(ticks=np.arange(n_sing_values))
  ax0.set_xticklabels(labels=item_names, rotation='vertical')

  ax1 = plt.subplot(gs[0])
  ax1.set_title("Learning Dynamics", fontsize=16)
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel("Mean Squared Error", c='r')
  ax1.tick_params(axis='y', labelcolor='r', direction="in")
  ax1.plot(np.arange(n_ep), loss_array, color='r')
  ax1.axvspan(i_ep-2, i_ep+2, alpha=0.2, color='m')

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values", c='b')
  ax2.tick_params(axis='y', labelcolor='b', direction="in")
  for i in range(n_sing_values):
    ax2.plot(np.arange(n_ep), sv_array[:, i], c=cmap(i))
  ax1.set_xlim(-1, n_ep+1)
  ax2.set_xlim(-1, n_ep+1)

  plt.show()
```

```python
#@title Helper functions

atform = AirtableForm('appn7VdPRseSoMXEG',\
         'W1D2_T3','https://portal.neuromatchacademy.org/api/redirect/to/f60119ed-1c22-4dae-9e18-b6a767f477e1')

def build_tree(n_levels, n_branches, probability, to_np_array=True):
  """Builds a tree
  """
  assert 0.0 <= probability <= 1.0

  tree = {}

  tree["level"] = [0]
  for i in range(1, n_levels+1):
    tree["level"].extend([i]*(n_branches**i))

  tree["pflip"] = [probability]*len(tree["level"])

  tree["parent"] = [None]
  k = len(tree["level"])-1
  for j in range(k//n_branches):
    tree["parent"].extend([j]*n_branches)

  if to_np_array:
    tree["level"] = np.array(tree["level"])
    tree["pflip"] = np.array(tree["pflip"])
    tree["parent"] = np.array(tree["parent"])

  return tree

def sample_from_tree(tree, n):
  """ Generates n samples from a tree
  """
  items = [i for i, v in enumerate(tree["level"]) if v == max(tree["level"])]
  n_items = len(items)
  x = np.zeros(shape=(n, n_items))
  rand_temp = np.random.rand(n, len(tree["pflip"]))
  flip_temp = np.repeat(tree["pflip"].reshape(1, -1), n, 0)
  samp = (rand_temp > flip_temp) * 2 - 1

  for i in range(n_items):
    j = items[i]
    prop = samp[:, j]
    while tree["parent"][j] is not None:
      j = tree["parent"][j]
      prop = prop * samp[:, j]
    x[:, i] = prop.T
  return x

def generate_hsd():
  # building the tree
  n_branches = 2  # 2 branches at each node
  probability = .15  # flipping probability
  n_levels = 3  # number of levels (depth of tree)
  tree = build_tree(n_levels, n_branches, probability, to_np_array=True)
  tree["pflip"][0] = 0.5
  n_samples = 10000 # Sample this many features

  tree_labels = np.eye(n_branches**n_levels)
  tree_features = sample_from_tree(tree, n_samples).T
  return tree_labels, tree_features

def linear_regression(X, Y):
  """Analytical Linear regression

  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dx == Dy
  W = Y @ X.T @ np.linalg.inv(X @ X.T)
  return W

def add_feature(existing_features, new_feature):
  assert isinstance(existing_features, np.ndarray)
  assert isinstance(new_feature, list)
  new_feature = np.array([new_feature]).T
  # return np.hstack((tree_features, new_feature*2-1))
  return np.hstack((tree_features, new_feature))

def net_svd(model, in_dim):
  """Performs a Singular Value Decomposition on a given model weights

  Args:
    model (torch.nn.Module): neural network model
    in_dim (int): the input dimension of the model

  Returns:
    U, Σ, V (Tensors): Orthogonal, diagonal, and orthogonal matrices
  """
  W_tot = torch.eye(in_dim)
  for weight in model.parameters():
    W_tot = weight.detach() @ W_tot
  U, SIGMA, V = torch.svd(W_tot)
  return U, SIGMA, V

def net_rsm(h):
  """Calculates the Representational Similarity Matrix

  Arg:
    h (torch.Tensor): activity of a hidden layer

  Returns:
    (torch.Tensor): Representational Similarity Matrix
  """
  rsm = h @ h.T
  return rsm

def initializer_(model, gamma=1e-12):
  """(in-place) Re-initialization of weights

  Args:
    model (torch.nn.Module): PyTorch neural net model
    gamma (float): initialization scale
  """
  for weight in model.parameters():
    n_out, n_in = weight.shape
    sigma = gamma / math.sqrt(n_in + n_out)
    nn.init.normal_(weight, mean=0.0, std=sigma)

def test_initializer_ex(seed):
  torch.manual_seed(seed)
  model = LNNet(5000, 5000, 1)
  try:
    ex_initializer_(model, gamma=1)
    std = torch.std(next(iter(model.parameters())).detach()).item()
    if -1e-5 <= (std - 0.01) <= 1e-5:
      print("Well done! Seems to be correct!")
    else:
      print("Please double check your implementation!")
  except:
    print("Faulty Implementation!")

def test_net_svd_ex(seed):
  torch.manual_seed(seed)
  model = LNNet(8, 30, 100)
  try:
    U_ex, Σ_ex, V_ex = ex_net_svd(model, 8)
    U, Σ, V = net_svd(model, 8)
    if (torch.all(torch.isclose(U_ex.detach(), U.detach(), atol=1e-6)) and
        torch.all(torch.isclose(Σ_ex.detach(), Σ.detach(), atol=1e-6)) and
        torch.all(torch.isclose(V_ex.detach(), V.detach(), atol=1e-6))):
      print("Well done! Seems to be correct!")
    else:
      print("Please double check your implementation!")
  except:
    print("Faulty Implementation!")

def test_net_rsm_ex(seed):
  torch.manual_seed(seed)
  x = torch.rand(7, 17)
  try:
    y_ex = ex_net_rsm(x)
    y = x @ x.T
    if (torch.all(torch.isclose(y_ex, y, atol=1e-6))):
      print("Well done! Seems to be correct!")
    else:
      print("Please double check your implementation!")
  except:
    print("Faulty Implementation!")
```

```python
#@title Set random seed

#@markdown Executing `set_seed(seed=seed)` you are setting the seed

# for DL its critical to set the random seed so that students can have a
# baseline to compare their results to expected results.
# Read more here: https://pytorch.org/docs/stable/notes/randomness.html

# Call `set_seed` function in the exercises to ensure reproducibility.
import random
import torch

def set_seed(seed=None, seed_torch=True):
  if seed is None:
    seed = np.random.choice(2 ** 32)
  random.seed(seed)
  np.random.seed(seed)
  if seed_torch:
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

  print(f'Random seed {seed} has been set.')

# In case that `DataLoader` is used
def seed_worker(worker_id):
  worker_seed = torch.initial_seed() % 2**32
  np.random.seed(worker_seed)
  random.seed(worker_seed)
```

```python
#@title Set device (GPU or CPU). Execute `set_device()`
# especially if torch modules used.

# inform the user if the notebook uses GPU or CPU.

def set_device():
  device = "cuda" if torch.cuda.is_available() else "cpu"
  if device != "cuda":
    print("GPU is not enabled in this notebook. \n"
          "If you want to enable it, in the menu under `Runtime` -> \n"
          "`Hardware accelerator.` and select `GPU` from the dropdown menu")
  else:
    print("GPU is enabled in this notebook. \n"
          "If you want to disable it, in the menu under `Runtime` -> \n"
          "`Hardware accelerator.` and select `None` from the dropdown menu")

  return device
```

```python
SEED = 2021
set_seed(seed=SEED)
DEVICE = set_device()
```

This colab notebook is GPU free!

---
# Section 0: Prelude
*Time estimate: ~10 mins*

**A note on the exercises**: Most of the exercises are marked Optional (Bonus) and should only read through them if you are in a tight timeline. Therefore we would not rely on the implementation of the exercises. If necessary, you can look at the *Helper Functions* cell above to find the functions and classes used in this tutorial.

Throughout this tutorial, we will use a linear neural net with a single hidden layer. We have also excluded `bias` from the layers. Please note that the forward loop returns the hidden activation, besides the network output (prediction). we will need it in section 3.

```python
class LNNet(nn.Module):
  """A Linear Neural Net with one hidden layer
  """

  def __init__(self, in_dim, hid_dim, out_dim):
    """
    Args:
      in_dim (int): input dimension
      out_dim (int): ouput dimension
      hid_dim (int): hidden dimension
    """
    super().__init__()
    self.in_hid = nn.Linear(in_dim, hid_dim, bias=False)
    self.hid_out = nn.Linear(hid_dim, out_dim, bias=False)

  def forward(self, x):
    """
    Args:
      x (torch.Tensor): input tensor
    """
    hid = self.in_hid(x)  # hidden activity
    out = self.hid_out(hid)  # output (prediction)
    return out, hid
```

Other than `net_svd` and `net_rsm` functions, the training loop should be mostly familiar to you. We will define these functions in the coming sections.

**important**: Please note that the two functions are part of inner training loop and are therefore executed and recorded at every iteration.

```python
def train(model, inputs, targets, n_epochs, lr, illusory_i=0):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    inputs (torch.Tensor): features (input) with shape `[batch_size, input_dim]`
    targets (torch.Tensor): targets (labels) with shape `[batch_size, output_dim]`
    n_epochs (int): number of training epochs (iterations)
    lr (float): learning rate
    illusory_i (int): index of illusory feature

  Returns:
    np.ndarray: record (evolution) of training loss
    np.ndarray: record (evolution) of singular values (dynamic modes)
    np.ndarray: record (evolution) of representational similarity matrices
    np.ndarray: record of network prediction for the last feature
  """
  in_dim = inputs.size(1)

  losses = np.zeros(n_epochs)  # loss records
  modes = np.zeros((n_epochs, in_dim))  # singular values (modes) records
  rs_mats = []  # representational similarity matrices
  illusions = np.zeros(n_epochs)  # prediction for the given feature

  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()

  for i in range(n_epochs):
    optimizer.zero_grad()
    predictions, hiddens = model(inputs)
    loss = criterion(predictions, targets)
    loss.backward()
    optimizer.step()

    # Section 2 Singular value decomposition
    U, Σ, V = net_svd(model, in_dim)

    # Section 3 calculating representational similarity matrix
    RSM = net_rsm(hiddens.detach())

    # Section 4 network prediction of illusory_i inputs for the last feature
    pred_ij = predictions.detach()[illusory_i, -1]

    # logging (recordings)
    losses[i] = loss.item()
    modes[i] = Σ.detach().numpy()
    rs_mats.append(RSM.numpy())
    illusions[i] = pred_ij.numpy()

  return losses, modes, np.array(rs_mats), illusions
```

We also need take over the initialization of the weights. In PyTorch, [`nn.init`](https://pytorch.org/docs/stable/nn.init.html) provides us with the functions to initialize tensors from a given distribution.

## Coding Exercise 0: Re-initialization (Optional)

Complete the function `ex_initializer_`, such that the weights are sampled from the following distribution:

\begin{equation}
\mathcal{N}\left(\mu=0, ~~\sigma=\gamma \sqrt{\dfrac{1}{n_{in} + n_{out}}} \right)
\end{equation}

where $\gamma$ is the initialization scale, $n_{in}$ and $n_{out}$ are respectively input and output dimensions of the layer. the Underscore ("_") in `ex_initializer_` and other functions, denotes "[in-place](https://discuss.pytorch.org/t/what-is-in-place-operation/16244/2)" operation.

**important note**: since we did not include bias in the layers, the `model.parameters()` would only return the weights in each layer.

```python
#add event to airtable
atform.add_event('Coding Exercise 0: Re-initialization')

def ex_initializer_(model, gamma=1e-12):
  """(in-place) Re-initialization of weights

  Args:
    model (torch.nn.Module): PyTorch neural net model
    gamma (float): initialization scale
  """
  for weight in model.parameters():
    n_out, n_in = weight.shape
    #################################################
    ## Define the standard deviation (sigma) for the normal distribution
    # as given in the equation above
    # Complete the function and remove or comment the line below
    raise NotImplementedError("Function `ex_initializer_`")
    #################################################
    sigma = ...
    nn.init.normal_(weight, mean=0.0, std=sigma)

## uncomment and run
# test_initializer_ex(SEED)
```

```python
# to_remove solution

#add event to airtable
atform.add_event('Coding Exercise 0: Re-initialization')

def ex_initializer_(model, gamma=1e-12):
  """(in-place) Re-initialization of weights

  Args:
    model (torch.nn.Module): PyTorch neural net model
    gamma (float): initialization scale
  """
  for weight in model.parameters():
    n_out, n_in = weight.shape
    sigma = gamma / math.sqrt(n_in + n_out)
    nn.init.normal_(weight, mean=0.0, std=sigma)

## uncomment and run
test_initializer_ex(SEED)
```

---
# Section 1: Deep Linear Neural Nets

*Time estimate: ~20 mins*

```python
# @title Video 1: Intro to Representation Learning
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1iM4y1T7eJ", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"DqMSU4Bikt0", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 1: Intro to Representation Learning')

display(out)
```

So far, depth just seems to slow down the learning. And we know that a single nonlinear hidden layer (given enough number of neurons and infinite training samples) has the potential to approximate any function. So it seems fair to ask: **What is depth good for**?

One reason can be that shallow nonlinear neural networks hardly meet their true potential in practice. In the contrast, deep neural nets are often surprisingly powerful in learning complex functions without sacrificing generalization. A core intuition behind deep learning is that deep nets derive their power through learning internal representations. How does this work? To address representation learning, we have to go beyond the 1D chain.

For this and the next couple of exercises, we use syntactically generated hierarchically structured data through a *branching diffusion process* (see [this reference](https://www.pnas.org/content/pnas/suppl/2019/05/16/1820226116.DCSupplemental/pnas.1820226116.sapp.pdf) for more details).

 hierarchically structured data (a tree)

The inputs to the network are labels (i.e. names), while the outputs are the features (i.e. attributes). For example, for the label "Goldfish", the network has to learn all the (artificially created) features, such as "*can swim*", "*is cold-blooded*", "*has fins*", and more. Given that we are training on hierarchically structured data, network could also learn the tree structure, that Goldfish and Tuna have rather similar features, and Robin has more in common with Tuna, compared to Rose.

```python
# @markdown #### Run to generate and visualize training samples from tree

tree_labels, tree_features = generate_hsd()

# convert (cast) data from np.ndarray to torch.Tensor
label_tensor = torch.tensor(tree_labels).float()
feature_tensor = torch.tensor(tree_features).float()

item_names = ['Goldfish', 'Tuna', 'Robin', 'Canary',
              'Rose', 'Daisy', 'Pine', 'Oak']
plot_tree_data()

# dimensions
print("---------------------------------------------------------------")
print("Input Dimension: {}".format(tree_labels.shape[1]))
print("Output Dimension: {}".format(tree_features.shape[1]))
print("Number of samples: {}".format(tree_features.shape[0]))
```

To continue this tutorial, it is vital to understand the premise of our training data and what the task is. Therefore, please take your time to discuss them with your pod.

 The neural network used for this tutorial

## Interactive Demo 1: Training the deep LNN

Training a neural net on our data is straight forward. But before executing the next cell, remember the training loss curve from previous tutorial.

```python
# @markdown #### Make sure you execute this cell to train the network and plot

lr = 100.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_hidden = 30  # hidden neurons
dim_output = 10000  # output dimension = `feature_tensor.size(1)`

# model instantiation
dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

# weights re-initialization
initializer_(dlnn_model, gamma)

# training
losses, *_ = train(dlnn_model,
                  label_tensor,
                  feature_tensor,
                  n_epochs=n_epochs,
                  lr=lr)

# plotting
plot_loss(losses)
```

**Think!**

Why haven't we seen these "bumps" in training before? And should we look for them in the future? What do these bumps mean?

Recall from previous tutorial, that we are always interested in learning rate ($\eta$) and initialization ($\gamma$) that would give us the fastest but yet stable (reliable) convergence. Try finding the optimal $\eta$ and $\gamma$ using the following widgets. More specifically, try large $\gamma$ and see if we can recover the bumps by tuning the $\eta$.

```python
# @markdown #### Make sure you execute this cell to enable the widget!

def loss_lr_init(lr, gamma):
  """Trains and plots the loss evolution given lr and initialization
  Args:
    lr (float): learning rate
    gamma (float): initialization scale
  """
  n_epochs = 250  # number of epochs
  dim_input = 8  # input dimension = `label_tensor.size(1)`
  dim_hidden = 30  # hidden neurons
  dim_output = 10000  # output dimension = `feature_tensor.size(1)`

  # model instantiation
  dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

  # weights re-initialization
  initializer_(dlnn_model, gamma)

  losses, *_ = train(dlnn_model,
                    label_tensor,
                    feature_tensor,
                    n_epochs=n_epochs,
                    lr=lr)

  plot_loss(losses)

_ = interact(loss_lr_init,
             lr = FloatSlider(min=1.0, max=200.0,
                              step=1.0, value=100.0,
                              continuous_update=False,
                              readout_format='.1f',
                              description='eta'),
             epochs = fixed(250),
             gamma = FloatLogSlider(min=-15, max=1,
                                    step=1, value=1e-12, base=10,
                                    continuous_update=False,
                                    description='gamma')
             )
```

---
# Section 2: Singular Value Decomposition (SVD)

*Time estimate: ~20 mins*

```python
# @title Video 2: SVD
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1bw411R7DJ", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"18oNWRziskM", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 2: SVD')

display(out)
```

In this section, we intend to study the learning (training) dynamics we just saw. First, we should know that a linear neural network is performing sequential matrix multiplications, which can be simplified to:

\begin{align}
\mathbf{y} &= \mathbf{W}_{L}~\mathbf{W}_{L-1}~\dots~\mathbf{W}_{1} ~ \mathbf{x} \\
 &= (\prod_{i=1}^{L}{\mathbf{W}_{i}}) ~ \mathbf{x} \\
 &= \mathbf{W}_{tot} ~ \mathbf{x}
\end{align}

where $L$ denotes the number of layers in our network.

[Saxe et al. (2013)](https://arxiv.org/abs/1312.6120) showed that to analyze and to understanding the nonlinear learning dynamics of a deep LNN, we can use [Singular Value Decomposition (SVD)](https://en.wikipedia.org/wiki/Singular_value_decomposition) to decompose the $\mathbf{W}_{tot}$ into orthogonal vectors, where orthogonality of the vectors would ensure their "individuality (independence)". This means we can break a deep wide LNN into multiple deep narrow LNN, so their activity is untangled from each other.

__A Quick intro to SVD__

Any real-valued matix $A$ (yes, ANY) can be decomposed (factorized) to 3 matrices:

\begin{equation}
\mathbf{A} = \mathbf{U} \mathbf{Σ} \mathbf{V}^{\top}
\end{equation}

where $U$ is an orthogonal matrix, $\Sigma$ is a diagonal matrix, and $V$ is again an orthogonal matrix. The diagonal elements of $\Sigma$ are called **singular values**.

The main difference between SVD and EigenValue Decomposition (EVD), is that EVD requires $A$ to be squared and does not guarantee the eigenvectors to be orthogonal.

We strongly recommend the [Singular Value Decomposition (the SVD)](https://www.youtube.com/watch?v=mBcLRGuAFUk) by the amazing [Gilbert Strang](http://www-math.mit.edu/~gs/) if you would like to learn more.

## Coding Exercise 2: SVD (Optional)

The goal is to perform the SVD on $\mathbf{W}_{tot}$ in every epoch, and record the singular values (modes) during the training.

Complete the function `ex_net_svd`, by first calculating the $\mathbf{W}_{tot} = \prod_{i=1}^{L}{\mathbf{W}_{i}}$ and finally performing SVD on the $\mathbf{W}_{tot}$. Please use the PyTorch [`torch.svd`](https://pytorch.org/docs/stable/generated/torch.svd.html) instead of NumPy [`np.linalg.svd`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.svd.html).

```python
def ex_net_svd(model, in_dim):
  """Performs a Singular Value Decomposition on a given model weights

  Args:
    model (torch.nn.Module): neural network model
    in_dim (int): the input dimension of the model

  Returns:
    U, Σ, V (Tensors): Orthogonal, diagonal, and orthogonal matrices
  """
  W_tot = torch.eye(in_dim)
  for weight in model.parameters():
    #################################################
    ## Calculate the W_tot by multiplication of all weights
    # and then perform SVD on the W_tot using pytorch's `torch.svd`
    # Remember that weights need to be `.detach()` from the graph
    # Complete the function and remove or comment the line below
    raise NotImplementedError("Function `ex_net_svd`")
    #################################################
    W_tot = ...
  U, Σ, V = ...
  return U, Σ, V

#add event to airtable
atform.add_event('Coding Exercise 2: SVD')

## Uncomment and run
# test_net_svd_ex(SEED)
```

```python
# to_remove solution
def ex_net_svd(model, in_dim):
  """Performs a Singular Value Decomposition on a given model weights

  Args:
    model (torch.nn.Module): neural network model
    in_dim (int): the input dimension of the model

  Returns:
    U, Σ, V (Tensors): Orthogonal, diagonal, and orthogonal matrices
  """
  W_tot = torch.eye(in_dim)
  for weight in model.parameters():
    W_tot = weight @ W_tot
  U, Σ, V = torch.svd(W_tot)
  return U, Σ, V

#add event to airtable
atform.add_event('Coding Exercise 2: SVD')

## Uncomment and run
test_net_svd_ex(SEED)
```

```python
# @markdown #### Make sure you execute this cell to train the network and plot

lr = 100.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_hidden = 30  # hidden neurons
dim_output = 10000  # output dimension = `feature_tensor.size(1)`

# model instantiation
dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

# weights re-initialization
initializer_(dlnn_model, gamma)

# training
losses, modes, *_ = train(dlnn_model,
                          label_tensor,
                          feature_tensor,
                          n_epochs=n_epochs,
                          lr=lr)

plot_loss_sv_twin(losses, modes)
```

**Think!**

In EigenValue decomposition, the amount of variance explained by eigenvectors is proportional to the corresponding eigenvalues. What about the SVD? We see that the gradient descent guides the network to first learn the features that carry more information (have higher singular value)!

```python
# @title Video 3: SVD - Discussion
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1t54y1J7Tb", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"JEbRPPG2kUI", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 3: SVD - Discussion')

display(out)
```

---
# Section 3: Representational Similarity Analysis (RSA)

*Time estimate: ~20 mins*

```python
# @title Video 4: RSA
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV19f4y157zD", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"YOs1yffysX8", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 4: RSA')

display(out)
```

The previous section ended with an interesting remark. SVD helped to break our deep "wide" linear neural net into 8 deep "narrow" linear neural nets.

The first narrow net (highest singular value) converges fastest, while the last four narrow nets, converge almost simultaneously and have the smallest singular values. Can it be that the narrow net with larger mode is learning the difference between "living things" and "objects", while another narrow net with smaller mode is learning the difference between Fish and Birds? how could we check this hypothesis?

Representational Similarity Analysis (RSA) is an approach that could help us understand the internal representation of our network. The main idea is that the activity of hidden units (neurons) in the network must be similar when the network is presented with similar input. For our dataset (hierarchically structured data), we expect the activity of neurons in the hidden layer to be more similar for Tuna and Canary, and less similar for Tuna and Oak.

For similarity measure, we can use the good old dot (scalar) product, which is also called cosine similarity. For calculating the dot product between multiple vectors (which would be our case), we can simply use matrix multiplication. Therefore the Representational Similarity Matrix for multiple-input (batch) activity could be calculated as follow:

\begin{equation}
RSM = \mathbf{H} \mathbf{H}^{\top}
\end{equation}

where $\mathbf{H} = \mathbf{X} \mathbf{W_1}$ is the activity of hidden neurons for a given batch $\mathbf{X}$.

## Coding Exercise 3: RSA (Optional)

The task is simple. We would need to measure the similarity between hidden layer activities $~\mathbf{h} =  \mathbf{x} ~\mathbf{W_1}$) for every input $\mathbf{x}$.

If we perform RSA in every iteration, we could also see the evolution of representation learning.

```python
def ex_net_rsm(h):
  """Calculates the Representational Similarity Matrix

  Arg:
    h (torch.Tensor): activity of a hidden layer

  Returns:
    (torch.Tensor): Representational Similarity Matrix
  """
  #################################################
  ## Calculate the Representational Similarity Matrix
  # Complete the function and remove or comment the line below
  raise NotImplementedError("Function `ex_net_rsm`")
  #################################################
  rsm = ...
  return rsm

#add event to airtable
atform.add_event(' Coding Exercise 3: RSA')

## Uncomment and run
# test_net_rsm_ex(SEED)
```

```python
# to_remove solution
def ex_net_rsm(h):
  """Calculates the Representational Similarity Matrix

  Arg:
    h (torch.Tensor): activity of a hidden layer

  Returns:
    (torch.Tensor): Representational Similarity Matrix
  """
  rsm = h @ h.T
  return rsm

#add event to airtable
atform.add_event(' Coding Exercise 3: RSA')

## Uncomment and run
test_net_rsm_ex(SEED)
```

Now we can train the model while recording the losses, modes, and RSMs at every iteration. First, use the epoch slider to explore the evolution of RSM without changing default lr ($\eta$) and initialization ($\gamma$). Then, as we did before, set $\eta$ and $\gamma$ to larger values to see whether you can retrieve the sequential structured learning of representations.

```python
#@markdown #### Make sure you execute this cell to enable widgets

def loss_svd_rsm_lr_gamma(lr, gamma, i_ep):
  """
  Args:
    lr (float): learning rate
    gamma (float): initialization scale
    i_ep (int): which epoch to show

  """
  n_epochs = 250  # number of epochs
  dim_input = 8  # input dimension = `label_tensor.size(1)`
  dim_hidden = 30  # hidden neurons
  dim_output = 10000  # output dimension = `feature_tensor.size(1)`

  # model instantiation
  dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

  # weights re-initialization
  initializer_(dlnn_model, gamma)

  # training
  losses, modes, rsms, _ = train(dlnn_model,
                                 label_tensor,
                                 feature_tensor,
                                 n_epochs=n_epochs,
                                 lr=lr)
  plot_loss_sv_rsm(losses, modes, rsms, i_ep)

i_ep_slider = IntSlider(min=10, max=241, step=1, value=61,
                        continuous_update=False,
                        description='Epoch',
                        layout=Layout(width='630px'))

lr_slider = FloatSlider(min=20.0, max=200.0, step=1.0, value=100.0,
                        continuous_update=False,
                        readout_format='.1f',
                        description='eta')

gamma_slider = FloatLogSlider(min=-15, max=1, step=1,
                              value=1e-12, base=10,
                              continuous_update=False,
                              description='gamma')

widgets_ui = VBox([lr_slider, gamma_slider, i_ep_slider])

widgets_out = interactive_output(loss_svd_rsm_lr_gamma,
                                 {'lr': lr_slider,
                                  'gamma': gamma_slider,
                                  'i_ep': i_ep_slider})

display(widgets_ui, widgets_out)
```

Let's take a moment to analyze this more. A deep neural net is learning the representations, rather than a naive mapping (look-up table). This is thought to be the reason for deep neural nets supreme generalization and transfer learning ability. Unsurprisingly, neural nets with no hidden layer are incapable of representation learning, even with extremely small initialization.

```python
# @title Video 5: RSA - Discussion
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV18y4y1j7Xr", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"vprldATyq1o", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 5: RSA - Discussion')
display(out)
```

---
# Section 4: Illusory Correlations

*Time estimate: ~20-30 mins*

```python
# @title Video 6: Illusory Correlations
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1vv411E7Sq", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"RxsAvyIoqEo", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 6: Illusory Correlations')

display(out)
```

Let's recall the training loss curves. There was often a long plateau (where the weights are stuck at a saddle point), followed by a sudden drop. For very deep complex neural nets, such plateaus can take hours of training, and we are often tempted to stop the training, because we believe it is "as good as it gets"! Another side effect of "immature interruption" of training is the network finding (learning) illusory correlations.

To better understand this, let's do the next demonstration and exercise.

## Demonstration: Illusory Correlations

Our original dataset has 4 animals: Canary, Robin, Goldfish, and Tuna. These animals all have bones. Therefore if we include a "has bone" feature, the network would learn it at the second level (i.e. second bump, second mode convergence), when it learns the animal-plants distinction.

What if the dataset has Shark instead of Goldfish. Sharks don't have bones (their skeletons are made of cartilaginous, which is much lighter than true bone and more flexible). Then we will have a feature which is *True* (i.e. +1) for Tuna, Robin, and Canary, but *False* (i.e. -1) for all the plants and the shark! Let's see what the network does.

First, we add the new feature to the targets. We then start training our LNN and in every epoch, record the network prediction for "sharks having bones".

```python
# sampling new data from the tree
tree_labels, tree_features = generate_hsd()

# replacing Goldfish with Shark
item_names = ['Shark', 'Tuna', 'Robin', 'Canary',
              'Rose', 'Daisy', 'Pine', 'Oak']

# index of label to record
illusion_idx = 0  # Shark is the first element

# the new feature (has bones) vector
new_feature = [-1, 1, 1, 1, -1, -1, -1, -1]
its_label = 'has_bones'

# adding feature has_bones to the feature array
tree_features = add_feature(tree_features, new_feature)

# plotting
plot_tree_data(item_names, tree_features, its_label)
```

You can see the new feature shown in the last column of the plot above.

Now we can train the network on the new data, and record the network prediction (output) for Shark (indexed 0) label and "has bone" feature (last feature, indexed -1), during the training.

Here is the snippet from the training loop that keeps track of network prediction for `illusory_i`th label and last (`-1`) feature:

```python
pred_ij = predictions.detach()[illusory_i, -1]
```

```python
#@markdown #### Make sure you execute this cell to train the network and plot

# convert (cast) data from np.ndarray to torch.Tensor
label_tensor = torch.tensor(tree_labels).float()
feature_tensor = torch.tensor(tree_features).float()

lr = 100.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_hidden = 30  # hidden neurons
dim_output = feature_tensor.size(1)

# model instantiation
dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

# weights re-initialization
initializer_(dlnn_model, gamma)

# training
_, modes, _, ill_predictions = train(dlnn_model,
                                     label_tensor,
                                     feature_tensor,
                                     n_epochs=n_epochs,
                                     lr=lr,
                                     illusory_i=illusion_idx)

# a label for the plot
ill_label = f"Prediction for {item_names[illusion_idx]} {its_label}"

# plotting
plot_ills_sv_twin(ill_predictions, modes, ill_label)
```

It seems that the network starts by learning an "illusory correlation" that sharks have bones, and in later epochs, as it learns deeper representations, it can see (learn) beyond the illusory correlation. This is important to remember that we never presented the network with any data saying that sharks have bones.

## Exercise 4: Illusory Correlations

This exercise is just for you to explore the idea of illusory correlations. Think of medical, natural, or possibly social illusory correlations which can test the learning power of deep linear neural nets.

**important notes**: the generated data is independent of tree labels, therefore the names are just for convenience.

Here is our example for **Non-human Living things do not speak**. The lines marked by `{edit}` are for you to change in your example.

```python
# sampling new data from the tree
tree_labels, tree_features = generate_hsd()

# {edit} replacing Canary with Parrot
item_names = ['Goldfish', 'Tuna', 'Robin', 'Parrot',
              'Rose', 'Daisy', 'Pine', 'Oak']

# {edit} index of label to record
illusion_idx = 3  # Parrot is the fourth element

# {edit} the new feature (cannot speak) vector
new_feature = [1, 1, 1, -1, 1, 1, 1, 1]
its_label = 'cannot_speak'

# adding feature has_bones to the feature array
tree_features = add_feature(tree_features, new_feature)

# plotting
plot_tree_data(item_names, tree_features, its_label)
```

```python
# @markdown #### Make sure you execute this cell to train the network and plot

# convert (cast) data from np.ndarray to torch.Tensor
label_tensor = torch.tensor(tree_labels).float()
feature_tensor = torch.tensor(tree_features).float()

lr = 100.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_hidden = 30  # hidden neurons
dim_output = feature_tensor.size(1)

# model instantiation
dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

# weights re-initialization
initializer_(dlnn_model, gamma)

# training
_, modes, _, ill_predictions = train(dlnn_model,
                                     label_tensor,
                                     feature_tensor,
                                     n_epochs=n_epochs,
                                     lr=lr,
                                     illusory_i=illusion_idx)

# a label for the plot
ill_label = f"Prediction for {item_names[illusion_idx]} {its_label}"

# plotting
plot_ills_sv_twin(ill_predictions, modes, ill_label)
```

```python
# @title Video 7: Illusory Correlations - Discussion
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1vv411E7rg", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"6VLHKQjQJmI", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 7: Illusory Correlations')

display(out)
```

---
# Summary

The second day of the course has ended. So, in the third tutorial of the linear deep learning day we have learned more advanced topics. In the beginning we implemented a deep linear neural network and then we studied its learning dynamics using the linear algebra tool called singular value decomposition. Then, we learned about the representational similarity analysis and the illusory correlation.

```python
# @title Video 8: Outro
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1AL411n7ns", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"N2szOIsKyXE", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

#add event to airtable
atform.add_event('Video 8: Outro')

display(out)
```

```python
# @title Airtable Submission Link
from IPython import display as IPydisplay
IPydisplay.HTML(
   f"""
 <div>
   <a href= "{atform.url()}" target="_blank">
   </a>
   </div>""" )
```

---
# Bonus

*Time estimate: ~20-30 mins*

```python
# @title Video 9: Linear Regression
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1Pf4y1L71L", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"uULOAbhYaaE", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

## Section 5.1: Linear Regression

Generally, *regression* refers to a set of methods for modeling the mapping (relationship) between one (or more) independent variable(s) (i.e., features) and one (or more) dependent variable(s) (i.e., labels). For example, if we want to examine the relative impacts of calendar date, GPS coordinates, and time of the say (the independent variables) on air temperature (the dependent variable). On the other hand, regression can be used for predictive analysis. Thus the independent variables are also called predictors. When the model contains more than one predictor, then the method is called *multiple regression*, and if it contains more than one dependent variable called *multivariate regression*. Regression problems pop up whenever we want to predict a numerical (usually continuous) value.

The independent variables are collected in vector $\mathbf{x} \in \mathbb{R}^M$, where $M$ denotes the number of independent variables, while the dependent variables are collected in vector $\mathbf{y} \in \mathbb{R}^N$, where $N$ denotes the number of dependent variables. And the mapping between them is represented by the weight matrix $\mathbf{W} \in \mathbb{R}^{N \times M}$ and a bias vector $\mathbf{b} \in \mathbb{R}^{N}$ (generalizing to affine mappings).

The multivariate regression model can be written as:

\begin{equation}
\mathbf{y} = \mathbf{W} ~ \mathbf{x} + \mathbf{b}
\end{equation}

or it can be written in matrix format as:

\begin{equation}
\begin{bmatrix} y_{1} \\ y_{2} \\ \vdots \\ y_{N} \\ \end{bmatrix} = \begin{bmatrix} w_{1,1} & w_{1,2} & \dots & w_{1,M} \\ w_{2,1} & w_{2,2} & \dots & w_{2,M} \\ \vdots & \ddots & \ddots & \vdots \\ w_{N,1} & w_{N,2} & \dots & w_{N,M} \end{bmatrix} \begin{bmatrix} x_{1} \\ x_{2} \\ \vdots \\ x_{M} \\ \end{bmatrix} + \begin{bmatrix} b_{1} \\ b_{2} \\ \vdots \\b_{N} \\ \end{bmatrix}
\end{equation}

## Section 5.2: Vectorized regression

Linear regression can be simply extended to multi-samples ($D$) input-output mapping, which we can collect in a matrix $\mathbf{X} \in \mathbb{R}^{M \times D}$, sometimes called the design matrix. The sample dimension also shows up in the output matrix $\mathbf{Y} \in \mathbb{R}^{N \times D}$. Thus, linear regression takes the following form:

\begin{equation}
\mathbf{Y} = \mathbf{W} ~ \mathbf{X} + \mathbf{b}
\end{equation}

where matrix $\mathbf{W} \in \mathbb{R}^{N \times M}$ and the vector $\mathbf{b} \in \mathbb{R}^{N}$ (broadcasted over sample dimension) are the desired parameters to find.

## Section 5.3: Analytical Linear Regression

Linear regression is a relatively simple optimization problem. Unlike most other models that we will see in this course, linear regression for mean squared loss can be solved analytically.

For $D$ samples (batch size), $\mathbf{X} \in \mathbb{R}^{M \times D}$, and $\mathbf{Y} \in \mathbb{R}^{N \times D}$, the goal of linear regression is to find $\mathbf{W} \in \mathbb{R}^{N \times M}$ such that:

\begin{equation}
\mathbf{Y} = \mathbf{W} ~ \mathbf{X}
\end{equation}

Given the Squared Error loss function, we have:

\begin{equation}
Loss(\mathbf{W}) = ||\mathbf{Y} - \mathbf{W} ~ \mathbf{X}||^2
\end{equation}

So, using matrix notation, the optimization problem is given by:

\begin{align}
\mathbf{W^{*}} &= \underset{\mathbf{W}}{\mathrm{argmin}} \left( Loss (\mathbf{W})  \right) \\
 &= \underset{\mathbf{W}}{\mathrm{argmin}} \left( ||\mathbf{Y} - \mathbf{W} ~ \mathbf{X}||^2  \right) \\
&= \underset{\mathbf{W}}{\mathrm{argmin}} \left( \left( \mathbf{Y} - \mathbf{W} ~ \mathbf{X}\right)^{\top} \left( \mathbf{Y} - \mathbf{W} ~ \mathbf{X}\right) \right)
\end{align}

To solve the minimization problem, we can simply set the derivative of the loss with respect to $\mathbf{W}$ to zero.

\begin{equation}
\dfrac{\partial Loss}{\partial \mathbf{W}} = 0
\end{equation}

Assuming that $\mathbf{X}\mathbf{X}^{\top}$ is full-rank, and thus it is invertible we can write:

\begin{equation}
\mathbf{W}^{\mathbf{*}} = \mathbf{Y} \mathbf{X}^{\top} \left( \mathbf{X}  \mathbf{X}^{\top} \right) ^{-1}
\end{equation}

### Coding Exercise 5.3.1: Analytical solution to LR

Complete the function `linear_regression` for finding the analytical solution to linear regression.

```python
def linear_regression(X, Y):
  """Analytical Linear regression

  Args:
    X (np.ndarray): design matrix
    Y (np.ndarray): target ouputs

  return:
    np.ndarray: estimated weights (mapping)
  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dx == Dy
  #################################################
  ## Complete the linear_regression_exercise function
  # Complete the function and remove or comment the line below
  raise NotImplementedError("Linear Regression `linear_regression`")
  #################################################
  W = ...

  return W

W_true = np.random.randint(low=0, high=10, size=(3, 3)).astype(float)

X_train = np.random.rand(3, 37)  # 37 samples
noise = np.random.normal(scale=0.01, size=(3, 37))
Y_train = W_true @ X_train + noise

## Uncomment and run
# W_estimate = linear_regression(X_train, Y_train)
# print(f"True weights:\n {W_true}")
# print(f"\nEstimated weights:\n {np.round(W_estimate, 1)}")
```

```python
# to_remove solution
def linear_regression(X, Y):
  """Analytical Linear regression

  Args:
    X (np.ndarray): design matrix
    Y (np.ndarray): target ouputs

  return:
    np.ndarray: estimated weights (mapping)
  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dx == Dy

  W = Y @ X.T @ np.linalg.inv(X @ X.T)

  return W

W_true = np.random.randint(low=0, high=10, size=(3, 3)).astype(float)

X_train = np.random.rand(3, 37)  # 37 samples
noise = np.random.normal(scale=0.01, size=(3, 37))
Y_train = W_true @ X_train + noise

## Uncomment and run
W_estimate = linear_regression(X_train, Y_train)
print(f"True weights:\n {W_true}")
print(f"\nEstimated weights:\n {np.round(W_estimate, 1)}")
```

## Demonstration: Linear Regression vs. DLNN

A linear neural network with NO hidden layer is very similar to linear regression in its core. We also know that no matter how many hidden layers a linear network has, it can be compressed to linear regression (no hidden layers).

In this demonstration, we use the hierarchically structured data to:

* analytically find the mapping between features and labels
* train a zero-depth LNN to find the mapping
* compare them to the $W_{tot}$ from the already trained deep LNN

```python
# sampling new data from the tree
tree_labels, tree_features = generate_hsd()

# convert (cast) data from np.ndarray to torch.Tensor
label_tensor = torch.tensor(tree_labels).float()
feature_tensor = torch.tensor(tree_features).float()
```

```python
# calculating the W_tot for deep network (already trained model)

lr = 100.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_hidden = 30  # hidden neurons
dim_output = 10000  # output dimension = `feature_tensor.size(1)`

# model instantiation
dlnn_model = LNNet(dim_input, dim_hidden, dim_output)

# weights re-initialization
initializer_(dlnn_model, gamma)

# training
losses, modes, rsms, ills = train(dlnn_model,
                                  label_tensor,
                                  feature_tensor,
                                  n_epochs=n_epochs,
                                  lr=lr)

deep_W_tot = torch.eye(dim_input)
for weight in dlnn_model.parameters():
  deep_W_tot = weight @ deep_W_tot
deep_W_tot = deep_W_tot.detach().numpy()
```

```python
# analytically estimation of weights
# our data is batch first dimension, so we need to transpose our data
analytical_weights = linear_regression(tree_labels.T, tree_features.T)
```

```python
class LRNet(nn.Module):
  """A Linear Neural Net with ZERO hidden layer (LR net)
  """

  def __init__(self, in_dim, out_dim):
    """
    Args:
      in_dim (int): input dimension
      hid_dim (int): hidden dimension
    """
    super().__init__()
    self.in_out = nn.Linear(in_dim, out_dim, bias=False)

  def forward(self, x):
    """
    Args:
      x (torch.Tensor): input tensor
    """
    out = self.in_out(x)  # output (prediction)
    return out
```

```python
lr = 1000.0  # learning rate
gamma = 1e-12  # initialization scale
n_epochs = 250  # number of epochs
dim_input = 8  # input dimension = `label_tensor.size(1)`
dim_output = 10000  # output dimension = `feature_tensor.size(1)`

# model instantiation
LR_model = LRNet(dim_input, dim_output)
optimizer = optim.SGD(LR_model.parameters(), lr=lr)
criterion = nn.MSELoss()

losses = np.zeros(n_epochs)  # loss records
for i in range(n_epochs):  # training loop
  optimizer.zero_grad()
  predictions = LR_model(label_tensor)
  loss = criterion(predictions, feature_tensor)
  loss.backward()
  optimizer.step()
  losses[i] = loss.item()

# trained weights from zero_depth_model
LR_model_weights = next(iter(LR_model.parameters())).detach().numpy()

plot_loss(losses, "Training loss for zero depth LNN", c="r")
```

```python
print("The final weights from all methods are approximately equal?! "
"{}!".format(
  (np.allclose(analytical_weights, LR_model_weights, atol=1e-02) and \
   np.allclose(analytical_weights, deep_W_tot, atol=1e-02))
  )
)
```

As you may have guessed, they all arrive at the same results but through very different paths.

```python
# @title Video 10: Linear Regression - Discussion
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
      self.id=id
      src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
      super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV18v411E7Wg", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"gG15_J0i05Y", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```