# DL Neuromatch Academy: Week 1, Day 2, Tutorial 3
# Deep Linear Neural Networks

__Content creators:__ Andrew Saxe, Saeed Salehi, Vladimir Haltakov

__Content reviewers:__ Polina Turishcheva, Atnafu Lambebo, Yu-Fang Yang

__Content editors:__ Anoop Kulkarni

__Production editors:__ Khalid Almubarak, , Spiros Chavlis

---
#Tutorial Objectives

* Deep linear neural networks
* Learning dynamics and singular value decomposition
* Representational Similarity Analysis
* Illusory correlations & ethics.

```python
#@markdown Tutorial slides
# you should link the slides for all tutorial videos here (we will store pdfs on osf)

from IPython.display import HTML
HTML('')
```

---
# Setup

```python
# Imports

import numpy as np
import matplotlib.pyplot as plt
import random
from collections import OrderedDict

import torch
import torch.nn as nn
import torch.optim as optim

from tqdm.notebook import tqdm, trange
import time
from math import sqrt

! pip install treelib --quiet
from treelib import Node, Tree
```

      Building wheel for treelib (setup.py) ... done

```python
# @title Figure settings
# import ipywidgets as widgets
# from ipywidgets import interact, fixed, HBox, Layout, VBox, interactive, Label
# from ipywidgets import interact, IntSlider, FloatSlider, interact_manual
# from mpl_toolkits.mplot3d import Axes3D
from matplotlib import gridspec
from ipywidgets import interact, IntSlider, FloatSlider, interact_manual, fixed
from ipywidgets import FloatLogSlider, HBox, Layout, VBox, interactive, Label
from ipywidgets import interactive_output

import warnings
warnings.filterwarnings("ignore")

%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
#@title Plotting functions

def plot_x_y_hier_data(im1, im2, subplot_ratio=[1, 2]):
  fig = plt.figure(figsize=(12, 5))
  gs = gridspec.GridSpec(1, 2, width_ratios=subplot_ratio)
  ax0 = plt.subplot(gs[0])
  ax1 = plt.subplot(gs[1])
  ax0.imshow(im1, cmap="cool")
  ax1.imshow(im2, cmap="cool")
  # plt.suptitle("The whole dataset as imshow plot", y=1.02)
  ax0.set_title("Labels of all samples")
  ax1.set_title("Features of all samples")
  ax0.set_axis_off()
  ax1.set_axis_off()
  plt.tight_layout()
  plt.show()

def plot_x_y_hier_one(im1, im2, subplot_ratio=[1, 2]):
  fig = plt.figure(figsize=(12, 1))
  gs = gridspec.GridSpec(1, 2, width_ratios=subplot_ratio)
  ax0 = plt.subplot(gs[0])
  ax1 = plt.subplot(gs[1])
  ax0.imshow(im1, cmap="cool")
  ax1.imshow(im2, cmap="cool")
  ax0.set_title("Labels of a single sample")
  ax1.set_title("Features of a single sample")
  ax0.set_axis_off()
  ax1.set_axis_off()
  plt.tight_layout()
  plt.show()

def plot_tree_data(im1, im2, label_list):
  im1_dim1, im1_dim2 = im1.shape

  fig = plt.figure(figsize=(12, 5))
  gs = gridspec.GridSpec(1, 2, width_ratios=[1, 2])
  ax0 = plt.subplot(gs[0])
  ax1 = plt.subplot(gs[1])
  ax0.imshow(im1, cmap="cool")
  ax1.imshow(im2[:, -im1_dim1*2:], cmap="cool", vmin=0.0, vmax=1.0)
  ax0.set_title("all the Labels")
  ax1.set_title("last {} Features".format(-im1_dim1*2))
  ax0.set_yticks(ticks=np.arange(im1_dim1))
  ax0.set_yticklabels(labels=label_list)
  ax0.set_xticks(ticks=np.arange(im1_dim1))
  ax0.set_xticklabels(labels=item_names, rotation='vertical')
  ax1.set_axis_off()
  plt.tight_layout()
  plt.show()

def plot_loss(loss_array, title="Training loss (Mean Squared Error)", c="r"):
  plt.figure(figsize=(9, 5))
  plt.plot(loss_array, color=c)
  plt.xlabel("Epoch")
  plt.ylabel("MSE")
  plt.title(title)
  plt.show()

def plot_loss_sv(loss_array, sv_array):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("Set1", n_sing_values)

  _, (plot1, plot2) = plt.subplots(2, 1, sharex=True, figsize=(10, 10))
  plot1.set_title("Training loss (Mean Squared Error)")
  plot1.plot(loss_array, color='r')

  plot2.set_title("Evolution of singular values (modes)")
  for i in range(n_sing_values):
    plot2.plot(sv_array[:, i], c=cmap(i))
  plot2.set_xlabel("Epoch")
  plt.show()

def plot_loss_sv_twin(loss_array, sv_array):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(11, 6))
  ax1 = plt.gca()
  ax1.set_title("Learning Dynamics")
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel("Mean Squared Error", c='r')
  ax1.tick_params(axis='y', labelcolor='r')
  ax1.plot(loss_array, color='r')

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values (modes)", c='b')
  ax2.tick_params(axis='y', labelcolor='b')
  for i in range(n_sing_values):
    ax2.plot(sv_array[:, i], c=cmap(i))

  fig.tight_layout()
  plt.show()

def plot_ills_sv_twin(ill_array, sv_array):
  n_sing_values = sv_array.shape[1]
  sv_array = sv_array / np.max(sv_array)
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(11, 6))
  ax1 = plt.gca()
  ax1.set_title("Network evolution in learning the Illusory Correlations")
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel("Illusory Correlations", c='r')
  ax1.tick_params(axis='y', labelcolor='r')
  ax1.plot(ill_array, color='r', linewidth=3)
  ax1.set_ylim(-0.05, 1.0)

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values (modes)", c='b')
  ax2.tick_params(axis='y', labelcolor='b')
  for i in range(n_sing_values):
    ax2.plot(sv_array[:, i], c=cmap(i))

  fig.tight_layout()
  plt.show()

def plot_loss_sv_rsm(loss_array, sv_array, rsm_array, i_ep):
  rsm_array = rsm_array / np.max(rsm_array, axis=0)
  sv_array = sv_array / np.max(sv_array)

  n_sing_values = sv_array.shape[1]
  cmap = plt.cm.get_cmap("winter", n_sing_values)

  fig = plt.figure(figsize=(15, 5))
  gs = gridspec.GridSpec(1, 2, width_ratios=[2, 1])

  ax0 = plt.subplot(gs[1])
  ax0.yaxis.tick_right()
  ax0.imshow(rsm_array[i_ep], cmap="Purples", vmin=0.0, vmax=1.1)
  ax0.set_title("RSM at epoch {}".format(i_ep), fontsize=16)
  # ax0.set_axis_off()
  ax0.set_yticks(ticks=np.arange(n_sing_values))
  ax0.set_yticklabels(labels=item_names)
  # ax0.set_xticks([])
  ax0.set_xticks(ticks=np.arange(n_sing_values))
  ax0.set_xticklabels(labels=item_names, rotation='vertical')

  ax1 = plt.subplot(gs[0])
  ax1.set_title("Learning Dynamics", fontsize=16)
  ax1.set_xlabel("Epoch")
  ax1.set_ylabel("Mean Squared Error", c='r')
  ax1.tick_params(axis='y', labelcolor='r')
  ax1.plot(loss_array, color='r')
  ax1.axvspan(i_ep-2, i_ep+2, alpha=0.2, color='m')

  ax2 = ax1.twinx()
  ax2.set_ylabel("Singular values", c='b')
  ax2.tick_params(axis='y', labelcolor='b')
  for i in range(n_sing_values):
    ax2.plot(sv_array[:, i], c=cmap(i))

  plt.show()

class SimpleTree:
  def __init__(self, plot=False):
    tree = Tree()
    tree.create_node("Living things", 0)
    tree.create_node("Animal", 1, parent=0)
    tree.create_node("Plant", 2, parent=0)
    tree.create_node("Fish", 3, parent=1)
    tree.create_node("Bird", 4, parent=1)
    tree.create_node("Flower", 5, parent=2)
    tree.create_node("Tree", 6, parent=2)
    tree.create_node("Goldfish", 7, parent=3)
    tree.create_node("Tuna", 8, parent=3)
    tree.create_node("Robin", 9, parent=4)
    tree.create_node("Canary", 10, parent=4)
    tree.create_node("Rose", 11, parent=5)
    tree.create_node("Daisy", 12, parent=5)
    tree.create_node("Pine", 13, parent=6)
    tree.create_node("Oak", 14, parent=6)
    self.tree = tree
    if plot: self.plot()

  def plot(self):
    self.tree.show(line_type="ascii-em")

  def rename(self, old, new, plot=False):
    for nodes in self.tree.all_nodes():
      if nodes.tag == old:
        nodes.tag = new
        break
    if plot: self.plot()
```

```python
#@title Helper functions

seed = 2015  # LeCun, Bengio, & Hinton (2015)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.cuda.manual_seed(seed)
np.random.seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

class VariableDepthWidth(nn.Module):
  def __init__(self, in_dim, out_dim, hid_dims=[], gamma=1e-12):
    """Variable depth linear network

    Args:
      in_dim (int): input dimension
      out_dim (int): ouput dimension
      hid_dims (list): a list, containing the number of neurons in each hidden layer
        default is empty list (`[]`) for linear regression.
        example: For 2 hidden layers, first with 5 and second with 7 neurons,
                 we use: `hid_dims = [5, 7]`
    """
    super().__init__()
    assert isinstance(in_dim, int)
    assert isinstance(out_dim, int)
    assert isinstance(hid_dims, list)
    n_hidden_layers = len(hid_dims)  # number of hidden layers
    layers = OrderedDict()

    if n_hidden_layers == 0:  # linear regression
      layers["map"] = nn.Linear(in_dim, out_dim, bias=False)

    else:  # shallow and deep linear neural net
      layers["in->"] = nn.Linear(in_dim, hid_dims[0], bias=False)

      for i in range(n_hidden_layers-1):  # creating hidden layers
        layers["hid {}".format(i+1)] = nn.Linear(hid_dims[i],
                                                    hid_dims[i+1],
                                                    bias=False)

      layers["->out"] = nn.Linear(hid_dims[-1], out_dim, bias=False)

    for k in layers:  # re-initialization of the weights
      sigma = gamma / sqrt(layers[k].weight.shape[0] + layers[k].weight.shape[1])
      nn.init.normal_(layers[k].weight, std=sigma)

    self.layers = nn.Sequential(layers)

  def forward(self, input_tensor):
    """Forward pass
    """
    return self.layers(input_tensor)

def build_tree(n_levels, n_branches, probability, to_np_array=True):
  """Builds a tree
  """
  assert 0.0 <= probability <= 1.0

  tree = {}

  tree["level"] = [0]
  for i in range(1, n_levels+1):
    tree["level"].extend([i]*(n_branches**i))

  tree["pflip"] = [probability]*len(tree["level"])

  tree["parent"] = [None]
  k = len(tree["level"])-1
  for j in range(k//n_branches):
    tree["parent"].extend([j]*n_branches)

  if to_np_array:
    tree["level"] = np.array(tree["level"])
    tree["pflip"] = np.array(tree["pflip"])
    tree["parent"] = np.array(tree["parent"])

  return tree

def sample_from_tree(tree, n):
  """ Generates n samples from a tree
  """
  items = [i for i, v in enumerate(tree["level"]) if v == max(tree["level"])]
  n_items = len(items)
  x = np.zeros(shape=(n, n_items))
  rand_temp = np.random.rand(n, len(tree["pflip"]))
  flip_temp = np.repeat(tree["pflip"].reshape(1, -1), n, 0)
  samp = (rand_temp > flip_temp) * 2 - 1

  for i in range(n_items):
    j = items[i]
    prop = samp[:, j]
    while tree["parent"][j] is not None:
      j = tree["parent"][j]
      prop = prop * samp[:, j]
    x[:, i] = prop.T
  return x

def generate_hsd():
  # building the tree
  n_branches = 2  # 2 branches at each node
  probability = .15  # flipping probability
  n_levels = 3  # number of levels (depth of tree)
  tree = build_tree(n_levels, n_branches, probability, to_np_array=True)
  tree["pflip"][0] = 0.5
  n_samples = 10000 # Sample this many features

  tree_labels = np.eye(n_branches**n_levels)
  tree_features = sample_from_tree(tree, n_samples).T
  return tree_labels, tree_features

def linear_regression(X, Y):
  """Analytical Linear regression

  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dy == Dy
  W = Y @ X.T @ np.linalg.inv(X @ X.T)
  return W

# #@markdown Run this cell to define the train function!

def train_svd_rsa_track(model, in_features, out_features, n_epochs, lr, ill_i=0):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    lr (float): learning rate
    ill_i (int): index of illusory feature

  Returns:
    np.ndarray: record (evolution) of losses
    np.ndarray: record (evolution) of singular values
    np.ndarray: record (evolution) of representational similarity matrices
    np.ndarray: record of network prediction for the last feature
  """

  assert in_features.shape[0] == out_features.shape[0]
  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()
  xd = in_features.shape[1]
  loss_record = []  # losses
  sv_record = []  # singular values
  rsm_record = []  # represent sim mats
  pred_record = []  # network prediction for the last feature

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    # calculating the W_tot by multiplying all layers' weights
    W_tot = model.layers[-1].weight.detach()  # starting from the last layer
    for i in range(2, len(model.layers)+1):
      W_tot = W_tot @ model.layers[-i].weight.detach()
    U, Σ, V = torch.svd(W_tot)  # performing the SVD!

    # calculating representational similarity matrix
    H1 = model.layers[0].weight.detach() @ in_features
    RSM = H1.T @ H1

    # network prediction of ill_i in_feature for the last feature
    ill_pred = y_pred[ill_i, -1].detach().numpy()

    loss_record.append(loss.item())
    sv_record.append(Σ.numpy())
    rsm_record.append(RSM.numpy())
    pred_record.append(ill_pred)

  return np.array(loss_record), np.array(sv_record), np.array(rsm_record), np.array(pred_record)

def add_feature(existing_features, new_feature):
  assert isinstance(existing_features, np.ndarray)
  assert isinstance(new_feature, list)
  new_feature = np.array([new_feature]).T
  # return np.hstack((tree_features, new_feature*2-1))
  return np.hstack((tree_features, new_feature))
```

---
# Section 0: Prelude

## Exercise 0: Variable depth and width LNN
Throughout this tutorial, we will need several neural nets with different depth and width. So first, let's create a model with variable depth and width.

This can be easily done by using [`OrderedDict()`](https://docs.python.org/3/library/collections.html#collections.OrderedDict) and `nn.Sequential` function. The model is defined by its input and output dimensions, and a list containing the width of each hidden layer. If the list is left empty, the neural network will perform a linear regression (coming up next). We also exclude the `bias` from all the layers.

We also take over the initialization. In PyTorch, we can use [`nn.init`](https://pytorch.org/docs/stable/nn.init.html) to initialize tensors from a given distribution. Here, we sample the weights from the following distribution:

$$\mathcal{N}\left(\mu=0, ~~\sigma=\gamma \sqrt{\dfrac{1}{n_{in} + n_{out}}} \right)$$

where $\gamma$ is given as an argument. the Underscore ("_") in `nn.init.random_` and other functions, denote "in-place" operation. Note that `nn.Linear` layers are initialized at definition, so we re-initialize them.

```python
class VariableDepthWidthExercise(nn.Module):
  def __init__(self, in_dim, out_dim, hid_dims=[], gamma=1e-12):
    """Variable depth linear network

    Args:
      in_dim (int): input dimension
      out_dim (int): ouput dimension
      hid_dims (list): a list, containing the number of neurons in each hidden layer
        default is empty list (`[]`) for linear regression.
        example: For 2 hidden layers, first with 5 and second with 7 neurons,
                 we use: `hid_dims = [5, 7]`
    """
    super().__init__()
    assert isinstance(in_dim, int)
    assert isinstance(out_dim, int)
    assert isinstance(hid_dims, list)
    n_hidden_layers = len(hid_dims)  # number of hidden layers
    layers = OrderedDict()

    if n_hidden_layers == 0:  # linear regression
      layers["map"] = nn.Linear(in_dim, out_dim, bias=False)

    else:  # shallow and deep linear neural net
      layers["in->"] = nn.Linear(in_dim, hid_dims[0], bias=False)

      for i in range(n_hidden_layers-1):  # creating hidden layers
        #################################################
        ## Complete the hidden loop of DeepLNNExercise class
        # Complete the function and remove or comment the line below
        raise NotImplementedError("Network model `DeepLNNExercise`")
        #################################################

    for k in layers:  # re-initialization of the weights
      sigma = gamma / sqrt(layers[k].weight.shape[0] + layers[k].weight.shape[1])
      nn.init.normal_(layers[k].weight, std=sigma)

    self.layers = nn.Sequential(layers)

  def forward(self, input_tensor):
    """Forward pass
    """
    return self.layers(input_tensor)

# # Uncomment and run
# print("Deep LNN:\n",
#       VariableDepthWidthExercise(64, 100, [32, 16, 16, 32]))

# print("\nLinear Regression model:\n",
#       VariableDepthWidthExercise(64, 100,[]))
```

```python
# to_remove solution
class VariableDepthWidthExercise(nn.Module):
  def __init__(self, in_dim, out_dim, hid_dims=[], gamma=1e-12):
    """Variable depth linear network

    Args:
      in_dim (int): input dimension
      out_dim (int): ouput dimension
      hid_dims (list): a list, containing the number of neurons in each hidden layer
        default is empty list (`[]`) for linear regression.
        example: For 2 hidden layers, first with 5 and second with 7 neurons,
                 we use: `hid_dims = [5, 7]`
    """
    super().__init__()
    assert isinstance(in_dim, int)
    assert isinstance(out_dim, int)
    assert isinstance(hid_dims, list)
    n_hidden_layers = len(hid_dims)  # number of hidden layers
    layers = OrderedDict()

    if n_hidden_layers == 0:  # linear regression
      layers["map"] = nn.Linear(in_dim, out_dim, bias=False)

    else:  # shallow and deep linear neural net
      layers["in->"] = nn.Linear(in_dim, hid_dims[0], bias=False)

      for i in range(n_hidden_layers-1):  # creating hidden layers
        layers["hid {}".format(i+1)] = nn.Linear(hid_dims[i],
                                                    hid_dims[i+1],
                                                    bias=False)

      layers["->out"] = nn.Linear(hid_dims[-1], out_dim, bias=False)

    for k in layers:  # re-initialization of the weights
      sigma = gamma / sqrt(layers[k].weight.shape[0] + layers[k].weight.shape[1])
      nn.init.normal_(layers[k].weight, std=sigma)

    self.layers = nn.Sequential(layers)

  def forward(self, input_tensor):
    """Forward pass
    """
    return self.layers(input_tensor)

# # Uncomment and run
print("Deep LNN:\n",
      VariableDepthWidthExercise(64, 100, [32, 16, 16, 32]))

print("\nLinear Regression model:\n",
      VariableDepthWidthExercise(64, 100,[]))
```

    Deep LNN:
     VariableDepthWidthExercise(
      (layers): Sequential(
        (in->): Linear(in_features=64, out_features=32, bias=False)
        (hid 1): Linear(in_features=32, out_features=16, bias=False)
        (hid 2): Linear(in_features=16, out_features=16, bias=False)
        (hid 3): Linear(in_features=16, out_features=32, bias=False)
        (->out): Linear(in_features=32, out_features=100, bias=False)
      )
    )

    Linear Regression model:
     VariableDepthWidthExercise(
      (layers): Sequential(
        (map): Linear(in_features=64, out_features=100, bias=False)
      )
    )

We have prepare the training function (very similar to that of tutorial 1) for you already. Just look check if everything is okay.

```python
def train(model, in_features, out_features, n_epochs, lr,
          criterion=None, optimizer=None, show_progress_bar=False):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    criterion (function): loss function (default 'nn.MSELoss()')
    optimizer(function): optimizer (default 'optim.SGD')
    lr(float): learning rate

  Returns:
    list: record (evolution) of losses
  """

  assert in_features.shape[0] == out_features.shape[0]
  loss_record = []  # for recoding losses

  if optimizer is None:
      optimizer = optim.SGD(model.parameters(), lr=lr)
  if criterion is None:
      criterion = nn.MSELoss()

  model.train()  # we first put the model in training mode

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    loss_record.append(loss.item())

  model.eval()  # putting the model to evaluation mode

  return loss_record
```

---
# Section 00: Analytical Linear Regression

Linear regression is a relatively simple optimization problem. Unlike most other models that we will see in this course, linear regression for mean squared loss can be solved analytically.

For $D$ samples (batch size), $\mathbf{X} \in \mathbb{R}^{M \times D}$, and $\mathbf{Y} \in \mathbb{R}^{N \times D}$, the goal of linear regression is to find $\mathbf{W} \in \mathbb{R}^{N \times M}$ such that:

$$\mathbf{Y} = \mathbf{W} ~ \mathbf{X} $$

Given the Squared Error loss function, we have:

\begin{equation}
Loss(\mathbf{W}) = ||\mathbf{Y} - \mathbf{W} ~ \mathbf{X}||^2
\end{equation}

So, using matrix notation, the optimization problem is given by:

\begin{align}
\mathbf{W^{*}} &= \underset{\mathbf{W}}{\mathrm{argmin}} \left( Loss (\mathbf{W})  \right) \\
 &= \underset{\mathbf{W}}{\mathrm{argmin}} \left( ||\mathbf{Y} - \mathbf{W} ~ \mathbf{X}||^2  \right) \\
&= \underset{\mathbf{W}}{\mathrm{argmin}} \left( \left( \mathbf{Y} - \mathbf{W} ~ \mathbf{X}\right)^{\top} \left( \mathbf{Y} - \mathbf{W} ~ \mathbf{X}\right) \right)
\end{align}

To solve the minimization problem, we can simply set the derivative of the loss with respect to $\mathbf{W}$ to zero.

\begin{equation}
\dfrac{\partial Loss}{\partial \mathbf{W}} = 0
\end{equation}

Assuming that $\mathbf{X}\mathbf{X}^{\top}$ is full-rank, and thus it is invertible we can write:

\begin{equation}
\mathbf{W}^{\mathbf{*}} = \mathbf{Y} \mathbf{X}^{\top} \left( \mathbf{X}  \mathbf{X}^{\top} \right) ^{-1}
\end{equation}

## Exercise 00: Analytical solution to LR

Complete the function `linear_regression` for finding the analytical solution to linear regression.

```python
def linear_regression_exercise(X, Y):
  """Analytical Linear regression

  Args:
    X (np.ndarray): design matrix
    Y (np.ndarray): target ouputs

  return:
    np.ndarray: estimated weights (mapping)
  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dy == Dy

  #################################################
  ## Complete the linear_regression_exercise function
  # Complete the function and remove or comment the line below
  raise NotImplementedError("Linear Regression `linear_regression_exercise`")
  #################################################
  W = ...

  return W

W_true = np.random.randint(low=0, high=10, size=(3, 3)).astype(float)

X_train = np.random.rand(3, 37)  # 37 samples
noise = np.random.normal(scale=0.01, size=(3, 37))
Y_train = W_true @ X_train + noise

# # Uncomment and run
# W_estimate = linear_regression_exercise(X_train, Y_train)

# print("True weights:\n", W_true)
# print("\nEstimated weights:\n", np.round(W_estimate, 1))
```

```python
# to_remove solution
def linear_regression_exercise(X, Y):
  """Analytical Linear regression

  Args:
    X (np.ndarray): design matrix
    Y (np.ndarray): target ouputs

  return:
    np.ndarray: estimated weights (mapping)
  """
  assert isinstance(X, np.ndarray)
  assert isinstance(Y, np.ndarray)
  M, Dx = X.shape
  N, Dy = Y.shape
  assert Dy == Dy

  W = Y @ X.T @ np.linalg.inv(X @ X.T)

  return W

W_true = np.random.randint(low=0, high=10, size=(3, 3)).astype(float)

X_train = np.random.rand(3, 37)  # 37 samples
noise = np.random.normal(scale=0.01, size=(3, 37))
Y_train = W_true @ X_train + noise

# # Uncomment and run
W_estimate = linear_regression_exercise(X_train, Y_train)

print("True weights:\n", W_true)
print("\nEstimated weights:\n", np.round(W_estimate, 1))
```

    True weights:
     [[8. 1. 5.]
     [3. 8. 6.]
     [9. 5. 9.]]

    Estimated weights:
     [[8. 1. 5.]
     [3. 8. 6.]
     [9. 5. 9.]]

---
# Section 1: Deep Linear Neural Nets

```python
#@title Video 1: Representation Learning (Intro)
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="MRPy6uZRxms", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/MRPy6uZRxms

So far depth just seems to slow down the learning. And we know that a single nonlinear hidden layer (given enough number of neurons) has the potential to approximate any function. So it's seems fair to ask: **What is depth good for**? One reason can be that, shallow nonlinear neural networks hardly meet their true potential in practice.

In the contrast, deep neural nets are often surprisingly powerful in learning complex functions without sacrificing the generalization. A core intuition behind deep learning is that deep nets derive their power through learning internal representations. How does this work? To address representation learning, we have to go beyond the 1D chain, to a deep Linear Neual Network (LNN).

For this and the next couple of exercises, we use syntactically generated hierarchically structured data through a *branching diffusion process* (see [this reference](https://www.pnas.org/content/pnas/suppl/2019/05/16/1820226116.DCSupplemental/pnas.1820226116.sapp.pdf) for more details).

 hierarchically structured data

## Exercise 1: Training a deep LNN

This is a rather simple exercise. We will generate some hierarchically structured data, instantiate a LNN from `VariableDepthWidth` class and train it on the data.

**Important note**:

* Datasets are often generated as `numpy.ndarray`, and pass to PyTorch, which needs `torch.Tensor` for training. You can use `torch.tensor(toy_data).float()` for the "conversion" of float datatype.

```python
#@markdown #### Run to generate and visualize training samples from tree

tree_labels, tree_features = generate_hsd()
item_names = ['Goldfish', 'Tuna', 'Robin', 'Canary', 'Rose', 'Daisy', 'Pine', 'Oak']
plot_tree_data(tree_labels, tree_features, item_names)

# dimensions
print()
print("---------------------------------------------------------------")
print("Input Dimension: {}".format(tree_labels.shape[1]))
print("Output Dimension: {}".format(tree_features.shape[1]))
print("Number of samples: {}".format(tree_features.shape[0]))
```

```python
def exercise_1(η=100.0, epochs=250 , γ=1e-12):
  """Training a LNN

  Args:
  η (float): learning rate (default 100.0)
  epochs (int): number of epochs (default 250)
  γ (float): initialization scale (default 1e-12)

  """
  n_hidden = [30]

  dim_input = tree_labels.shape[1]
  dim_output = tree_features.shape[1]

  deep_model = VariableDepthWidth(in_dim=dim_input,
                                     out_dim=dim_output,
                                     hid_dims=n_hidden,
                                     gamma=γ)

  # convert (cast) data from np.ndarray to torch.Tensor
  input_tensor = torch.tensor(tree_labels).float()
  #################################################
  ## convert output_data from np.ndarray to torch.Tensor
  # Complete the function and remove or comment the line below
  raise NotImplementedError("Cast output_data as torch.Tensor")
  #################################################
  output_tensor = ...

  training_losses = train(deep_model,
                          input_tensor,
                          output_tensor,
                          n_epochs=epochs,
                          lr=η)

  plot_loss(training_losses)

# # Uncomment and run
# exercise_1()
```

```python
# to_remove solution
def exercise_1(η=100.0, epochs=250 , γ=1e-12):
  """Training a LNN

  Args:
  η (float): learning rate (default 100.0)
  epochs (int): number of epochs (default 250)
  γ (float): initialization scale (default 1e-12)

  """
  n_hidden = [30]

  dim_input = tree_labels.shape[1]
  dim_output = tree_features.shape[1]

  deep_model = VariableDepthWidth(in_dim=dim_input,
                                     out_dim=dim_output,
                                     hid_dims=n_hidden,
                                     gamma=γ)

  # convert (cast) data from np.ndarray to torch.Tensor
  input_tensor = torch.tensor(tree_labels).float()
  output_tensor = torch.tensor(tree_features).float()

  training_losses = train(deep_model,
                          input_tensor,
                          output_tensor,
                          n_epochs=epochs,
                          lr=η)

  plot_loss(training_losses)

# # Uncomment and run
with plt.xkcd():
  exercise_1()
```

**Question**: Why haven't we seen these "bumps" in training before? And should we look for them in the future? You can slide the widgets below and find your answer. Here, $\gamma$ is the initialization scale.

```python
#@markdown Make sure you execute this cell to enable the widget!
_ = interact(exercise_1,
             η = FloatSlider(min=1.0, max=200.0, step=2.0, value=100.0,
                             continuous_update=False, readout_format='.1f', description='η'),
             epochs = fixed(250),
             γ = FloatLogSlider(min=-15, max=1, step=1, value=1e-12, base=10,
                             continuous_update=False, description='γ'),
             )
```

    interactive(children=(FloatSlider(value=100.0, continuous_update=False, description='η', max=200.0, min=1.0, r…

---
# Section 2: Singular Value Decomposition (SVD)

```python
#@title Video 2: Singular Value Decomposition (SVD)
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="eTXNKMleEj8", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/eTXNKMleEj8

In this section, we would go deeper in understanding of the learning dynamics we just saw. First, we should know that a linear neural network is perfoming sequential matrix maltiplications, which can simplified to:

\begin{align}
\mathbf{y} &= \mathbf{W}_{L}~\mathbf{W}_{L-1}~\dots~\mathbf{W}_{1} ~ \mathbf{x} \\
 &= (\prod_{i=1}^{L}{\mathbf{W}_{i}}) ~ \mathbf{x} \\
 &= \mathbf{W}_{tot} ~ \mathbf{x}
\end{align}

where $L$ denotes the number of layers in our network.

Why did we just called the learning progress, "learning dynamics"? Learning through gradient descent seems very aike to evolution of a dynamic system. They both are described by set of differntial equations (gradients). Dynamical systems often have a "time-constant" which describes the rate of change, similar to the learning rate, only instead of time, gradient descent evolves through epochs.

[Saxe et al. (2013)](https://arxiv.org/abs/1312.6120) showed that to analyse and to understanding the nonlinear learning dynamics of a deep LNN, we can use [Singular Value Decomposition (SVD)](https://en.wikipedia.org/wiki/Singular_value_decomposition) to decompose the $\mathbf{W}_{tot}$ into orthogonal vectors, where orthogonality of the vecors would ensure their "individuality". This means we can break the a deep wide LNN to multiple deep narrow LNN, so their activity is untangled from each other.

__A Quick intro to SVD__

Any real-valued matix $A$ (yes, ANY) can be decomposed (factorized) to 3 matrices:

\begin{equation}
\mathbf{A} = \mathbf{U} \mathbf{Σ} \mathbf{V}^{\top}
\end{equation}

where $U$ is an orthogonal matrix, $\Sigma$ is a diagonal matrix, and $V$ is again an orthogonal matrix. The diagonal elements of $\Sigma$ are called **singular values**.

The main difference between SVD and Eigen Value Decomposition (EVD), is that EVD requires $A$ to be squred and does not guarantee the eigenvectors to be orthogonal. For the complex-valued matrix $A$, the factorization changes to $A = UΣV^*$ and $U$ and $V$ are unitary matrices.

We strongly recommend the [Singular Value Decomposition (the SVD)](https://www.youtube.com/watch?v=mBcLRGuAFUk) by the amazing [Gilbert Strang](http://www-math.mit.edu/~gs/) if you would like to learn more.

## Exercise 2. SVD

Let's put what we learned in practice. Here, we want to modify our training loop to perform the SVD on $\mathbf{W}_{tot}$ in every epoch, and record the singular values (diagonal values of $\Sigma$). SVD is implemented both in PyTorch [`torch.svd`](https://pytorch.org/docs/stable/generated/torch.svd.html) and in NumPy [`np.linalg.svd`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.svd.html), but we recommend the PyTorch method to avoid the conversion cost. Since $\Sigma$ is a diagonal matrix, often (e.g. PyTorch and NumPy) just the diagonal elements are returned as a vector, not the whole matrix.

We have removed the progress bar and the optional loss and optimizer arguments to make the exercise "cleaner".

```python
def train_svd_exercise(model, in_features, out_features, n_epochs, lr):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    lr(float): learning rate

  Returns:
    np.ndarray: record (evolution) of losses
    np.ndarray: record (evolution) of singular values
  """

  assert in_features.shape[0] == out_features.shape[0]
  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()
  loss_record = []  # for recoding losses
  sv_record = []  # for recoding singular values

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    # calculating the W_tot by multiplying all layers' weights
    W_tot = model.layers[-1].weight.detach()  # starting from the last layer
    for i in range(2, len(model.layers)+1):
      #################################################
      ## Complete the loop for calculating the W_tot
      # Complete the function and remove or comment the line below
      raise NotImplementedError("Calculate the W_tot")
      #################################################
      W_tot = ...

    # performing the SVD!
    #################################################
    ## calculate singular value decomposition of W_tot
    # Complete the function and remove or comment the line below
    raise NotImplementedError("Calculate the SVD for W_tot")
    #################################################
    U, Σ, V = ...

    loss_record.append(loss.item())
    sv_record.append(Σ.numpy())

  return np.array(loss_record), np.array(sv_record)
```

```python
# to_remove solution
def train_svd_exercise(model, in_features, out_features, n_epochs, lr):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    lr(float): learning rate

  Returns:
    np.ndarray: record (evolution) of losses
    np.ndarray: record (evolution) of singular values
  """

  assert in_features.shape[0] == out_features.shape[0]
  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()
  loss_record = []  # for recoding losses
  sv_record = []  # for recoding singular values

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    # calculating the W_tot by multiplying all layers' weights
    W_tot = model.layers[-1].weight.detach()  # starting from the last layer
    for i in range(2, len(model.layers)+1):
      W_tot = W_tot @ model.layers[-i].weight.detach()

    # performing the SVD!
    U, Σ, V = torch.svd(W_tot)

    loss_record.append(loss.item())
    sv_record.append(Σ.numpy())

  return np.array(loss_record), np.array(sv_record)
```

```python
#@markdown Make sure you execute this cell to train the network and plot

dim_input = tree_labels.shape[1]
dim_output = tree_features.shape[1]
input_tensor = torch.tensor(tree_labels).float()
output_tensor = torch.tensor(tree_features).float()

deep_model = VariableDepthWidth(in_dim=dim_input,
                                    out_dim=dim_output,
                                    hid_dims=[30])

training_losses, singular_values, _, _ = train_svd_rsa_track(deep_model,
                        input_tensor,
                        output_tensor,
                        n_epochs=250,
                        lr=100.0)

plot_loss_sv_twin(training_losses, singular_values)
```

**Question**: Isn't this beautiful? For EigenValue decomposition, the anount of variance explained by eigenvectors are proportional to the corresponding eigenvalues. What about the SVD? We definitely see that the gradient descent guides the network to first learn the features that carry more information (have higher singular value)!

---
# Section 3: Representational Similarity Analysis (RSA)

```python
#@title Video 3.1: Representational Similarity Analysis (RSA)
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="19seHV97WkI", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/19seHV97WkI

The previous section ended with an interesting remark! The network (through gradient descent), seems to prioritize learning features that explain most of the data, and gradually learn all the hidden representations. Given that we are training a hierarchically structured data, we may be able to see that progress as well.

To do so, we get help from Representational Similarity Analysis (RSA) approach to understand the internal representation of our network. The main idea is that the activity of hidden units (neurons) in the network must be similar when the network is presented with a similar input. The exercise will help more to get an intuition of this approach and the dynamics of representational learning.

## Exercise 3: RSA

We need to modify our training function once more. The task is to calculate similarity between the hidden layer activities (i.e. $~\mathbf{h_1} = \mathbf{W_1} \mathbf{x}~$) for all the inputs at every epoch. For similarity measure, we can use the good old dot (scalar) product, which is also called cosine similarity. For calculating the dot product between multiple vectors (which would be our case), you can simply use matrix multiplication. Therefore the Representational Similarity Matrix (SM) for multiple input activity could be calculated as follow:

$$ RSM = \mathbf{H_1}^{\top} \mathbf{H_1} $$

where $\mathbf{H_1} = \mathbf{W_1} \mathbf{X}$.

```python
def train_svd_rsa_exercise(model, in_features, out_features, n_epochs, lr):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    lr(float): learning rate

  Returns:
    np.ndarray: record (evolution) of losses
    np.ndarray: record (evolution) of singular values
    np.ndarray: record (evolution) of representational similarity matrices
  """

  assert in_features.shape[0] == out_features.shape[0]
  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()
  loss_record = []  # for recoding losses
  sv_record = []  # for recoding singular values
  rsm_record = []  # for recording representational similarity matrices

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    # calculating the W_tot by multiplying all layers' weights
    W_tot = model.layers[-1].weight.detach()  # starting from the last layer
    for i in range(2, len(model.layers)+1):
      W_tot = W_tot @ model.layers[-i].weight.detach()
    U, Σ, V = torch.svd(W_tot)  # performing the SVD!

    # calculating representational similarity matrix
    H1 = model.layers[0].weight.detach() @ in_features
    #################################################
    ## Use H1 to calculate the representational similarity matrix
    # Complete the function and remove or comment the line below
    raise NotImplementedError("Calculate the RSM")
    #################################################
    RSM = ...

    loss_record.append(loss.item())
    sv_record.append(Σ.numpy())
    rsm_record.append(RSM.numpy())

  return np.array(loss_record), np.array(sv_record), np.array(rsm_record)
```

```python
# to_remove solution
def train_svd_rsa_exercise(model, in_features, out_features, n_epochs, lr):
  """Training function

  Args:
    model (torch nn.Module): the neural network
    in_features (torch.Tensor): features (input) with shape `torch.Size([batch_size, input_dim])`
    out_features (torch.Tensor): targets (labels) with shape `torch.Size([batch_size, output_dim])`
    n_epochs (int): number of training epochs
    lr(float): learning rate

  Returns:
    np.ndarray: record (evolution) of losses
    np.ndarray: record (evolution) of singular values
    np.ndarray: record (evolution) of representational similarity matrices
  """

  assert in_features.shape[0] == out_features.shape[0]
  optimizer = optim.SGD(model.parameters(), lr=lr)
  criterion = nn.MSELoss()
  loss_record = []  # for recoding losses
  sv_record = []  # for recoding singular values
  rsm_record = []  # for recording representational similarity matrices

  for i in range(n_epochs):
    y_pred = model(in_features)  # forward pass
    loss = criterion(y_pred, out_features)  # calculating the loss
    optimizer.zero_grad()  # reset all the graph gradients to zero
    loss.backward()  # back propagation of the error
    optimizer.step()  # gradient step

    # calculating the W_tot by multiplying all layers' weights
    W_tot = model.layers[-1].weight.detach()  # starting from the last layer
    for i in range(2, len(model.layers)+1):
      W_tot = W_tot @ model.layers[-i].weight.detach()
    U, Σ, V = torch.svd(W_tot)  # performing the SVD!

    # calculating representational similarity matrix
    H1 = model.layers[0].weight.detach() @ in_features
    RSM = H1.T @ H1

    loss_record.append(loss.item())
    sv_record.append(Σ.numpy())
    rsm_record.append(RSM.numpy())

  return np.array(loss_record), np.array(sv_record), np.array(rsm_record)
```

```python
#@markdown Make sure you execute this cell to train the network

deep_model = VariableDepthWidth(in_dim=dim_input,
                                    out_dim=dim_output,
                                    hid_dims=[30])

training_losses, singular_values, rep_sim_mats, _ = train_svd_rsa_track(deep_model,
                        input_tensor,
                        output_tensor,
                        n_epochs=250,
                        lr=100.0)
```

Using the widget below, you can look at the representational similartiy matrix at any point of training.

```python
#@markdown Make sure you execute this cell to enable the widget!

i_ep_slider = IntSlider(min=5, max=245, step=1, value=50,
                        continuous_update=False, description='Epoch',
                        layout=Layout(width='680px'))

widgets_ui = HBox([i_ep_slider])

widgets_out = interactive_output(plot_loss_sv_rsm,
                                         {'loss_array': fixed(training_losses),
                                          'sv_array': fixed(singular_values),
                                          'rsm_array': fixed(rep_sim_mats),
                                          'i_ep': i_ep_slider})

display(widgets_ui, widgets_out)
```

    HBox(children=(IntSlider(value=50, continuous_update=False, description='Epoch', layout=Layout(width='680px'),…

    Output()

```python
#@title Video 3.2: Linear Regression
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="etsXyJJSru4", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/etsXyJJSru4

## Demonstration: Linear Regression vs. DLNN

A linear neural network with NO hidden layer, is very similar to linear regression in its core. We also know that no matter how many hidden layers a linear network has, it can be compressed to a linear regression (no hidden layers).

In this demonstration, we use the hierarchically structured data to:

* analytically find the mapping between features and labels
* train a zero-depth LNN to find the mapping
* compare them to the $W_{tot}$ from the already trained deep LNN

```python
# calculating the W_tot for deep network (already trained model)
deep_weight_tot = deep_model.layers[-1].weight.detach().numpy()
for i in range(2, len(deep_model.layers)+1):
  deep_weight_tot = deep_weight_tot @ deep_model.layers[-i].weight.detach().numpy()
```

```python
# analytically estimation of weights (map)
# our data is batch first dimension, so we need to transpose our data
analytical_weights = linear_regression(tree_labels.T, tree_features.T)
```

```python
# create a model instance of VariableDepthWidth
zero_depth_model = VariableDepthWidth(in_dim=dim_input,
                                    out_dim=dim_output,
                                    hid_dims=[])
# train the zero_depth_model
training_losses = train(zero_depth_model,
                        input_tensor,
                        output_tensor,
                        n_epochs=250,
                        lr=1000.0)
# trained weights from zero_depth_model
zero_depth_model_weights = zero_depth_model.layers[0].weight.detach().numpy()

plot_loss(training_losses, "Training loss for zero depth LNN", c="r")
```

```python
print("The final weights from all methods are approximately equal?! {}!\n".format(
  (np.allclose(analytical_weights, zero_depth_model_weights, atol=1e-02) and \
   np.allclose(analytical_weights, deep_weight_tot, atol=1e-02))))
```

    The final weights from all methods are approximately equal?! True!

As you may have guessed, they all arrive at the same results but through very different paths.

---
# Section 4: Illusory Correlations

```python
#@title Video 4.1: Illusory Correlations
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="t_-wmMjl9kk", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/t_-wmMjl9kk

So far, everything looks great, all our trainings are successful (training loss converging to zero), and very fast. We even could interpret the dynamics of our deep linear networks and relate them to the data. Unfortunately, this rarely happens in practice. Real world problems often require very deep and nonlinear networks with many hyper-parameters. And oridinarily, these complex networks take hours, if not days, to train.

Let's recall the training loss curves. There was often a long plateau (where the weights are stuck at a saddle point), followed by a sudden drop. For very deep complex neural network, such plateaus can last for hours of training, and we often decide to stop the training becasue we believe it "as good as it gets"! This riases a challenge of whether the network has learned all the "intended" hidden representations. But more importantly, the network might find an illusionary correlation between features that has never seen.

To better understand this, let's do the next demonstration and exercise.

## Demonstration: Illusory Correlations

So far we worked with a dataset that has 4 animals: Canary, Robin, Goldfish, and Tuna. These animals all have bones. Therefore if we include the "has bone" feature, the network would learn it at the second level (i.e. second bump, second singular value convergence), which is OK.

What if the dataset has Shark instead of Goldfish. Sharks don't have bones (their skeletons is made of cartilaginous, which is much lighter than true bone and more flexible). Then we will have feature which is *True* (i.e. +1) for Tuna, Robin, and Canary, but *False* (i.e. 0) for all the plants and the shark! Let's see what the network does.

First, we add the new feature to the targets. We then start training our LNN and in every epoch, record the network prediction for "sharks having bones".

```python
item_names = ['Shark', 'Tuna', 'Robin', 'Canary', 'Rose', 'Daisy', 'Pine', 'Oak']
has_bones = [0, 1, 1, 1, 0, 0, 0, 0]

tree_features = add_feature(tree_features, has_bones)
plot_tree_data(tree_labels, tree_features, item_names)
```

You can see the new feature shown in the last column of the plot above.

```python
#@markdown Make sure you execute this cell to train the network
dim_input = tree_labels.shape[1]
dim_output = tree_features.shape[1]
input_tensor = torch.tensor(tree_labels).float()
output_tensor = torch.tensor(tree_features).float()

deep_model = VariableDepthWidth(in_dim=dim_input,
                                out_dim=dim_output,
                                hid_dims=[30])

_, singular_values, _, ill_predictions = train_svd_rsa_track(deep_model,
                                              input_tensor,
                                              output_tensor,
                                              n_epochs=250,
                                              lr=100.0,
                                              ill_i=0)

plot_ills_sv_twin(ill_predictions, singular_values)
```

It seems that the network starts by learning the "[alternative fact](https://en.wikipedia.org/wiki/Alternative_facts)" that sharks have bones, and in later epochs, as it learns deeper representations, it can see (learn) beyond the illusory correlation. This is important to remember that we never presented the network with any data saying that sharks have bones.

```python
#@title Video 4.2: Illusory Correlations Explained
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="QMuTlq-atlc", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/QMuTlq-atlc

## Exercise 4: Illusory Correlations

This exercise is just for you to explore the idea of illusory correlations. Think of medical, natural or possibly social illusory correlations which can test the learning power of deep linear neural nets.

**Notes**: Before you start, there are few things to important to know:

* the generated data is independent of tree labels, therefore the names are just for convenience.

* you can rename any node in the tree object(from the `SimpleTree` class) to help you keep track of your tree. This tree, also is not the one generating the samples and is purely for convenience.

Here is our example for **Non-human Living things don't speak**:

```python
# this is just for plotting a tree and has no connection to data!
tree = SimpleTree()  # creates a tree
tree.rename("Canary", "Parrot")  # renames the Canary node to Parrot
tree.plot()  # plots the tree
```

    Living things
    ╠══ Animal
    ║   ╠══ Bird
    ║   ║   ╠══ Parrot
    ║   ║   ╚══ Robin
    ║   ╚══ Fish
    ║       ╠══ Goldfish
    ║       ╚══ Tuna
    ╚══ Plant
        ╠══ Flower
        ║   ╠══ Daisy
        ║   ╚══ Rose
        ╚══ Tree
            ╠══ Oak
            ╚══ Pine

```python
item_names = ['Goldfish', 'Tuna', 'Robin', 'Parrot', 'Rose', 'Daisy', 'Pine', 'Oak']
can_NOT_speak = [1, 1, 1, 0, 1, 1, 1, 1]  # creating the new feature
ill_id = 3  # the index of your feature

tree_labels, tree_features = generate_hsd()  # sampling new data from the tree
tree_features = add_feature(tree_features, can_NOT_speak) # adding the feature
plot_tree_data(tree_labels, tree_features, item_names)  # plot
```

```python
#@markdown Make sure you execute this cell to train the network and plot the output
dim_input = tree_labels.shape[1]
dim_output = tree_features.shape[1]
input_tensor = torch.tensor(tree_labels).float()
output_tensor = torch.tensor(tree_features).float()

deep_model = VariableDepthWidth(in_dim=dim_input,
                                out_dim=dim_output,
                                hid_dims=[30])

_, singular_values, _, ill_predictions = train_svd_rsa_track(deep_model,
                                              input_tensor,
                                              output_tensor,
                                              n_epochs=250,
                                              lr=100.0,
                                              ill_i=ill_id)

plot_ills_sv_twin(ill_predictions, singular_values)
```

---
# Wrap up

```python
#@title Video 4.3: Outro
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="Y0JfyCtikhc", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/Y0JfyCtikhc

---
# Appendix

Generally, *regression* refers to a set of methods for modeling the mapping (relationship) between one (or more) independent variable(s) (i.e. features) and one (or more) dependent variable(s) (i.e. labels). For example, if we want to examine the relative impacts of calendar date, GPS coordinates, and time of the say (the independent variables) on air temperature (the dependent variable). On the other hand, regression can be used for predictive analysis. Thus the independent variables are also called predictors. When the model contains more than one predictor, then the method is called *multiple regression*, and if it contains more than one dependent variable called *multivariate regression*. Regression problems pop up whenever we want to predict a numerical (usually continuous) value.

The independent variables are collected in vector $\mathbf{x} \in \mathbb{R}^M$, where $M$ denotes the number of independent variables, while the dependent variables are collected in vector $\mathbf{y} \in \mathbb{R}^N$, where $N$ denotes the number of independent variables. And the mapping between them is represented by the weight matrix $\mathbf{W} \in \mathbb{R}^{N \times M}$ and a bias vector $\mathbf{b} \in \mathbb{R}^{N}$ (generalizing to affine mappings).

The multivariate regression model can be written as:

\begin{equation}
\mathbf{y} = \mathbf{W} ~ \mathbf{x} + \mathbf{b}
\end{equation}

or it can be written in matrix format as:

\begin{equation}
\begin{bmatrix} y_{1} \\ y_{2} \\ \vdots \\ y_{N} \\ \end{bmatrix} = \begin{bmatrix} w_{1,1} & w_{1,2} & \dots & w_{1,M} \\ w_{2,1} & w_{2,2} & \dots & w_{2,M} \\ \vdots & \ddots & \ddots & \vdots \\ w_{N,1} & w_{N,2} & \dots & w_{N,M} \end{bmatrix} \begin{bmatrix} x_{1} \\ x_{2} \\ \vdots \\ x_{M} \\ \end{bmatrix} + \begin{bmatrix} b_{1} \\ b_{2} \\ \vdots \\b_{N} \\ \end{bmatrix}
\end{equation}

__Vectorized regression__

Linear regression can be simply extended to multi-samples ($D$) input-output mapping, which we can collect in a matrix $\mathbf{X} \in \mathbb{R}^{M \times D}$, sometimes called the design matrix. The sample dimension also shows up in the output matrix $\mathbf{Y} \in \mathbb{R}^{N \times D}$. Thus, linear regression takes the following form:

\begin{equation}
\mathbf{Y} = \mathbf{W} ~ \mathbf{X} + \mathbf{b}
\end{equation}

where matrix $\mathbf{W} \in \mathbb{R}^{N \times M}$ and the vector $\mathbf{b} \in \mathbb{R}^{N}$ (broudcasted over sample dimension) are the desired parameters to find.


# 灰色预测模型(Grey Model, GM)

**只适合中短期预测**

1. GM(1, 1)建模
    - 要求: 需要一次累加序列近似为指数规律发展
    - 模型使用数据不是原有数据

2. GM(2, 1), DGM, Verhulst
    - 对于非单调的摆动发展序列或饱和的S形序列.

3. 灰色Verhulst预测模型
    - 用来描述具有饱和状态的S型模型. 常用于人口预测, 生物生长, 繁殖预测, 产品经济寿命预测等.

- 已知$x^{(0)}=(x^{(0)}(1),x^{(0)}(2),\cdots,x^{(0)}(n))$
- 累加生成序列:$x^{(1)}=(x^{(0)}(1),x^{(0)}(1)+x^{(0)}(2),\cdots,x^{(0)}(1)+\cdots+x^{(0)}(n))$
- 均值生成序列:$z^{(1)}=(z^{(1)}(1),z^{(1)}(2),\cdots,z^{(1)}(n))$, 其中$z^{(1)}(k)=0.5x^{(1)}(k)+0.5x^{(1)}(k-1)$
- 灰微分方程: $x^{(0)}(k)+az^{(1)}(k)=b,\ k=2,3,\cdots,n$
- 相应白化微分方程: $\frac{dx^{(1)}}{dt}+ax^{(1)}(t)=b$
- 可容覆盖: $\Theta=(e^{-\frac{2}{n+1}}, e^{\frac{2}{n+1}})$

```python
# GM(1, 1)建模
# 要求: 需要一次累加序列近似为指数规律发展
# 例15.2

import numpy as np
import pandas as pd

# 1.级比检验
x0 = np.array([71.1, 72.4, 72.4, 72.1, 71.4, 72.0, 71.6], dtype=np.float)
lamb = np.array([x0[i] / x0[i+1] for i in range(x0.shape[0]-1)])
print('级比:', lamb)
Theta = (np.exp(-2/(x0.shape[0]+1)), np.exp(2/(x0.shape[0]+2)))
print('可容覆盖:', Theta)
OK = [i <= Theta[1] and i >= -Theta[0] for i in lamb]
print('可容性:', OK)

# GM(1, 1)建模
x1 = np.array([x0[:i].sum() for i in range(1, x0.shape[0]+1)])
print('一次累加序列:', x1)
B = np.array([[-(x1[i]+x1[i+1])/2, 1] for i in range(x1.shape[0]-1)])
Y = np.array([x0[i] for i in range(1, x0.shape[0])])
print('B矩阵:', B, sep='\n')
print('Y值:', Y)
u = np.matmul(np.linalg.inv(np.matmul(B.T, B)), np.matmul(B.T, Y))
print('最小二乘法得到a, b:', u)
a, b = u[0], u[1]
x1_k = lambda k: (x0[0] - b/a) * np.exp(-a*k) + b/a

# 预测
x1_pred = np.array([x1_k(k) for k in range(x0.shape[0])])
print('x1预测值:', x1_pred)
x0_pred = np.array([x1_pred[i] - x1_pred[i-1] if i != 0 else x1_pred[0] for i in range(x1_pred.shape[0])])
print('x0预测值:', x0_pred)

# 检验
test_table_columns = ['Year', 'Raw Data', 'Predction', 'Error', 'Relative_Error', '级比偏差']
test_table = pd.DataFrame(np.c_[[str(i) for i in range(1986, 1993)],
                                x0,
                                x0_pred,
                                x0 - x0_pred,
                                np.divide((x0 - x0_pred), x0),
                                [np.nan] + [1-(1-0.5*a)/(1+0.5*a)*lamb[k] for k in range(lamb.shape[0])]],
                          columns=test_table_columns)
test_table['Predction'] = test_table['Predction'].apply(lambda x: format(float(x), '.4f'))
test_table['Error'] = test_table['Error'].apply(lambda x: format(float(x), '.4f'))
test_table['Relative_Error'] = test_table['Relative_Error'].apply(lambda x: format(float(x), '.2%'))
test_table['级比偏差'] = test_table['级比偏差'].apply(lambda x: format(float(x), '.4f'))
print('\n', test_table, sep='')
```

    级比: [0.9820442  1.         1.00416089 1.00980392 0.99166667 1.00558659]
    可容覆盖: (0.7788007830714049, 1.2488488690016821)
    可容性: [True, True, True, True, True, True]
    一次累加序列: [ 71.1 143.5 215.9 288.  359.4 431.4 503. ]
    B矩阵:
    [[-107.3     1.  ]
     [-179.7     1.  ]
     [-251.95    1.  ]
     [-323.7     1.  ]
     [-395.4     1.  ]
     [-467.2     1.  ]]
    Y值: [72.4 72.4 72.1 71.4 72.  71.6]
    最小二乘法得到a, b: [2.34378648e-03 7.26572696e+01]
    x1预测值: [ 71.1        143.50574144 215.741978   287.8091065  359.70752283
     431.43762195 502.9997979 ]
    x0预测值: [71.1        72.40574144 72.23623656 72.0671285  71.89841633 71.73009912
     71.56217595]

       Year Raw Data Predction    Error Relative_Error     级比偏差
    0  1986     71.1   71.1000   0.0000          0.00%      nan
    1  1987     72.4   72.4057  -0.0057         -0.01%   0.0203
    2  1988     72.4   72.2362   0.1638          0.23%   0.0023
    3  1989     72.1   72.0671   0.0329          0.05%  -0.0018
    4  1990     71.4   71.8984  -0.4984         -0.70%  -0.0074
    5  1991     72.0   71.7301   0.2699          0.37%   0.0107
    6  1992     71.6   71.5622   0.0378          0.05%  -0.0032

```python
# GM(2, 1)模型 (DGM(2, 1)模型是它的一个简化, 实现相似, 这里不再实现)
# 例15.3

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sympy

# 1.级比检验
x0 = np.array([41, 49, 61, 78, 96, 104], dtype=np.float)
lamb = np.array([x0[i] / x0[i+1] for i in range(x0.shape[0]-1)])
print('级比:', lamb)
Theta = (np.exp(-2/(x0.shape[0]+1)), np.exp(2/(x0.shape[0]+2)))
print('可容覆盖:', Theta)
OK = [i <= Theta[1] and i >= -Theta[0] for i in lamb]
print('可容性:', OK)

# GM(2, 1)建模
x1 = np.array([x0[:i].sum() for i in range(1, x0.shape[0]+1)])
print('1-AGO序列:', x1)
alpha1_x0 = np.array([x0[i] - x0[i-1] for i in range(1, x0.shape[0])])
print('1-IAGO序列:', alpha1_x0)
z1 = np.array([(x1[i]+x1[i+1])/2 for i in range(x1.shape[0]-1)])
print('均值生成序列:', z1)
B = np.array(np.c_[-x0[1:], -z1, np.ones(z1.shape[0])])
print('B矩阵:', B, sep='\n')
Y = alpha1_x0
print('Y值:', Y)
u = np.matmul(np.linalg.inv(np.matmul(B.T, B)), np.matmul(B.T, Y))
print('最小二乘法得到a1, a2, b:', u)

sympy.init_printing()
# 定义符号常量x 与 f(x)
t = sympy.Symbol('t')
a1, a2, b = sympy.symbols('a1, a2, b')
x1_t = sympy.Function('x1_t')
# 微分方程ode
ode = sympy.Eq(x1_t(t).diff(t, t) + a1*x1_t(t).diff(t) + a2*x1_t(t), b)
init1 = sympy.Eq(x1_t(0), x1[0])
init2 = sympy.Eq(x1_t(1), x1[1])
# 调用dsolve函数,返回一个Eq对象，hint控制精度
ode_sol = sympy.dsolve((ode), x1_t(t))
print('得到带参符号解:', ode_sol)
# 初始条件字典
x1_to_value = {x1_t(0): 41, x1_t(x1.shape[0]-1): x1[-1]}
# 确定待定常数集
known_params = {a1, a2, b, t}
free_params = ode_sol.free_symbols - set(known_params)
# 将初始条件带入微分方程的后得到的多元方程
C_eqs = [sympy.Eq(ode_sol.lhs.subs(t, 0).subs(x1_to_value), ode_sol.rhs.subs(t, 0)),
         sympy.Eq(ode_sol.lhs.subs(t, x1.shape[0]-1).subs(x1_to_value), ode_sol.rhs.subs(t, x1.shape[0]-1))
        ]
print('待定常数: ', free_params)
print('关于待定常数的多元方程组:', C_eqs, sep='\n')
sol_params = sympy.solve(C_eqs, free_params)
print('解得待定常数:', sol_params, '\n')
known_params_value = {a1: u[0], a2: u[1], b: u[2]}
final_sol_eq = sympy.Eq(ode_sol.lhs, ode_sol.rhs.subs(sol_params).subs(known_params_value))
print('最终解:', final_sol_eq.lhs, '=', final_sol_eq, '\n')

# 预测
x1_pred = np.array([final_sol_eq.rhs.subs(t,i) for i in range(x1.shape[0])])
print('x1预测值:', x1_pred)
x0_pred = np.array([x1_pred[i] - x1_pred[i-1] if i != 0 else x1_pred[i] for i in range(x1_pred.shape[0])])
print('x0预测值:', x0_pred)

# 检验
test_table_columns = ['Raw Data', 'Predction', 'Error', 'Relative_Error']
test_table = pd.DataFrame(np.c_[x0,
                                x0_pred,
                                x0 - x0_pred,
                                np.divide((x0 - x0_pred), x0)],
                          columns=test_table_columns)
test_table['Predction'] = test_table['Predction'].apply(lambda x: format(float(x), '.4f'))
test_table['Error'] = test_table['Error'].apply(lambda x: format(float(x), '.4f'))
test_table['Relative_Error'] = test_table['Relative_Error'].apply(lambda x: format(float(x), '.2%'))
print('\n', test_table, sep='')

plt.plot(np.linspace(0, x0.shape[0]-1, x0.shape[0]), x0, c='red', label='Raw Data')
x1_pred = np.array([final_sol_eq.rhs.subs(t,i) for i in range(7)])
x0_pred = np.array([x1_pred[i] - x1_pred[i-1] if i != 0 else x1_pred[i] for i in range(x1_pred.shape[0])])
plt.plot(np.linspace(0, x0_pred.shape[0]-1, x0_pred.shape[0]), x0_pred, c='blue', label='Pred Data')
plt.legend()
plt.show()
```

```python
# 灰色Verhulst预测模型
# 例15.5

import numpy as np
import pandas as pd

# 原始数据
x0 = np.array([4.93, 2.33, 3.87, 4.35, 6.63, 7.15, 5.37, 6.39, 7.81, 8.35], dtype=np.float)

# 计算1次累加序列
x1 = np.array([x0[:i].sum() for i in range(1, x0.shape[0]+1)])
print('一次累加序列:', x1)
z1 = np.array([(x1[i]+x1[i+1])/2 for i in range(x1.shape[0]-1)])
print('均值生成序列:', z1)
B = np.array([[-z1[i], z1[i]*z1[i]] for i in range(x1.shape[0]-1)])
print('B矩阵:', B, sep='\n')
Y = np.array([x0[i] for i in range(1, x0.shape[0])])
print('Y值:', Y)
u = np.matmul(np.linalg.inv(np.matmul(B.T, B)), np.matmul(B.T, Y))
print('最小二乘法得到a, b:', u)
a, b = u[0], u[1]
x1_k = lambda k: (x0[0] - b/a) * np.exp(-a*k) + b/a

# 预测
x1_pred = np.array([x1_k(k) for k in range(x0.shape[0])])
print('x1预测值:', x1_pred)
x0_pred = np.array([x1_pred[i] - x1_pred[i-1] if i != 0 else x1_pred[0] for i in range(x1_pred.shape[0])])
print('x0预测值:', x0_pred)

# 检验
test_table_columns = ['Raw Data', 'Predction', 'Error', 'Relative_Error']
test_table = pd.DataFrame(np.c_[x0,
                                x0_pred,
                                x0 - x0_pred,
                                np.divide((x0 - x0_pred), x0)],
                          columns=test_table_columns)
test_table['Predction'] = test_table['Predction'].apply(lambda x: format(float(x), '.4f'))
test_table['Error'] = test_table['Error'].apply(lambda x: format(float(x), '.4f'))
test_table['Relative_Error'] = test_table['Relative_Error'].apply(lambda x: format(float(x), '.2%'))
print('\n', test_table, sep='')
```

    一次累加序列: [ 4.93  7.26 11.13 15.48 22.11 29.26 34.63 41.02 48.83 57.18]
    均值生成序列: [ 6.095  9.195 13.305 18.795 25.685 31.945 37.825 44.925 53.005]
    B矩阵:
    [[  -6.095      37.149025]
     [  -9.195      84.548025]
     [ -13.305     177.023025]
     [ -18.795     353.252025]
     [ -25.685     659.719225]
     [ -31.945    1020.483025]
     [ -37.825    1430.730625]
     [ -44.925    2018.255625]
     [ -53.005    2809.530025]]
    Y值: [2.33 3.87 4.35 6.63 7.15 5.37 6.39 7.81 8.35]
    最小二乘法得到a, b: [-0.35762668 -0.00410372]
    x1预测值: [  4.93         7.04462985  10.06840608  14.39219957  20.57492891
      29.41580956  42.05766501  60.13465479  85.98351544 122.94562055]
    x0预测值: [ 4.93        2.11462985  3.02377624  4.32379348  6.18272934  8.84088065
     12.64185545 18.07698979 25.84886065 36.96210511]

       Raw Data Predction     Error Relative_Error
    0      4.93    4.9300    0.0000          0.00%
    1      2.33    2.1146    0.2154          9.24%
    2      3.87    3.0238    0.8462         21.87%
    3      4.35    4.3238    0.0262          0.60%
    4      6.63    6.1827    0.4473          6.75%
    5      7.15    8.8409   -1.6909        -23.65%
    6      5.37   12.6419   -7.2719       -135.42%
    7      6.39   18.0770  -11.6870       -182.89%
    8      7.81   25.8489  -18.0389       -230.97%
    9      8.35   36.9621  -28.6121       -342.66%


# Lithium Ion Battery Example

## Note: the example below requires Cantera 2.5.0a2 or newer

If the stable release of 2.5.0 is not yet available, you might consider [building from source](https://cantera.org/install/compiling-install.html). Note also that 2.5.0, once released, is more recent than and supercedes 2.5.0a2, which is an alpha release.

## Open circuit calculations as a function of anode and cathode lithium content.

In this example we will illustrate how to calculate the open circuit voltage (voltage when the external applied current is 0) for a lithium ion battery.  The thermodynamics are based on a graphite anode and a LiCoO2 cathode (the typical/standard active materials for commercial batteries, as of 2019), and are modeled using the 'BinarySolutionTabulatedThermo' class.

For the sake of simplicity, we're going to assume that the anode and cathode capacities are perfectly balanced (i.e. if the cathode lithium content is X percent of its max possible (i.e. its capacity), then we will assume that the anode is at 1-X percent.  Without loss of generality, we will define the anode composition.

The routine below returns the cell voltage (in Volt) of a lithium-ion cell for a given cell current and active material lithium stoichiometries.

Note that the function 'E_cell' below has even greater capabilities than what we use, here. It calculates the steady state cell voltage, at a given composition and cell current, for a given electrolyte ionic resistance.  This functionality is presented in greater detail in the reference (which also describes the derivation of the BinarySolutionTabulatedThermo class):

>M. Mayur, S. C. DeCaluwe, B. L. Kee, W. G. Bessler, “Modeling and simulation of the thermodynamics of lithium-ion battery intercalation materials in the open-source software Cantera,” _Electrochim. Acta_ 323, 134797 (2019), https://doi.org/10.1016/j.electacta.2019.134797

In the future, this example may be developed further to demonstrate simulating charge-discharge of the lithium ion battery.

Other than the typical Cantera dependencies, plotting functions require that you have matplotlib installed, and finding the electrode potentials uses Scipy's fsolve. See https://matplotlib.org/ and https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fsolve.html for additional info.

The open circuit (i.e. equilibrium) voltage, here, is calculated via two means: kinetically and thermodynamically.

The system here, can be thought of as consisting of two particles--anode and cathode--connected by a liquid electrolyte:

### Kinetic equilibrium calculations
In the kinetic equilibrium problem, steady state is achieved when the faradaic current at each electrode (anode and cathode) surface and the ionic current through the electrolyte are all zero.  There are, essentially, four relevant electric potentials which must be determined:

- $\phi_{\rm anode}$: electric potential of graphite anode.
- $\phi_{\rm elyte,\, anode}$: electric potential of electrolyte at anode interface.
- $\phi_{\rm elyte,\, cathode}$: electric potential of electrolyte at cathode interface.
- $\phi_{\rm cathode}$: electric potential of LCO cathode.

Setting one of these four to the reference potential of zero (because it is the difference in electric potential which drives currents, the actual potential values are irrelevant. Let's assume the anode electric potential is zero), there is only one distribution of electric potentials across the cell such that the current is invariant across the cell.  I.e. we want the potentials such that:

\begin{equation}
i_{\rm Far,\, anode} = i_{\rm ionic} = i_{\rm Far,\,cathode}= i_{\rm app}
\end{equation}

where $i_{\rm app}$ is the user input for the applied current.  For this example, we assume an applied current of 0, to calculate the equilibrium voltage.

#### Faradaic current
The Faradaic current for this model is calculated using Butler-Volmer kinetics.  For a Li-ion battery, this is:

\begin{equation}
i = S_{\rm elde}i_\circ\left[\exp\left(\frac{F\beta\eta}{RT}\right) - \exp\left(-\frac{F(1-\beta)\eta}{RT}\right) \right]
\end{equation}

where $S_{\rm elde}$ is the specific surface area of the electrode in question, $F$ is Faraday's constant, $\beta$ is the charge-transfer symmetry parameter, $R$ the universal gas constant, $T$ the temperature, and $\eta$ the overpotential, which is the electric potential difference between the electrode and electrolyte, $\Delta \phi = \phi_{\rm elde} - \phi_{\rm elyte}$, relative to that at equilibrium, $\Delta \phi_{\rm eq}$:

\begin{equation}
\eta = \Delta \phi - \Delta \phi_{\rm eq}
\end{equation}

$i_\circ$ is known as the "exchange current density," which is equal to the rate of the forward and reverse current at equilibrium (which are equal).  $i_\circ$ and $\beta$ are provided as user inputs in the cti file.  At any particular state, (user-specified electric potentials, pressure, temperature, and chemical compositions), Cantera calculates $\eta$ as part of the routine to evaluate reaction rates of progress $\left(\dot{q} = \frac{i_{\rm Far}}{F}\right)$. The user simply sets the state values mentioned above.

#### Ionic current
The electrolyte is modeled as a resistor with user-defined ionic resistance $R_{\rm io}$, and hence the ionic current is calculated as:

\begin{equation}
i_{\rm ionic} = \frac{\phi_{\rm elyte,\,ca} - \phi_{\rm elyte,\,an}}{R_{\rm io}}
\end{equation}

where positive current is defined as delivering Li$^+$ to the anode interface.  Given $i_{\rm app}$, this equation can be inverted, to calculate the electric potential of the electrolyte at the cathode interface, relative to that at the anode interface:

\begin{equation}
\phi_{\rm elyte,\,ca} = \phi_{\rm elyte,\,an} + R_{\rm io}i_{\rm app}
\end{equation}

Again: in this example, $i_{\rm app} = 0$ and hence the two electric potential values in the electrolyte are equal.

#### Numerical routine
For the kinetic routine, there are three processes to determine the cell voltage $\phi_{\rm cathode} - \phi_{\rm anode}$ which corresponds to the user-provided $i_{\rm app}$:
1. Determine the $\phi_{\rm elyte,\,anode}$ value which corresponds to $i_{\rm app}$, given $X_{\rm Li, anode}$, the percentage of Li in the anode active material.
2. Determine $\phi_{\rm elyte,\,cathode}$, given $\phi_{\rm elyte,\,anode}$ and $i_{\rm app}$.
3. Determine the $\phi_{\rm cathode}$ which corresponds to $i_{\rm app}$, given $\phi_{\rm elyte,\,cathode}$ and $X_{\rm Li, anode}$, the percentage of Li in the anode active material.

The routines below are written generally such that an interested user may set $i_{\rm app}$ to any value of interest.

#### Import necessary packages:

```python
import cantera as ct
print('Runnning Cantera version: ' + ct.__version__)
```

    Runnning Cantera version: 2.5.0a2

```python
import numpy as np

from scipy.optimize import fsolve

# Used for timing our calculations:
import time

# Plotting:
%matplotlib notebook
import matplotlib.pyplot as plt

plt.rcParams['axes.labelsize'] = 16
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['figure.autolayout'] = True
```

### Define the phases

The phase thermodynamics are defined according to experimentally-measured open circuit voltage values, as desscribed in the reference provided above:

```python
inputCTI = 'data/lithium_ion_battery.cti'
anode = ct.Solution(inputCTI, 'anode');
cathode = ct.Solution(inputCTI, 'cathode');
# The 'elde' electrode phase is needed as a source/sink for electrons:
elde = ct.Solution(inputCTI, 'electron');
elyte = ct.Solution(inputCTI, 'electrolyte');
anode_interface = ct.Interface(inputCTI, 'edge_anode_electrolyte', [anode, elde, elyte]);
cathode_interface = ct.Interface(inputCTI, 'edge_cathode_electrolyte', [cathode, elde, elyte]);
```

### Define battery conditions : temperature, pressure, stoichiometry, electrolyte resistance:

Inputs are:
- Stoichiometries X_Li_ca and X_Li_an [-] (can be vectors)
- Temperature T [K]
- Pressure P [Pa]
- Externally-applied current i_app [A]
- Electrolyte resistance R_elyte [Ohm]
- Anode total surface area S_an [m^2]
- Cathode total surface area S_ca [m^2]

```python
# Array of lithium mole fractions in the anode:
X_Li_an = np.arange(0.005, 0.995, 0.02)
# Assume that the cathode and anode capacities are balanced:
X_Li_ca = 1. - X_Li_an;

# I_app = 0: Open circuit
I_app = 0.;

# At zero current, electrolyte resistance is irrelevant:
R_elyte = 0.;

# Temperature and pressure
T = 300 # K
P = ct.one_atm

F = ct.faraday

S_ca = 1.1167; # [m^2] Cathode total active material surface area
S_an = 0.7824; # [m^2] Anode total active material surface area
```

### Set phase temperatures and pressures:

```python
phases = [anode, elde, elyte, cathode, anode_interface, cathode_interface];
for ph in phases:
    ph.TP = T, P
```

### Helper Functions:

```python
def anode_curr(phi_l,I_app,phi_s,X_Li_an):

    # Set the active material mole fraction
    anode.X = 'Li[anode]:' + str(X_Li_an) + ', V[anode]:' + str(1 - X_Li_an)

    # Set the electrode and electrolyte potential
    elde.electric_potential = phi_s
    elyte.electric_potential = phi_l

    # Get the net product rate of electrons in the anode (per m2^ interface)
    r_elec = anode_interface.get_net_production_rates(elde)

    anCurr = r_elec*ct.faraday*S_an;
    diff = I_app + anCurr

    return diff

def cathode_curr(phi_s,I_app,phi_l,X_Li_ca):

    # Set the active material mole fractions
    cathode.X = 'Li[cathode]:' + str(X_Li_ca) + ', V[cathode]:' + str(1 - X_Li_ca)

    # Set the electrode and electrolyte potential
    elde.electric_potential = phi_s
    elyte.electric_potential = phi_l

    # Get the net product rate of electrons in the cathode (per m2^ interface)
    r_elec = cathode_interface.get_net_production_rates(elde)

    caCurr = r_elec*ct.faraday*S_an;
    diff = I_app - caCurr

    return diff
```

### Run the calculations for all stoichiometries:

```python
# Tic
t0 = time.time()

# Initialize array of OCVs:
E_cell_kin = np.zeros_like(X_Li_ca)

for i,X_an in enumerate(X_Li_an):
    #Set anode electrode potential to 0:
    phi_s_an = 0
    E_init = 3.0

    phi_l_an = fsolve(anode_curr,E_init,args=(I_app, phi_s_an, X_an))

    # Calculate electrolyte potential at cathode interface:
    phi_l_ca = phi_l_an + I_app*R_elyte;

    # Calculate cathode electrode potential
    phi_s_ca = fsolve(cathode_curr,E_init,args=(I_app, phi_l_ca, X_Li_ca[i]))

    # Calculate cell voltage
    E_cell_kin[i] = phi_s_ca - phi_s_an

# Toc
t1 = time.time()
print('{:d} cell voltages calculated in {:3.2f} seconds.'.format(i, t1 - t0))
```

    49 cell voltages calculated in 0.61 seconds.

### Plot cell voltage, as a function of the cathode stoichiometry:

```python
plt.figure()
plt.plot(100*X_Li_ca, E_cell_kin, color='b', linewidth=2.5)
plt.ylim([2.5, 4.3])
plt.xlabel('Li Fraction in Cathode (%)', fontname='Times New Roman', fontsize=18)
plt.ylabel('Open Circuit Potential (V)', fontname='Times New Roman', fontsize=18)

ax = plt.gca()

for tick in ax.xaxis.get_major_ticks():
    tick.label1.set_fontsize(14)
    tick.label1.set_fontname('Times New Roman')
for tick in ax.yaxis.get_major_ticks():
    tick.label1.set_fontsize(14)
    tick.label1.set_fontname('Times New Roman')
```

    <IPython.core.display.Javascript object>

### Thermodynamic Equilibrium Calculation
For the $i_{app} = 0$ case, we can also calcualte the voltage using thermodynamics.  At equilibrium, the net electrochemical potential change of the reaction must be zero:

\begin{equation}
\sum_k\nu_k\tilde{\mu}_k = 0
\end{equation}

where $\tilde{\mu}_k = \mu_k + z_kF\Phi_k$, where, in turn $\mu_k = \frac{\partial g_k}{\partial n_k}$ is the chemical potential, $\nu_k$ the net stoichiometric coefficient, $z_k$ the net elementary charge, and $\Phi_k$ the phase electric potential for species $k$.

From this, we can calculate the equilibrium electric potential difference $\Delta \Phi_{\rm eq} = \left(\Phi_{\rm elde} - \Phi_{\rm elyte}\right)_{\rm eq}$ as:

\begin{equation}
\Delta \Phi_{\rm eq} = -\frac{\Delta g_{\rm rxn}}{n_{\rm charge}F}
\end{equation}

where $\Delta g_{\rm rxn} = \sum_k \nu_k\mu_k$ is the chemical potential of the reaction and and $n_{\rm charge} = \sum_{k,\,{\rm elde}} \nu_k z_k$ is the net elementary charge transferred from the electrolyte to the electrode.

```python
# Tic
t0 = time.time()

# Initialize array of OCVs:
E_cell_therm = np.zeros_like(X_Li_ca)

for i, X_an in enumerate(X_Li_an):
    #Set anode electrode potential to 0:
    anode.X = 'Li[anode]:' + str(X_an) + ', V[anode]:' + str(1 - X_an)
    dG_an = anode_interface.delta_gibbs[0]
    n_charge = -1.
    E_eq_an = -dG_an/n_charge/ct.faraday

    cathode.X = 'Li[cathode]:' + str(1. - X_an) + ', V[cathode]:' + str(X_an)
    dG_ca = cathode_interface.delta_gibbs[0]
    n_charge = 1.
    E_eq_ca = -dG_ca/n_charge/ct.faraday

    E_cell_therm[i] = E_eq_ca - E_eq_an

# Toc
t1 = time.time()
print('{:d} cell voltages calculated in {:3.2f} seconds.'.format(i, t1 - t0))
```

    49 cell voltages calculated in 0.03 seconds.

### Plot thermodynamic OCV, and compare to results from kinetic method

```python
plt.figure()
plt.plot(100*X_Li_ca, E_cell_therm,color='b', linewidth=2.5)
plt.plot(100*X_Li_ca, E_cell_kin,linewidth=0., marker='o', markerfacecolor='none', markeredgecolor='r')
plt.ylim([2.5, 4.3])
plt.xlabel('Li Fraction in Cathode (%)', fontname='Times New Roman', fontsize=18)
plt.ylabel('Open Circuit Potential (V)', fontname='Times New Roman', fontsize=18)
plt.legend(['Thermodynamic', 'Kinetic'])

ax = plt.gca()

for tick in ax.xaxis.get_major_ticks():
    tick.label1.set_fontsize(14)
    tick.label1.set_fontname('Times New Roman')
for tick in ax.yaxis.get_major_ticks():
    tick.label1.set_fontsize(14)
    tick.label1.set_fontname('Times New Roman')
```

    <IPython.core.display.Javascript object>

As one would expect, the two approaches give identical results.  While both methods are incredibly fast, the thermodynamic method is roughly 30 times faster.

A large part of this is that the thermodynamic approach is an analytical approach (i.e. the answer is known from theory), while the kinetic approach relies on the root-finding fzero method to fit the correct voltage.  Note also that the kinetic method, because of the use of Butler-Volmer kinetics, calculates the thermodynamic voltage, in order to calculate the overpotential $\eta = \Delta \Phi - \Delta \Phi_{\rm eq}$.

However, it is at last important to note that, while slower, the kinetic method is of course more robust, and can be used to find results away from equilibrium.  The thermodynamic method is only applicable at equilibrium (zero current).


# Elastic wave equation implementation on a staggered grid

This is a first attempt at implemenenting the elastic wave equation as described in:

[1] Jean Virieux (1986). ”P-SV wave propagation in heterogeneous media: Velocity‐stress finite‐difference method.” GEOPHYSICS, 51(4), 889-901. https://doi.org/10.1190/1.1442147

The current version actually attempts to mirror the FDELMODC implementation by Jan Thorbecke:

[2] https://janth.home.xs4all.nl/Software/fdelmodcManual.pdf

## Explosive source

We will first attempt to replicate the explosive source test case described in [1], Figure 4. We start by defining the source signature $g(t)$, the derivative of a Gaussian pulse, given by Eq 4:

$$g(t) = -2 \alpha(t - t_0)e^{-\alpha(t-t_0)^2}$$

```python
from devito import *
from examples.seismic.source import WaveletSource, RickerSource, GaborSource, TimeAxis
from examples.seismic import plot_image
import numpy as np

from sympy import init_printing, latex
init_printing(use_latex=True)
```

```python
# Initial grid: 1km x 1km, with spacing 100m
extent = (2000., 2000.)
shape = (201, 201)
x = SpaceDimension(name='x', spacing=Constant(name='h_x', value=extent[0]/(shape[0]-1)))
z = SpaceDimension(name='z', spacing=Constant(name='h_z', value=extent[1]/(shape[1]-1)))
grid = Grid(extent=extent, shape=shape, dimensions=(x, z))
```

```python
class DGaussSource(WaveletSource):

    def wavelet(self, f0, t):
        a = 0.004
        return -2.*a*(t - 1/f0) * np.exp(-a * (t - 1/f0)**2)

# Timestep size from Eq. 7 with V_p=6000. and dx=100
t0, tn = 0., 600.
dt = (10. / np.sqrt(2.)) / 6.
time_range = TimeAxis(start=t0, stop=tn, step=dt)

src = RickerSource(name='src', grid=grid, f0=0.01, time_range=time_range)
src.coordinates.data[:] = [1000., 1000.]
```

```python
#NBVAL_SKIP

src.show()
```

```python
# Now we create the velocity and pressure fields
so = 2
vx= TimeFunction(name='vx', grid=grid, staggered=x, space_order=so)
vz = TimeFunction(name='vz', grid=grid, staggered=z, space_order=so)
txx = TimeFunction(name='txx', grid=grid, staggered=NODE, space_order=so)
tzz = TimeFunction(name='tzz', grid=grid, staggered=NODE, space_order=so)
txz = TimeFunction(name='txz', grid=grid, staggered=(x, z), space_order=so)

```

```python
# Now let's try and create the staggered updates
t = grid.stepping_dim
time = grid.time_dim

# We need some initial conditions
V_p = 4.0
V_s = 1.0
density = 3.

# The source injection term
src_xx = src.inject(field=txx.forward, expr=src)
src_zz = src.inject(field=tzz.forward, expr=src)

#c1 = 9.0/8.0;
#c2 = -1.0/24.0;

# Thorbecke's parameter notation
cp2 = V_p*V_p
cs2 = V_s*V_s
ro = 1/density

mu = cs2*ro
l = (cp2*ro - 2*mu)

# fdelmodc reference implementation
u_vx = Eq(vx.forward, vx - dt*ro*(txx.dx + txz.dz))

u_vz = Eq(vz.forward, vz - ro*dt*(txz.dx + tzz.dz))

u_txx = Eq(txx.forward, txx - (l+2*mu)*dt * vx.forward.dx -  l*dt * vz.forward.dz)
u_tzz = Eq(tzz.forward, tzz - (l+2*mu)*dt * vz.forward.dz -  l*dt * vx.forward.dx)

u_txz = Eq(txz.forward, txz - mu*dt * (vx.forward.dz + vz.forward.dx))
```

```python
op = Operator([u_vx, u_vz, u_txx, u_tzz, u_txz] + src_xx + src_zz)

```

```python
# Reset the fields
vx.data[:] = 0.
vz.data[:] = 0.
txx.data[:] = 0.
tzz.data[:] = 0.
txz.data[:] = 0.
```

```python
#NBVAL_IGNORE_OUTPUT

op()
```

    Operator `Kernel` run in 0.43 s

```python
#NBVAL_SKIP

# Let's see what we got....
plot_image(vx.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(vz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(txx.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(tzz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(txz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
```

```python
# Now that looks pretty! But let's do it again with a higher order...
so = 12
vx= TimeFunction(name='vx', grid=grid, staggered=x, space_order=so)
vz = TimeFunction(name='vz', grid=grid, staggered=z, space_order=so)
txx = TimeFunction(name='txx', grid=grid, staggered=NODE, space_order=so)
tzz = TimeFunction(name='tzz', grid=grid, staggered=NODE, space_order=so)
txz = TimeFunction(name='txz', grid=grid, staggered=(x, z), space_order=so)

# fdelmodc reference implementation
u_vx = Eq(vx.forward, vx - dt*ro*(txx.dx + txz.dz))

u_vz = Eq(vz.forward, vz - ro*dt*(txz.dx + tzz.dz))

u_txx = Eq(txx.forward, txx - (l+2*mu)*dt * vx.forward.dx -  l*dt * vz.forward.dz)
u_tzz = Eq(tzz.forward, tzz - (l+2*mu)*dt * vz.forward.dz -  l*dt * vx.forward.dx)

u_txz = Eq(txz.forward, txz - mu*dt * (vx.forward.dz + vz.forward.dx))

op = Operator([u_vx, u_vz, u_txx, u_tzz, u_txz] + src_xx + src_zz)

# Reset the fields
vx.data[:] = 0.
vz.data[:] = 0.
txx.data[:] = 0.
tzz.data[:] = 0.
txz.data[:] = 0.
```

```python
#NBVAL_IGNORE_OUTPUT

op()
```

    Operator `Kernel` run in 2.60 s

```python
#NBVAL_SKIP

plot_image(vx.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(vz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(txx.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(tzz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
plot_image(txz.data[0], vmin=-.5*1e-2, vmax=.5*1e-2, cmap="seismic")
```


```python
from IPython.core.display import HTML
HTML("""
<style>
.output_png {
    display: table-cell;
    text-align: center;
    vertical-align: middle;
}
</style>
""")
```

<style>
.output_png {
    display: table-cell;
    text-align: center;
    vertical-align: middle;
}
</style>

# *Circuitos Elétricos I - Semana 11*

### A integral de Laplace

Seja $f(t)$ uma função definida no intervalo $0\leq t \leq \infty$, com $t$ e $f(t)$ reais, então a função $F(s)$, definida pela integral de Laplace

$$\large
\begin{equation}
F(s)=\mathcal{L}\{f(t)\}=\int_{0}^{\infty} f(t) e^{-s t} dt,\;\; s \in \mathbb{C},
\end{equation}
$$

é conhecida como a transformada de Laplace de $f(t)$.

#### A exponencial complexa

Temos que $s = \sigma + j\omega$, logo

$$
e^{-s t} = e^{-(\sigma + j\omega) t} = e^{-\sigma t}e^{-j\omega t} = e^{-\sigma t} [\cos(\omega t) + j\sin(\omega t)]
$$

$$
\begin{align}
\mathcal{L}\{f(t)\}&=\int_{0}^{\infty} f(t) e^{-\sigma t} [\cos(\omega t) + j\sin(\omega t)] dt\\
\mathcal{L}\{f(t)\}&=\int_{0}^{\infty} f(t) e^{-\sigma t} \cos(\omega t) dt + j\int_{0}^{\infty} f(t) e^{-\sigma t}\sin(\omega t) dt\\
\mathcal{L}\{f(t)\}&=\int_{0}^{\infty} \left[\frac{f(t)}{e^{\sigma t}}\right] \cos(\omega t) dt + j\int_{0}^{\infty} \left[\frac{f(t)}{e^{\sigma t}}\right]\sin(\omega t) dt
\end{align}
$$

**Teorema da existência:** se $f(t)$ é uma função contínua por pedaços para $t$ no intervalo $[a,\infty)$ e é exponencial de ordem $\sigma_0$, então a integral de Laplace converge para $\Re{(s)}>a$.

```python
import matplotlib.pyplot as plt
import numpy as np
import sympy as sp
from utils import round_expr, symdisp, symplot

# temp workaround
import warnings
from matplotlib import MatplotlibDeprecationWarning
warnings.filterwarnings('ignore', category=MatplotlibDeprecationWarning)
```

```python
sp.init_printing()

plt.rcParams['figure.figsize'] = 6, 4
plt.rcParams['legend.fontsize'] = 13
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['axes.grid'] = False
```

#### Definindo algumas variáveis simbólicas de interesse

```python
t     = sp.symbols('t', real=True)
s     = sp.symbols('s')
a     = sp.symbols('a', real=True, positive=True)
omega = sp.symbols('omega', real=True)
```

## Transformada de Laplace no Sympy

```python
# transformada de Laplace
def L(f,t,s):
    return sp.laplace_transform(f, t, s, noconds=True)

# transformada inversa de Laplace
def invL(F,s,t):
    return sp.inverse_laplace_transform(F, s, t, noconds=True)
```

```python
help(sp.laplace_transform)
```

    Help on function laplace_transform in module sympy.integrals.transforms:

    laplace_transform(f, t, s, **hints)
        Compute the Laplace Transform `F(s)` of `f(t)`,

        .. math :: F(s) = \int_0^\infty e^{-st} f(t) \mathrm{d}t.

        Explanation
        ===========

        For all "sensible" functions, this converges absolutely in a
        half plane  `a < \operatorname{Re}(s)`.

        This function returns ``(F, a, cond)``
        where ``F`` is the Laplace transform of ``f``, `\operatorname{Re}(s) > a` is the half-plane
        of convergence, and ``cond`` are auxiliary convergence conditions.

        If the integral cannot be computed in closed form, this function returns
        an unevaluated :class:`LaplaceTransform` object.

        For a description of possible hints, refer to the docstring of
        :func:`sympy.integrals.transforms.IntegralTransform.doit`. If ``noconds=True``,
        only `F` will be returned (i.e. not ``cond``, and also not the plane ``a``).

        Examples
        ========

        >>> from sympy.integrals import laplace_transform
        >>> from sympy.abc import t, s, a
        >>> laplace_transform(t**a, t, s)
        (s**(-a)*gamma(a + 1)/s, 0, re(a) > -1)

        See Also
        ========

        inverse_laplace_transform, mellin_transform, fourier_transform
        hankel_transform, inverse_hankel_transform

## Função degrau unitário

#### Domínio do tempo

```python
f = sp.Heaviside(t) # função degrau unitário

symdisp('f(t) =', f)
```

$\displaystyle f(t) =\theta\left(t\right)\; $

```python
# plota função no domínio do tempo
intervalo = np.arange(-4, 4, 0.01)
symplot(t, f, intervalo, 'u(t)')
```

#### Domínio de Laplace

```python
# calcula a transformada de Laplace de u(t)
F = L(f,t,s)

symdisp('F(s) =', F)
```

$\displaystyle F(s) =\frac{1}{s}\; $

```python
f = sp.Heaviside(t-2) # função degrau unitário em t=2

symdisp('f(t) =', f)
```

$\displaystyle f(t) =\theta\left(t - 2\right)\; $

```python
# plota função no domínio do tempo
intervalo = np.arange(-4, 4, 0.01)
symplot(t, f, intervalo, 'u(t-2)')
```

```python
F = L(f,t,s)

symdisp('F(s) =', F)
```

$\displaystyle F(s) =\frac{e^{- 2 s}}{s}\; $

```python
u1 = sp.Heaviside(t)   # função degrau unitário em t=0
u2 = sp.Heaviside(t-2) # função degrau unitário em t=2

# plota função no domínio do tempo
intervalo = np.arange(-4, 4, 0.01)
symplot(t, u1-u2, intervalo, 'u(t)-u(t-2)')
```

```python
G = L(u1-u2,t,s)

symdisp('G(s) =', G)
```

$\displaystyle G(s) =\frac{1}{s} - \frac{e^{- 2 s}}{s}\; $

## Função impulso unitário

#### Domínio do tempo

```python
f = sp.DiracDelta(t)

symdisp('f(t) =', f)
```

#### Domínio de Laplace

```python
# calcula a transformada de Laplace de δ(t)
F = L(f,t,s)

symdisp('F(s) =', F)
```

## Função exponencial

#### Domínio do tempo

```python
f = sp.exp(-a*t)

symdisp('f(t) =', f)
```

$\displaystyle f(t) =e^{- a t}\; $

```python
# plota função no domínio do tempo
intervalo = np.arange(-1, 4, 0.01)
symplot(t, f.subs({a:2}), intervalo, 'f(t)')
```

#### Domínio de Laplace

```python
# calcula a transformada de Laplace de f(t)
F = L(f,t,s)

symdisp('F(s) =', F)
```

$\displaystyle F(s) =\frac{1}{a + s}\; $

## Função cosseno amortecido

#### Domínio do tempo

```python
g = sp.exp(-a*t)*sp.cos(omega*t)

symdisp('g(t) =', g)
```

```python
# plota função no domínio do tempo
intervalo = np.arange(-1, 4, 0.01)
symplot(t, g.subs({a:2, omega:10}), intervalo, 'g(t)')
```

```python
G = L(g,t,s)

symdisp('G(s) =', G)
```

## Resposta subamortecida de um circuito de segunda ordem

#### Domínio do tempo

```python
B1, B2 = sp.symbols('B1, B2', real=True)

h = sp.exp(-a*t)*(B1*sp.cos(omega*t) + B2*sp.sin(omega*t))

symdisp('h(t) =', h)
```

$\displaystyle h(t) =\left(B_{1} \cos{\left(\omega t \right)} + B_{2} \sin{\left(\omega t \right)}\right) e^{- a t}\; $

#### Domínio de Laplace

```python
H = L(h,t,s)

symdisp('H(s) =', H)
```

$\displaystyle H(s) =\frac{B_{1} \left(a + s\right) + B_{2} \omega}{\omega^{2} + \left(a + s\right)^{2}}\; $

```python
h1 = invL(H,s,t)

symdisp('h_1(t) =', h1)
```

## Gere sua tabela de transformadas

```python
func = [1,
         t,
         sp.exp(-a*t),
         t*sp.exp(-a*t),
         t**2*sp.exp(-a*t),
         sp.sin(omega*t),
         sp.cos(omega*t),
         1 - sp.exp(-a*t),
         sp.exp(-a*t)*sp.sin(omega*t),
         sp.exp(-a*t)*sp.cos(omega*t),
         ]
func

symdisp('f(t) =', func)
```

$\displaystyle f(t) =\left[ 1, \  t, \  e^{- a t}, \  t e^{- a t}, \  t^{2} e^{- a t}, \  \sin{\left(\omega t \right)}, \  \cos{\left(\omega t \right)}, \  1 - e^{- a t}, \  e^{- a t} \sin{\left(\omega t \right)}, \  e^{- a t} \cos{\left(\omega t \right)}\right]\; $

```python
Fs = [L(f,t,s) for f in func]

symdisp('F(s) =', Fs)
```

$\displaystyle F(s) =\left[ \frac{1}{s}, \  \frac{1}{s^{2}}, \  \frac{1}{a + s}, \  \frac{1}{\left(a + s\right)^{2}}, \  \frac{2}{\left(a + s\right)^{3}}, \  \frac{\omega}{\omega^{2} + s^{2}}, \  \frac{s}{\omega^{2} + s^{2}}, \  \frac{a}{s \left(a + s\right)}, \  \frac{\omega}{\omega^{2} + \left(a + s\right)^{2}}, \  \frac{a + s}{\omega^{2} + \left(a + s\right)^{2}}\right]\; $

```python
F = sp.Function('f')(t)
F
```

```python
L(sp.diff(t**2*sp.exp(-a*t),t),t,s)
```

```python

```

Jupyter notebooks/Circuitos Eletricos I - Semana 11.1.ipynb