# Neuromatch Academy: Week 2, Day 1, Tutorial 2
# Deep Learning: Convolutional Neural Networks

**Content creators**: Jorge A. Menendez, Carsen Stringer

**Content reviewers**: Roozbeh Farhoodi, Ella Batty, Kshitij Dwivedi, Spiros Chavlis, Michael Waskom

---
#Tutorial Objectives

In this short tutorial, we'll go through an introduction to 2D convolutions and apply a convolutional network to an image to prepare for creating normative models in tutorial 3.

In this tutorial, we will
* Understand the basics of 2D convolution
* Build a convolutional layer using PyTorch
* Visualize and analyze its outputs
* (Bonus) Build an encoding model from images to neural responses

---
# Setup

```python
import os
import numpy as np
import torch
from torch import nn
from torch import optim
from matplotlib import pyplot as plt
import matplotlib as mpl
```

```python
#@title Figure settings
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
#@title Data retrieval and loading
import hashlib
import requests

fname = "W3D4_stringer_oribinned6_split.npz"
url = "https://osf.io/p3aeb/download"
expected_md5 = "b3f7245c6221234a676b71a1f43c3bb5"

if not os.path.isfile(fname):
  try:
    r = requests.get(url)
  except requests.ConnectionError:
    print("!!! Failed to download data !!!")
  else:
    if r.status_code != requests.codes.ok:
      print("!!! Failed to download data !!!")
    elif hashlib.md5(r.content).hexdigest() != expected_md5:
      print("!!! Data download appears corrupted !!!")
    else:
      with open(fname, "wb") as fid:
        fid.write(r.content)
```

```python
#@title Plotting Functions

def plot_tuning(ax, stimuli, respi_train, respi_test, neuron_index, linewidth=2):
  """Plot the tuning curve of a neuron"""

  ax.plot(stimuli, respi_train, 'y', linewidth=linewidth)  # plot its responses as a function of stimulus orientation
  ax.plot(stimuli, respi_test, 'm', linewidth=linewidth)  # plot its responses as a function of stimulus orientation
  ax.set_title('neuron %i' % neuron_index)
  ax.set_xlabel('stimulus orientation ($^o$)')
  ax.set_ylabel('neural response')
  ax.set_xticks(np.linspace(0, 360, 5))
  ax.set_ylim([-0.5, 2.4])

def show_stimulus(img, ax=None):
  """Visualize a stimulus"""
  if ax is None:
    ax = plt.gca()
  ax.imshow(img+0.5, cmap=mpl.cm.binary)
  ax.set_xticks([])
  ax.set_yticks([])
  ax.spines['left'].set_visible(False)
  ax.spines['bottom'].set_visible(False)

def plot_example_activations(stimuli, act, channels=[0]):
  """ plot activations act and corresponding stimulus
  Args:
        stimuli: stimulus input to convolutional layer (n x h x w) or (h x w)
        act: activations of convolutional layer (n_bins x conv_channels x n_bins)
        channels: which conv channels to plot
  """
  if stimuli.ndim>2:
    n_stimuli = stimuli.shape[0]
  else:
    stimuli = stimuli.unsqueeze(0)
    n_stimuli = 1

  fig, axs = plt.subplots(n_stimuli,1+len(channels),figsize=(12,12))

  # plot stimulus
  for i in range(n_stimuli):
    show_stimulus(stimuli[i].squeeze(), ax=axs[i, 0])
    axs[i, 0].set_title('stimulus')

    # plot example activations
    for k, (channel, ax) in enumerate(zip(channels, axs[i][1:])):
      img=ax.imshow(act[i,channel], vmin=-6, vmax=6, cmap='bwr')
      ax.set_xlabel('x-pos')
      ax.set_ylabel('y-pos')
      ax.set_title('channel %d'%channel)
  ax = fig.add_axes([1.05,0.8,0.01,0.1])
  plt.colorbar(img, cax=ax)
  ax.set_title('activation\n strength')

def plot_weights(weights, channels=[0], colorbar=True):
  """ plot convolutional channel weights
  Args:
        weights: weights of convolutional filters (conv_channels x K x K)
        channels: which conv channels to plot
  """
  wmax = torch.abs(weights).max()
  fig, axs = plt.subplots(1,len(channels), figsize=(12,2.5))
  for i, channel in enumerate(channels):
    im = axs[i].imshow(weights[channel,0], vmin=-wmax, vmax=wmax, cmap='bwr')
    axs[i].set_title('channel %d'%channel)

  if colorbar:
    ax = fig.add_axes([1, 0.1, 0.05, 0.8])
    plt.colorbar(im, ax=ax)
    ax.axis('off')

def plot_prediction(ax, y_pred, y_train, y_test):
  """ plot prediction of neural response + test neural response """
  ax.plot(y_train, 'y', linewidth=1)
  ax.plot(y_test,color='m')
  ax.plot(y_pred, 'g', linewidth=3)
  ax.set_xlabel('stimulus bin')
  ax.set_ylabel('response')

def plot_training_curves(train_loss, test_loss):
  """
  Args:
    train_loss (list): training error over iterations
    test_loss (list): n_test x 1 tensor with orientations of the
      stimuli corresponding to each row of train_data, in radians
    predicted_test_labels (torch.Tensor): n_test x 1 tensor with predicted orientations of the
      stimuli from decoding neural network

  """

  f, ax = plt.subplots()
  # Plot the training loss over iterations of GD
  ax.plot(train_loss)
  # Plot the testing loss over iterations of GD
  ax.plot(test_loss, '.', markersize=10)
  ax.legend(['train loss', 'test loss'])
  ax.set(xlabel="Gradient descent iteration", ylabel="Mean squared error")
```

```python
#@title Helper Functions

def load_data_split(data_name=fname):
  """Load mouse V1 data from Stringer et al. (2019)

  Data from study reported in this preprint:
  https://www.biorxiv.org/content/10.1101/679324v2.abstract

  These data comprise time-averaged responses of ~20,000 neurons
  to ~4,000 stimulus gratings of different orientations, recorded
  through Calcium imaginge. The responses have been normalized by
  spontaneous levels of activity and then z-scored over stimuli, so
  expect negative numbers. The repsonses were split into train and
  test and then each set were averaged in bins of 6 degrees.

  This function returns the relevant data (neural responses and
  stimulus orientations) in a torch.Tensor of data type torch.float32
  in order to match the default data type for nn.Parameters in
  Google Colab.

  It will hold out some of the trials when averaging to allow us to have test
  tuning curves.

  Args:
    data_name (str): filename to load

  Returns:
    resp_train (torch.Tensor): n_stimuli x n_neurons matrix of neural responses,
        each row contains the responses of each neuron to a given stimulus.
        As mentioned above, neural "response" is actually an average over
        responses to stimuli with similar angles falling within specified bins.
    resp_test (torch.Tensor): n_stimuli x n_neurons matrix of neural responses,
        each row contains the responses of each neuron to a given stimulus.
        As mentioned above, neural "response" is actually an average over
        responses to stimuli with similar angles falling within specified bins
    stimuli: (torch.Tensor): n_stimuli x 1 column vector with orientation
        of each stimulus, in degrees. This is actually the mean orientation
        of all stimuli in each bin.

  """
  with np.load(data_name) as dobj:
    data = dict(**dobj)
  resp_train = data['resp_train']
  resp_test = data['resp_test']
  stimuli = data['stimuli']

  # Return as torch.Tensor
  resp_train_tensor = torch.tensor(resp_train, dtype=torch.float32)
  resp_test_tensor = torch.tensor(resp_test, dtype=torch.float32)
  stimuli_tensor = torch.tensor(stimuli, dtype=torch.float32)

  return resp_train_tensor, resp_test_tensor, stimuli_tensor

def grating(angle, sf=1 / 28, res=0.1, patch=False):
  """Generate oriented grating stimulus

  Args:
    angle (float): orientation of grating (angle from vertical), in degrees
    sf (float): controls spatial frequency of the grating
    res (float): resolution of image. Smaller values will make the image
      smaller in terms of pixels. res=1.0 corresponds to 640 x 480 pixels.
    patch (boolean): set to True to make the grating a localized
      patch on the left side of the image. If False, then the
      grating occupies the full image.

  Returns:
    torch.Tensor: (res * 480) x (res * 640) pixel oriented grating image

  """

  angle = np.deg2rad(angle)  # transform to radians

  wpix, hpix = 640, 480  # width and height of image in pixels for res=1.0

  xx, yy = np.meshgrid(sf * np.arange(0, wpix * res) / res, sf * np.arange(0, hpix * res) / res)

  if patch:
    gratings = np.cos(xx * np.cos(angle + .1) + yy * np.sin(angle + .1))  # phase shift to make it better fit within patch
    gratings[gratings < 0] = 0
    gratings[gratings > 0] = 1
    xcent = gratings.shape[1] * .75
    ycent = gratings.shape[0] / 2
    xxc, yyc = np.meshgrid(np.arange(0, gratings.shape[1]), np.arange(0, gratings.shape[0]))
    icirc = ((xxc - xcent) ** 2 + (yyc - ycent) ** 2) ** 0.5 < wpix / 3 / 2 * res
    gratings[~icirc] = 0.5

  else:
    gratings = np.cos(xx * np.cos(angle) + yy * np.sin(angle))
    gratings[gratings < 0] = 0
    gratings[gratings > 0] = 1

  gratings -= 0.5

  # Return torch tensor
  return torch.tensor(gratings, dtype=torch.float32)

def filters(K=7):
  """ make example filters, some center-surround and gabors
  Returns:
      filters: 6 x K x K
  """
  grid = np.linspace(-K/2, K/2, K).astype(np.float32)
  xx,yy = np.meshgrid(grid, grid, indexing='ij')

  # create center-surround filters
  sigma = 1.1
  gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*sigma**2))
  wide_gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*(sigma*2)**2))
  center_surround = gaussian - 0.5 * wide_gaussian

  # create gabor filters
  thetas = np.array([0, 45, 90, 135]) * np.pi/180
  gabors = np.zeros((len(thetas), K, K), np.float32)
  lam = 10
  phi = np.pi/2
  gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*(sigma*0.4)**2))
  for i,theta in enumerate(thetas):
    x = xx*np.cos(theta) + yy*np.sin(theta)
    gabors[i] = gaussian * np.cos(2*np.pi*x/lam + phi)

  filters = np.concatenate((center_surround[np.newaxis,:,:],
                            -1*center_surround[np.newaxis,:,:],
                            gabors),
                           axis=0)
  filters /= np.abs(filters).max(axis=(1,2))[:,np.newaxis,np.newaxis]
  # convert to torch
  filters = torch.from_numpy(filters)
  # add channel axis
  filters = filters.unsqueeze(1)

  return filters

def regularized_MSE_loss(output, target, weights=None, L2_penalty=0, L1_penalty=0):
  """loss function for MSE

  Args:
    output (torch.Tensor): output of network
    target (torch.Tensor): neural response network is trying to predict
    weights (torch.Tensor): fully-connected layer weights of network (net.out_layer.weight)
    L2_penalty : scaling factor of sum of squared weights
    L1_penalty : scalaing factor for sum of absolute weights

  Returns:
    (torch.Tensor) mean-squared error with L1 and L2 penalties added

  """

  loss_fn = nn.MSELoss()
  loss = loss_fn(output, target)

  if weights is not None:
    L2 = L2_penalty * torch.square(weights).sum()
    L1 = L1_penalty * torch.abs(weights).sum()
    loss += L1 + L2

  return loss

def train(net, custom_loss, train_data, train_labels,
          test_data=None, test_labels=None,
          learning_rate=10, n_iter=500, L2_penalty=0., L1_penalty=0.):
  """Run gradient descent for network without batches

  Args:
    net (nn.Module): deep network whose parameters to optimize with SGD
    custom_loss: loss function for network
    train_data: training data (n_train x input features)
    train_labels: training labels (n_train x output features)
    test_data: test data (n_train x input features)
    test_labels: test labels (n_train x output features)
    learning_rate (float): learning rate for gradient descent
    n_epochs (int): number of epochs to run gradient descent
    L2_penalty (float): magnitude of L2 penalty
    L1_penalty (float): magnitude of L1 penalty

  Returns:
    train_loss: training loss across iterations
    test_loss: testing loss across iterations

  """
  optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9) # Initialize PyTorch SGD optimizer
  train_loss = np.nan * np.zeros(n_iter)  # Placeholder for train loss
  test_loss = np.nan * np.zeros(n_iter)  # Placeholder for test loss

  # Loop over epochs
  for i in range(n_iter):
    if n_iter < 10:
      for param_group in self.optimizer.param_groups:
        param_group['lr'] = np.linspace(0, learning_rate, 10)[n_iter]
    y_pred = net(train_data) # Forward pass: compute predicted y by passing train_data to the model.

    if L2_penalty>0 or L1_penalty>0:
      weights = net.out_layer.weight
      loss = custom_loss(y_pred, train_labels, weights, L2_penalty, L1_penalty)
    else:
      loss = custom_loss(y_pred, train_labels)

    ### Update parameters
    optimizer.zero_grad() # zero out gradients
    loss.backward() # Backward pass: compute gradient of the loss with respect to model parameters
    optimizer.step() # step parameters in gradient direction
    train_loss[i] = loss.item()  # .item() transforms the tensor to a scalar and does .detach() for us

    # Track progress
    if (i+1) % (n_iter // 10) == 0 or i==0:
      if test_data is not None and test_labels is not None:
        y_pred = net(test_data)
        if L2_penalty>0 or L1_penalty>0:
          loss = custom_loss(y_pred, test_labels, weights, L2_penalty, L1_penalty)
        else:
          loss = custom_loss(y_pred, test_labels)
        test_loss[i] = loss.item()
        print(f'iteration {i+1}/{n_iter} | train loss: {train_loss[i]:.4f} | test loss: {test_loss[i]:.4f}')
      else:
        print(f'iteration {i+1}/{n_iter} | train loss: {train_loss[i]:.4f}')

  return train_loss, test_loss
```

---
# Section 1: Introduction to 2D convolutions

## Section 1.1: What is a 2D convolution?

A 2D convolution is an integral of the product of a filter $f$ and an input image $I$ computed at various positions as the filter is slid across the input. The output of the convolution operation at position $(x,y)$ can be written as follows, where the filter $f$ is size $(K,K)$:
\begin{equation}
C(x,y) = \sum_{k_x=-K/2}^{K/2} \sum_{k_y=-K/2}^{K/2} f(k_x,k_y) I(x+k_x,y+k_y)
\end{equation}
This **convolutional filter** is often called a **kernel**.

Here is an illustration of a 2D convolution from this [article](https://towardsdatascience.com/light-on-math-machine-learning-intuitive-guide-to-convolution-neural-networks-e3f054dd5daa):

```python
#@title
#@markdown Execute this cell to view convolution gif

from IPython.display import Image
Image(url='https://miro.medium.com/max/700/1*5BwZUqAqFFP5f3wKYQ6wJg.gif')
```

## Section 1.2: 2D convolutions in deep learning

```python
#@title Video 1: 2D Convolutions
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="igLkHuuJurE", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

In the [intro](https://www.youtube.com/watch?v=IZvcy0Myb3M) you learned about convolutional networks for computer vision tasks and applications to neuroscience. Object recognition was essentially an unsolved problem in machine learning until the [advent](https://en.wikipedia.org/wiki/AlexNet) of techniques for effectively training *deep* convolutional neural networks. See Bonus Section 1 for more information on why we use CNNs to model the visual system.

Convolutional neural networks consist of 2D convolutional layers, RELU non-linearities, 2D pooling layers, and at the output, a fully connected layer. We will see an example network with all these components in tutorial 3.

A 2D convolutional layer consists of multiple channels. Each **channel** is a 2D convolutional filter applied to the input resulting in several units, where the number of units depends on the *stride* you set. In most applications, especially with small filter sizes, a stride of 1 is used.

(*Technical note*: if filter size *K* is odd and you set the *pad=K//2* and *stride=1* (as is shown below), you get a **channel** of units that is the same size as the input.)

```python
#@title
#@markdown Execute this cell to view convolution gif

from IPython.display import Image
Image(url='https://miro.medium.com/max/790/1*1okwhewf5KCtIPaFib4XaA.gif')
```

## Section 1.3: 2D convolutions in Pytorch

In Tutorial 1, fully connected linear layers were used to decode stimuli from neural activity. Convolutional layers are different from their fully connected counterparts in two ways:
  * In a fully connected layer, each unit computes a weighted sum over all the input units. In a convolutional layer, on the other hand, each unit computes a weighted sum over only a small patch of the input, referred to as the unit's **receptive field**. When the input is an image, the receptive field can be thought of as a local patch of pixels.
  * In a fully connected layer, each unit uses its own independent set of weights to compute the weighted sum. In a convolutional layer, all the units (within the same channel) **share the same weights**. This set of shared weights is called the **convolutional filter or kernel**. The result of this computation is a convolution, where each unit has computed the same weighted sum over a different part of the input. This reduces the number of parameters in the network substantially.

We will compute the difference in the number of weights for a fully connected layer versus a convolutional layer in the Think exercise below.

First, let's visualize the stimuli in the dataset from tutorial 1. During the neural recordings from [Stringer et al 2019](https://www.biorxiv.org/content/10.1101/679324v2.abstract), mice were presented oriented gratings:

```python
#@title
#@markdown Execute this cell to plot example stimuli

orientations = np.linspace(-90, 90, 5)

h = 3
n_col = len(orientations)
fig, axs = plt.subplots(1, n_col, figsize=(h * n_col, h))

h, w  = grating(0).shape  # height and width of stimulus
print('stimulus size: %i x %i' % (h, w))

for i, ori in enumerate(orientations):
  stimulus = grating(ori)
  axs[i].set_title(f'{ori: .0f}$^o$')
  show_stimulus(stimulus, axs[i])
```

Now let's implement 2D convolutional operations. We will use mutliple convolutional channels and implement this operation efficiently using pytorch. A *layer* of convolutional channels can be implemented with one line of code using the PyTorch class `nn.Conv2d()`, which requires the following arguments for initialization (see full documentation [here](https://pytorch.org/docs/master/generated/torch.nn.Conv2d.html)):
  * $C^{in}$: the number of input channels
  * $C^{out}$: the number of output channels (number of different convolutional filters)
  * $K$: the size of the $C^{out}$ different convolutional filters

When you run the network, you can input a stimulus of arbitrary size $(H^{in}, W^{in})$, but it needs to be shaped as a 4D input $(N, C^{in}, H^{in}, W^{in})$, where $N$ is the number of images. In our case, $C^{in}=1$ because there is only one color channel (our images are grayscale, but often $C^{in}=3$ in image processing).

```python
class ConvolutionalLayer(nn.Module):
  """Deep network with one convolutional layer
     Attributes: conv (nn.Conv2d): convolutional layer
  """
  def __init__(self, c_in=1, c_out=6, K=7, filters=None):
    """Initialize layer

    Args:
        c_in: number of input stimulus channels
        c_out: number of output convolutional channels
        K: size of each convolutional filter
        filters: (optional) initialize the convolutional weights

    """
    super().__init__()
    self.conv = nn.Conv2d(c_in, c_out, kernel_size=K,
                          padding=K//2, stride=1)
    if filters is not None:
      self.conv.weight = nn.Parameter(filters)
      self.conv.bias = nn.Parameter(torch.zeros((c_out,), dtype=torch.float32))

  def forward(self, s):
    """Run stimulus through convolutional layer

    Args:
        s (torch.Tensor): n_stimuli x h x w tensor with stimuli

    Returns:
        (torch.Tensor): n_stimuli x c_out x w x h tensor with convolutional layer unit activations.

    """
    s = s.unsqueeze(1) # add channel dimension
    a = self.conv(s)  # output of convolutional layer

    return a
```

### Coding Exercise 1.2: 2D convolution in pytorch

We will now run the convolutional layer on our stimulus. We will use gratings stimuli made using the function `grating`, which returns a stimulus which is 48 x 64.

Reminder, `nn.Conv2d` takes in a tensor of size $(N, C^{in}, H^{in}, W^{in}$) where $N$ is the number of stimuli, $C^{in}$ is the number of input channels, and $(H^{in}, W^{in})$ is the size of the stimulus. We will need to add these first two dimensions to our stimulus, then input it to the convolutional layer.

We will plot the outputs of the convolution. `convout` is a tensor of size $(N, C^{out}, H^{in}, W^{in})$ where $N$ is the number of examples and $C^{out}$ are the number of convolutional channels. It is the same size as the input because we used a stride of 1 and padding that is half the kernel size.

```python
# Convolution layer parameters
K = 7 # filter size
conv_channels = 6 # how many convolutional channels to have in our layer
convout = np.zeros(0) # assign convolutional activations to convout

################################################################################
## TODO for students: create convolutional layer in pytorch
# Complete and uncomment
################################################################################

### Initialize conv layer and add weights from function filters
# you need to specify :
# * the number of input channels c_in
# * the number of output channels c_out
# * the filter size K
# convLayer = ConvolutionalLayer(..., filters=filters(K))

### Create stimuli (H_in, W_in)
stimuli = torch.zeros((len(orientations), 48, 64), dtype=torch.float32)
orientations = [-90, -45, 0, 45, 90]
for i,ori in enumerate(orientations):
  stimuli[i] = grating(ori)

# convout = convLayer(...)
# convout = convout.detach() # detach gradients

# plot_example_activations(stimuli, convout, channels=np.arange(0, conv_channels))
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_3f2bd25a.py)

*Example output:*

#### Think! 2.3: Output and weight shapes in a convolutional layer

Let's think about the shape of the weights and outputs of `convLayer`:
  - How many convolutional activations are there in each channel, and why is it this size?
  - How many weights does `convLayer` have?
  - How many weights would it have if it were a fully connected layer?

Additionally, let's think about why the activations look the way they do. It seems like for all channels the activations are only non-zero for edges of the gratings (where the grating goes from white-to-black and from black-to-white).
  -  Channel 0 and 1 seem to respond to every edge regardless of the orientation, but their signs are different. What type of filter might produce these types of responses?
  - Channels 2-5 seem to respond differently depending on the orientation of the stimulus. What type of filter might produce these types of responses?

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_455be0bb.py)

Please see Bonus Section 2 to visualize the convolutional filter weights. See Bonus Section 3 to use CNNs as encoding models of neurons (by fitting directly to neural responses).

---
# Summary

In this notebook, we built a 2D convolutional layer which is meant to represent the responses of neurons in mouse visual cortex, or the responses of neurons
which are inputs to mouse visual cortex.

In Tutorial 3, we will add to this 2D convolutional layer a fully-connected layer and train this model to predict whether an orientation is left or right. We will see if the convolutional filters it learns are similar to mouse visual cortex.

---
# Bonus

## Bonus Section 1: Why CNN's?

CNN models are particularly [well-suited](https://www.nature.com/articles/nn.4244) to modeling the visual system for a number of reasons:

1. **Distributed computation**: like any other neural network, CNN's use distributed representations to compute -- much like the brain seems to do. Such models, therefore, provide us with a vocabulary with which to talk and think about such distributed representations. Because we know the exact function the model is built to perform (e.g. orientation discrimination), we can analyze its internal representations with respect to this function and begin to interpret why the representations look the way they do. Most importantly, we can then use these insights to analyze the structure of neural representations observed in recorded population activity. We can qualitatively and quantitatively compare the representations we see in the model and in a given population of real neurons to hopefully tease out the computations it performs.

2. **Hierarchical architecture**: like in any other deep learning architecture, each layer of a deep CNN comprises a non-linear transformation of the previous layer. Thus, there is a natural hierarchy whereby layers closer to the network output represent increasingly more abstract information about the input image. For example, in a network trained to do object recognition, the early layers might represent information about edges in the image, whereas later layers closer to the output might represent various object categories. This resembles the [hierarchical structure of the visual system](https://pubmed.ncbi.nlm.nih.gov/1822724/), where [lower-level areas](https://www.jneurosci.org/content/25/46/10577.short) (e.g. retina, V1) represent visual features of the sensory input and [higher-level areas](https://www.sciencedirect.com/science/article/pii/S089662731200092X) (e.g. V4, IT) represent properties of objects in the visual scene. We can then naturally use a single CNN to model multiple visual areas, using early CNN layers to model lower-level visual areas and late CNN layers to model higher-level visual areas.

  Relative to fully connected networks, CNN's, in fact, have further hierarchical structure built-in through the max pooling layers. Recall that each output of a convolution + pooling block is the result of processing a local patch of the inputs to that block. If we stack such blocks in a sequence, then the outputs of each block will be sensitive to increasingly larger regions of the initial raw input to the network: an output from the first block is sensitive to a single patch of these inputs, corresponding to its "receptive field"; an output from the second block is sensitive to a patch of outputs from the first block, which together are sensitive to a larger patch of raw inputs comprising the union of their receptive fields. Receptive fields thus get larger for deeper layers (see [here](http://colah.github.io/posts/2014-07-Conv-Nets-Modular/) for a nice visual depiction of this). This resembles primate visual systems, where neurons in higher-level visual areas respond to stimuli in wider regions of the visual field than neurons in lower-level visual areas.

3. **Convolutional layers**: through the weight sharing constraint, the outputs of each channel of a convolutional layer process different parts of the input image in exactly the same way. This architectural constraint effectively builds into the network the assumption that objects in the world typically look the same regardless of where they are in space. This is useful for modeling the visual system for two (largely separate) reasons:
  * Firstly, this assumption is generally valid in mammalian visual systems, since mammals tend to view the same object from many perspectives. Two neurons at a similar hierarchy in the visual system with different receptive fields could thus end up receiving statistically similar synaptic inputs, so that the synaptic weights developed over time may end up being similar as well.
  * Secondly, this architecture significantly improves object recognition ability. Object recognition was essentially an unsolved problem in machine learning until the [advent](https://en.wikipedia.org/wiki/AlexNet) of techniques for effectively training *deep* convolutional neural networks. Fully connected networks on their own can't achieve object recognition abilities anywhere close to human levels, making them bad models of human object recognition. Indeed, it is generally the case that [the better a neural network model is at object recognition, the closer the match between its representations and those observed in the brain](https://www.pnas.org/content/111/23/8619.short). That said, it is worth noting that our much simpler orientation discrimination task (in Tutorial 3) can be solved by relatively simple networks.

---
## Bonus Section 2: Understanding activations from weight

### Bonus Coding Exercise 2: Visualizing convolutional filter weights

Why do the activations look the way they do? Let's look at the weights (`convLayer.conv.weight`) of the convolutional filters and try to interpret them.

```python
################################################################################
## TODO for students: create convolutional layer in pytorch
# Complete and uncomment
################################################################################

# get weights of conv layer in convLayer
# weights = ...
# print(weights.shape) # can you identify what each of the dimensions are?

# plot_weights(weights, channels=np.arange(0, conv_channels))
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_a1ae60f7.py)

*Example output:*

In the function `filters` we premade center-surround filters and Gabor filters of various orientations. [Gabor filters](https://en.wikipedia.org/wiki/Gabor_filter) have a positive region next to a negative region and the orientation of these regions of the filter determine the orientation of edges to which they respond.

In visual cortex, Hubel and Wiesel discovered simple cells, which would correspond to a unit in the channels 2-5 above with Gabor filters. There are also some neurons with activity that resembles center-surround filters, which would correspond to the first two convolutional channels above.

There were additional cells discovered by Hubel and Wiesel - complex cells - that respond to an oriented grating regardless of where the bars are exactly (note that the responses we see are specific to where the bars are). These cells therefore have some level of translation invariance. This is something that convolutional neural networks try to replicate -- e.g. a grating is still oriented horizontal even if it moves slightly, and a cat is still a cat even if it's in a different position in the image.

#### Think!
  - How might you create a complex cell and have responses that are translation invariant?
  - How might you create a cell that responds to multiple orientations?

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_13ec7f73.py)

---
## Bonus Section 3: Neural encoding model

```python
#@title Video 2: Convolutional Encoding Model
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="TCAQYIoqcgA", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

In neuroscience, we often want to understand how the brain represents external stimuli. One approach to discovering these representations is to build an encoding model that takes as input the external stimuli (in this case grating stimuli) and outputs the neural responses.

Because visual cortex is often thought to be a convolutional network where the same filters are combined across the visual field, we will use a model with a convolutional layer. We learned how to build a convolutional layer in the previous section. We will add to this convolutional layer a fully connected layer from the output of the convolutions to the neurons. We will then visualize the weights of this fully connected layer.

### Bonus Section 3.1: Neural tuning curves

In the next cell, we plot the turning curves of a random subset of neurons. We have binned the stimuli orientations more than in Tutorial 1. We create the gratings images as above for the 60 orientations below, and save them to a variable `grating_stimuli`.

Rerun the cell to look at different example neurons and observe the diversity of tuning curves in the population. How can we fit these neural responses with an encoding model?

```python
#@title
#@markdown Execute this cell to load data, create stimuli, and plot neural tuning curves

### Load data and bin at 8 degrees
# responses are split into test and train
resp_train, resp_test, stimuli = load_data_split()
n_stimuli, n_neurons = resp_train.shape
print('resp_train contains averaged responses of %i neurons to %i binned stimuli' % (n_neurons, n_stimuli))
#print(resp_train.shape)

# also make stimuli into images
orientations = np.linspace(0, 360, 61)[:-1] - 180
grating_stimuli = np.zeros((60, 12, 16), np.float32)
for i, ori in enumerate(orientations):
  grating_stimuli[i] = grating(ori, res=0.025)#[18:30, 24:40]

grating_stimuli = torch.from_numpy(grating_stimuli)
print('grating_stimuli contains 60 stimuli of size 12 x 16')

# Visualize tuning curves
fig, axs = plt.subplots(3, 5, figsize=(15,7))
for k, ax in enumerate(axs.flatten()):
  neuron_index = np.random.choice(n_neurons)  # pick random neuron
  plot_tuning(ax, stimuli, resp_train[:, neuron_index], resp_test[:, neuron_index], neuron_index, linewidth=2)
  if k==0:
    ax.text(1.0, 0.9, 'train', color='y', transform=ax.transAxes)
    ax.text(1.0, 0.65, 'test', color='m', transform=ax.transAxes)
fig.tight_layout()
plt.show()
```

### Bonus Section 3.2: Adding a fully-connected layer to create encoding model

We will build a torch model like above with a convolutional layer. Additionally, we will add a fully connected linear layer from the convolutional units to the neurons. We will use 6 convolutional channels ($C^{out}$) and a kernel size ($K$) of 7 with a stride of 1 and padding of $K/2$ (same as above). The stimulus is size `(12, 16)`. Then the convolutional unit activations will go through a linear layer to be transformed into neural responses.

#### Think! 3.2: Number of unis and weights

* How many units will the convolutional layer have?
* How many weights will the fully connected linear layer from convolutional units to neurons have?

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_1240d23d.py)

#### Bonus Coding Exercise 3.2: Add linear layer

Remember in Tutorial 1 we used linear layers. Use your knowledge from Tutorial 1 to add a linear layer to the model we created above.

```python
class ConvFC(nn.Module):
  """Deep network with one convolutional layer + one fully connected layer

  Attributes:
    conv (nn.Conv1d): convolutional layer
    dims (tuple): shape of convolutional layer output
    out_layer (nn.Linear): linear layer

  """

  def __init__(self, n_neurons, c_in=1, c_out=6, K=7, b=12*16,
               filters=None):
    """ initialize layer
    Args:
        c_in: number of input stimulus channels
        c_out: number of convolutional channels
        K: size of each convolutional filter
        h: number of stimulus bins, n_bins
    """
    super().__init__()
    self.conv = nn.Conv2d(c_in, c_out, kernel_size=K,
                          padding=K//2, stride=1)
    self.dims = (c_out, b)  # dimensions of conv layer output
    M = np.prod(self.dims) # number of hidden units

    ################################################################################
    ## TO DO for students: add fully connected layer to network (self.out_layer)
    # Fill out function and remove
    raise NotImplementedError("Student exercise: add fully connected layer to initialize network")
    ################################################################################
    self.out_layer = nn.Linear(M, ...)

    # initialize weights
    if filters is not None:
      self.conv.weight = nn.Parameter(filters)
      self.conv.bias = nn.Parameter(torch.zeros((c_out,), dtype=torch.float32))

    nn.init.normal_(self.out_layer.weight, std=0.01) # initialize weights to be small

  def forward(self, s):
    """ Predict neural responses to stimuli s

    Args:
        s (torch.Tensor): p x L tensor with stimuli

    Returns:
        torch.Tensor: p x N tensor with convolutional layer unit activations.

    """
    s = s.unsqueeze(1)  # p x 1 x W x H, add a singleton dimension for the single channel
    a = self.conv(s)  # output of convolutional layer
    a = a.view(-1, np.prod(self.dims))  # flatten each convolutional layer output into a vector

    ################################################################################
    ## TO DO for students: add fully connected layer to forward pass of network (self.out_layer)
    # Fill out function and remove
    raise NotImplementedError("Student exercise: add fully connected layer to network")
    ################################################################################
    y = ...

    return y

device = torch.device('cpu')

# (Optional) To speed up processing, go to "Runtime" menu and "Change runtime"
# and select GPU processing, then uncomment line below, otherwise runtime will
# be ~ 2 minutes
# device = torch.device('cuda')

# Initialize network
n_neurons = resp_train.shape[1]
K = 7
# we will initialize with the same filters as above
net = ConvFC(n_neurons, filters=filters(K))
net = net.to(device)

# Run GD on training set data
# ** this time we are also providing the test data to estimate the test loss
train_loss, test_loss = train(net, regularized_MSE_loss,
                              train_data=grating_stimuli.to(device), train_labels=resp_train.to(device),
                              test_data=grating_stimuli.to(device), test_labels=resp_test.to(device),
                              n_iter=200, learning_rate=2,
                              L2_penalty=5e-4, L1_penalty=1e-6)

# Plot the training loss over iterations of GD
plot_training_curves(train_loss, test_loss)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_490d72a9.py)

*Example output:*

How well can we fit single neuron tuning curves with this model? What aspects of the tuning curves are we capturing?

```python
#@title
#@markdown Execute this cell to examine predictions for random subsets of neurons

y_pred = net(grating_stimuli.to(device))
# Visualize tuning curves & plot neural predictions
fig, axs = plt.subplots(2, 5, figsize=(15,6))
for k, ax in enumerate(axs.flatten()):
  ineur = np.random.choice(n_neurons)
  plot_prediction(ax, y_pred[:,ineur].detach().cpu(),
                  resp_train[:,ineur],
                  resp_test[:,ineur])
  if k==0:
    ax.text(.1, 1., 'train', color='y', transform=ax.transAxes)
    ax.text(.1, 0.9, 'test', color='m', transform=ax.transAxes)
    ax.text(.1, 0.8, 'prediction', color='g', transform=ax.transAxes)

fig.tight_layout()
plt.show()
```

We can see if the convolutional channels changed at all from their initialization as center-surround and Gabor filters. If they don't then it means that they were a sufficient basis set to explain the responses of the neurons to orientations to the accuracy seen above.

```python
# get weights of conv layer in convLayer
weights = net.conv.weight.detach().cpu()
print(weights.shape) # can you identify what each of the dimensions are?

plot_weights(weights, channels=np.arange(0, conv_channels))
```


```python
import numpy as np
import sympy as sp
from PIL import Image
from scipy import interpolate
import matplotlib.pyplot as plt
```

# Tarea 3:Interpolación Bicúbica

### Felipe Montero Concha
### 201473611-8

## Instrucciones

* La tarea es individual.
* Las consultas sobre las tareas se deben realizar por medio de la plataforma Aula.
* La tarea debe ser realizada en `Jupyter Notebook` (`Python3`).
* Se evaluará la correcta utilización de librerias `NumPy`, `SciPy`, entre otras, así como la correcta implementación de algoritmos de forma vectorizada.
*  **El archivo de entrega debe denominarse ROL-tarea-numero.ipynb**. _De no respetarse este formato existirá un descuento de **50 puntos**_
* La fecha de entrega es el viernes 24 de Julio a las **18:00 hrs**.  Se aceptarán entregas hasta las 19:00 hrs sin descuento en caso de existir algun problema, posteriormente existirá un descuento lineal hasta las 20:00 hrs del mismo día.
* Las tareas que sean entregadas antes del jueves a mediodía recibirán una bonificación de 10 puntos
* Debe citar cualquier código ajeno utilizado (incluso si proviene de los Jupyter Notebooks del curso).

## Introducción

En la siguiente tarea estudiaremos un método de interpolación denominado **Interpolación Bicúbica**, utilizada frecuentemente sobre imágenes. Aplicaremos el método para aumentar la resolución de una imagen intentando preservar las propiedades de la versión original.

## Contexto

Supongamos que usted conoce $f$ y las derivadas $f_x$, $f_y$ y $f_{xy}$ dentro de las coordenadas $(0,0),(0,1),(1,0)$ y $(1,1)$ de un cuadrado unitario. La superficie que interpola estos 4 puntos es:

$$
p(x,y) = \sum\limits_{i=0}^3 \sum_{j=0}^3 a_{ij} x^i y^j.
$$

Como se puede observar el problema de interpolación se resume en determinar los 16 coeficientes $a_{ij}$ y para esto se genera un total de $16$ ecuaciones utilizando los valores conocidos de $f$,$f_x$,$f_y$ y $f_{xy}$. Por ejemplo, las primeras $4$ ecuaciones son:

$$
\begin{aligned}
f(0,0)&=p(0,0)=a_{00},\\
f(1,0)&=p(1,0)=a_{00}+a_{10}+a_{20}+a_{30},\\
f(0,1)&=p(0,1)=a_{00}+a_{01}+a_{02}+a_{03},\\
f(1,1)&=p(1,1)=\textstyle \sum \limits _{i=0}^{3}\sum \limits _{j=0}^{3}a_{ij}.
\end{aligned}
$$

Para las $12$ ecuaciones restantes se debe utilizar:

$$
\begin{aligned}
f_{x}(x,y)&=p_{x}(x,y)=\textstyle \sum \limits _{i=1}^{3}\sum \limits _{j=0}^{3}a_{ij}ix^{i-1}y^{j},\\
f_{y}(x,y)&=p_{y}(x,y)=\textstyle \sum \limits _{i=0}^{3}\sum \limits _{j=1}^{3}a_{ij}x^{i}jy^{j-1},\\
f_{xy}(x,y)&=p_{xy}(x,y)=\textstyle \sum \limits _{i=1}^{3}\sum \limits _{j=1}^{3}a_{ij}ix^{i-1}jy^{j-1}.
\end{aligned}
$$

Una vez planteadas las ecuaciones, los coeficientes se pueden obtener al resolver el problema $A\alpha=x$, donde $\alpha=\left[\begin{smallmatrix}a_{00}&a_{10}&a_{20}&a_{30}&a_{01}&a_{11}&a_{21}&a_{31}&a_{02}&a_{12}&a_{22}&a_{32}&a_{03}&a_{13}&a_{23}&a_{33}\end{smallmatrix}\right]^T$ y ${\displaystyle x=\left[{\begin{smallmatrix}f(0,0)&f(1,0)&f(0,1)&f(1,1)&f_{x}(0,0)&f_{x}(1,0)&f_{x}(0,1)&f_{x}(1,1)&f_{y}(0,0)&f_{y}(1,0)&f_{y}(0,1)&f_{y}(1,1)&f_{xy}(0,0)&f_{xy}(1,0)&f_{xy}(0,1)&f_{xy}(1,1)\end{smallmatrix}}\right]^{T}}$.

En un contexto más aplicado, podemos hacer uso de la interpolación bicúbica para aumentar la resolución de una imagen. Supongamos que tenemos la siguiente imagen de tamaño $5 \times 5$:

Podemos ir tomando segmentos de la imagen de tamaño $2 \times 2$ de la siguiente forma:

Por cada segmento podemos generar una superficie interpoladora mediante el algoritmo de interpolación cubica. Para el ejemplo anterior estariamos generando $16$ superficies interpoladoras distintas. La idea es hacer uso de estas superficies para estimar los valores de los pixeles correspondienets a una imagen más grande. Por ejemplo, la imagen $5 \times 5$ la podemos convertir a una imagen de $9 \times 9$ agregando un pixel entre cada par de pixeles originales además de uno en el centro para que no quede un hueco.

Aca los pixeles verdes son los mismos que la imagen original y los azules son obtenidos de evaluar cada superficie interpoladora. Notar que existen pixeles azules que se pueden obtener a partir de dos superficies interpoladoras distintas, en esos casos se puede promediar el valor de los pixeles o simplemente dejar uno de los dos.

Para trabajar con la interpolación bicubica necesitamos conocer los valores de $f_x$, $f_y$ y $f_{xy}$. En el caso de las imagenes solo tenemos acceso al valor de cada pixel por lo que deberemos estimar cual es el valor de estos. Para estimar $f_x$ haremos lo siguiente:

Para estimar el valor de $f_x$ en cada pixel haremos una interpolación con los algoritmos conocidos, usando tres pixels en dirección de las filas, luego derivaremos el polinomio obtenido y finalmente evaluaremos en la posición de interes. La misma idea aplica para $f_y$ solo que ahora interpolaremos en dirección de las columnas.

Por ejemplo si queremos obtener el valor de $f_x$ en la posición $(0,0)$ (imagen de la izquierda) entonces haremos una interpolación de Lagrange utilizando los pixeles $(0,-1),(0,0)$ y $(0,1)$. Derivaremos el polinomio interpolador y evaluaremos en $(0,0)$. Por otro lado si queremos obtener el valor de $f_y$ en la posición $(0,0)$ (imagen de la derecha) entonces interpolaremos los pixeles $(-1,0),(0,0)$ y $(1,0)$. Luego derivaremos el polinomio interpolador y evaluaremos en $(0,0)$.

Para obtener $f_{xy}$ seguiremos la idea anterior. Solo que esta vez se utilizaran los valores de $f_y$ y se interpolaran estos en dirección de las filas.

# Preguntas

```python
#Codigo para abrir y visualizar imágenes
img = Image.open('img1.png')
array=np.array(img)
imgplot = plt.imshow(array)
plt.show()
```

## 1. Interpolación bicubica

### 1.1  Obtener derivadas (30 puntos)

Implemente la función `derivativeValues` que reciba como input un arreglo con valores, el método de interpolación y si es que se considera el uso de  los puntos de chebyshev . La función debe retornar un arreglo de igual dimensión con los valores de las derivadas de los puntos obtenidas

Los métodos de interpolación serán representados por los siguientes valores

* Interpolación de lagrange: `'lagrange'`
* Diferencias divididas de Newton: `'newton'`
* Spline cubica: `'spline3'`

```python
def newtonDD(x_i, y_i):
    n = x_i.shape[-1]
    pyramid = np.zeros((n, n)) # Create a square matrix to hold pyramid
    pyramid[:,0] = y_i # first column is y
    for j in range(1,n):
        for i in range(n-j):
            # create pyramid by updating other columns
            pyramid[i][j] = (pyramid[i+1][j-1] - pyramid[i][j-1]) / (x_i[i+j] - x_i[i])
    a = pyramid[0] # f[ ... ] coefficients
    return a

def derivativeValues(values, method, cheb):
    """
    Parameters
    ----------
    values:	(int array) points values
    method:	(string)	interpolation method
    cheb:	(boolean)	if chebyshev points are used

    Returns
    -------
    d:	(float array) derivative value of interpolated points

    """
    x = np.array(range(0,len(values)))
    if cheb:
        x = np.array([np.cos(np.pi*(2*k-1)/(2*len(values))) for k in range(1, len(values)+1)])
        x = np.array(x)
    if method == "lagrange":
        pol = interpolate.lagrange(x,values)
        pol = pol.deriv()
    elif method == "newton":
        pol = newtonDD(x,values)
        pol = np.poly1d(pol)
        pol = pol.deriv()
    elif method == "splne3":
        x.sort()
        pol = interpolate.CubicSpline(x,values)
        pol = interpolate.CubicSpline.derivative(pol)
    d = pol
    print (d)

    return d

derivativeValues(np.array([2,4,8]),"newton",True)
```

    4 x - 2.309

    poly1d([ 4.        , -2.30940108])

### 1.2 Interpolación de imagen (50 puntos)
Implemente la función `bicubicInterpolation` que reciba como input la matriz de la imagen y cuantos píxeles extra se quiere agregar entre los píxeles originales y el algoritmo de interpolación a utilizar. La función debe retornar la matriz con la imagen de dimensión nueva. Considere que se debe aplicar el método de interpolación en cada canal RGB por separado.

```python
def bicubicInterpolation(image, interiorPixels, method,cheb):
    """
    Parameters
    ----------
    image:	(nxnx3 array) image array in RGB format
    interiorPixels:	(int)	interpolation method
    method:	(string)	interpolation method
    cheb:	(boolean)	if chebyshev points are used

    Returns
    -------
    newImage:	(nxnx3 array) image array in RGB format

    """
    return newImage
```

## 2. Evaluacion de algoritmos

### 2.1 Tiempo de ejecucion
Implemente la funcion `timeInterpolation` que mida el tiempo de interpolacion de una imagen dado el algoritmo de interpolacion , en segundos.(5 puntos)

```python
def timeInterpolation(image, interiorPixels, method,cheb):
    """
    Parameters
    ----------
    image:	(nxnx3 array) image array in RGB format
    interiorPixels:	(int)	interpolation method
    method:	(string)	interpolation method
    cheb:	(boolean)	if chebyshev points are used

    Returns
    -------
    time:	(float) time in seconds

    """
    return time
```

***Pregunta: ¿Cual es el metodo que presenta mayor velocidad en general? (5 puntos)***

### 2.2 Calculo de error
Implemente la funcion `errorInterpolation` la cual debe obtener el error de la imagen obtenida comparandola con una de referencia. El error debe ser calculado utilizando el indice SSIM (Structural similarity) (5 puntos)

```python
def errorInterpolation(original,new):
    """
    Parameters
    ----------
    image:	(nxn array) original image array in RGB format
    new:	(nxn array) new image array in RGB format obtained from interpolation

    Returns
    -------
    error:	(float) difference between images

    """
    return error
```

***Pregunta: ¿Cual metodo presenta menor error? (5 puntos)***

# Consideraciones

* Solo trabajaremos con imagenes cuadradas
* En el caso que el valor interpolado de un punto sea mayor a 255 o menor a 0, este se trunca a 255 o 0 respectivamente
* Esta permitido el uso de sympy para calcular derivadas y para construir el polinomio interpolador
* El calculo de error puede ser calculado utilizando la imagen en escala de grises [(ejemplo)](https://scikit-image.org/docs/dev/auto_examples/transform/plot_ssim.html)

```python

```


```python
import numpy as np
import sympy as sym
import numba
import pydae.build as db
```

```python

```

\begin{eqnarray}
\dot \delta &=& \Omega_b \left(\omega - \omega_s\right)\\
\dot \omega &=& 1/(2 H) \left(p_m - p_e - D (\omega - \omega_s) \right)\\
\end{eqnarray}

$$ \sf
\Omega_{b} \left(\sf \omega - \omega_{s}\right)
$$

## System definition

```python
params_dict = {'T_1':2.0}

u_ini_dict = {'u_1':0.0}  # for the initialization problem
u_run_dict = {'u_1':0.0}  # for the running problem (here initialization and running problem are the same)

x_list = ['x_1']
y_ini_list = ['y_1']
y_run_list = ['y_1']

sys_vars = {'params':params_dict,
            'u_list':u_run_dict,
            'x_list':x_list,
            'y_list':y_run_list}

exec(db.sym_gen_str())  # exec to generate the required symbolic varables and constants
```

```python
dx1 = T_1*(u_1 - x_1)
g_1 = x_1 - y_1

f_list = [dx1,dx2]
g_list = [g_1,g_2]
h_dict = {'x_1':x_1**2,'y_1':y_1}

sys = {'name':'test',
       'params_dict':params_dict,
       'f_list':f_list,
       'g_list':g_list,
       'x_list':x_list,
       'y_ini_list':y_ini_list,
       'y_run_list':y_run_list,
       'u_run_dict':u_run_dict,
       'u_ini_dict':u_ini_dict,
       'h_dict':h_dict
      }

sys = db.system(sys)
db.sys2num(sys)

```

```python
sys
```

    {'name': 'test',
     'params_dict': {'T_1': 2.0},
     'f_list': [T_1*(u_1 - x_1**3), T_2*(u_2 - x_2)],
     'g_list': [1 - y_1, 1 - y_2],
     'x_list': ['x_1'],
     'y_ini_list': ['y_1'],
     'y_run_list': ['y_1'],
     'u_run_dict': {'u_1': 0.0},
     'u_ini_dict': {'u_1': 0.0},
     'h_dict': {'x_1': x_1**2, 'y_1': y_1},
     'f': Matrix([
     [T_1*(u_1 - x_1**3)],
     [   T_2*(u_2 - x_2)]]),
     'g': Matrix([
     [1 - y_1],
     [1 - y_2]]),
     'x': Matrix([[x_1]]),
     'y_ini': Matrix([[y_1]]),
     'y_run': Matrix([[y_1]]),
     'u_ini': Matrix([[u_1]]),
     'u_run': Matrix([[u_1]]),
     'h': Matrix([
     [x_1**2],
     [   y_1]]),
     'Fx_run': Matrix([
     [-3*T_1*x_1**2],
     [            0]]),
     'Fy_run': Matrix([
     [0],
     [0]]),
     'Gx_run': Matrix([
     [0],
     [0]]),
     'Gy_run': Matrix([
     [-1],
     [ 0]]),
     'Fx_ini': Matrix([
     [-3*T_1*x_1**2],
     [            0]]),
     'Fy_ini': Matrix([
     [0],
     [0]]),
     'Gx_ini': Matrix([
     [0],
     [0]]),
     'Gy_ini': Matrix([
     [-1],
     [ 0]]),
     'Fu_run': Matrix([
     [T_1],
     [  0]]),
     'Gu_run': Matrix([
     [0],
     [0]]),
     'Hx_run': Matrix([
     [2*x_1],
     [    0]]),
     'Hy_run': Matrix([
     [0],
     [1]]),
     'Hu_run': Matrix([
     [0],
     [0]])}

```python

```


# `GiRaFFEfood`: Initial data for `GiRaFFE` Aligned Rotator

## Author: Zach Etienne & Patrick Nelson
### Formatting improvements courtesy Brandon Clark

[comment]: <> (Abstract: TODO)

**Notebook Status:**  Validated

**Validation Notes:** This tutorial notebook has been confirmed to be self-consistent with its corresponding NRPy+ module, as documented [below](#code_validation). The initial data has validated against the original `GiRaFFE`, as documented [here](Tutorial-Start_to_Finish_UnitTest-GiRaFFEfood_NRPy.ipynb).

### NRPy+ Source Code for this module: [GiRaFFEfood_NRPy_Aligned_Rotator.py](../edit/GiRaFFEfood_NRPy/GiRaFFEfood_NRPy_Aligned_Rotator.py)

## Introduction:

This module provides another initial data option for $\texttt{GiRaFFE}$. This is a flat-spacetime test with initial data $$A_{\phi} = \frac{\mu \varpi^2}{r^3},$$ where  $\mu = B_p R^3_{\rm NS} / 2$, $R_{\rm NS}$ is the neutron star radius, and $\varpi = \sqrt{x^2+y^2}$ is the cylindrical radius. We let $A_r = A_\theta = 0$.

Additionally, the drift velocity $v^i = \Omega \textbf{e}_z \times \textbf{r} = [ijk] \Omega \textbf{e}^j_z x^k$, where $[ijk]$ is the Levi-Civita permutation symbol, $\Omega$ is the angular velocity of the "neutron star", and $\textbf{e}^i_z = (0,0,1)$. See the [Tutorial-GiRaFFEfood_NRPy](Tutorial-GiRaFFEfood_NRPy.ipynb) tutorial notebook for details on how this is used.

# Table of Contents:
$$\label{toc}$$

This notebook is organized as follows

1. [Step 1](#initializenrpy): Import core NRPy+ modules and set NRPy+ parameters
1. [Step 2](#set_aphi): Set the vector $A_{\phi}$ Spherical coordinates
1. [Step 3](#jacobian): Use the Jacobian matrix to transform the vectors to Cartesian coordinates
1. [Step 4](#vi): Calculate $v^i$
1. [Step 5](#code_validation): Code Validation against `GiRaFFEfood_NRPy.GiRaFFEfood_NRPy_Aligned_Rotator` NRPy+ Module
1. [Step 6](#latex_pdf_output): Output this notebook to $\LaTeX$-formatted PDF file

# Step 1: Import core NRPy+ modules and set NRPy+ parameters \[Back to [top](#toc)\]
$$\label{initializenrpy}$$

Here, we will import the NRPy+ core modules and set the reference metric to Cartesian, set commonly used NRPy+ parameters, and set C parameters that will be set from outside the code eventually generated from these expressions. We will also set up a parameter to determine what initial data is set up, although it won't do much yet.

```python
# Step 0: Add NRPy's directory to the path
# https://stackoverflow.com/questions/16780014/import-file-from-parent-directory
import os,sys
nrpy_dir_path = os.path.join("..")
if nrpy_dir_path not in sys.path:
    sys.path.append(nrpy_dir_path)

# Step 0.a: Import the NRPy+ core modules and set the reference metric to Cartesian
import sympy as sp               # SymPy: The Python computer algebra package upon which NRPy+ depends
import NRPy_param_funcs as par   # NRPy+: Parameter interface
import indexedexp as ixp         # NRPy+: Symbolic indexed expression (e.g., tensors, vectors, etc.) support

import reference_metric as rfm
par.set_parval_from_str("reference_metric::CoordSystem","Cartesian")
rfm.reference_metric()

# Step 1a: Set commonly used parameters.
thismodule = "GiRaFFEfood_NRPy_Aligned_Rotator"

# The angular velocity of the "neutron star"
Omega_aligned_rotator = par.Cparameters("REAL",thismodule,"Omega_aligned_rotator",1e3)
B_p_aligned_rotator,R_NS_aligned_rotator = par.Cparameters("REAL",thismodule,
                                                           # B_p_aligned_rotator = the intensity of the magnetic field and
                                                           # R_NS_aligned_rotator= "Neutron star" radius
                                                           ["B_p_aligned_rotator","R_NS_aligned_rotator"],
                                                           [1e-5, 1.0])
```

# Step 2: Set the vector $A_{\phi}$ in Spherical coordinates \[Back to [top](#toc)\]
$$\label{set_aphi}$$

We will first build the fundamental vector $A_i$ in spherical coordinates (see [Table 3](https://arxiv.org/pdf/1704.00599.pdf)). Note that we use [reference_metric.py](../edit/reference_metric.py) to set $r$ and $\varpi$ in terms of Cartesian coordinates; this will save us a step later when we convert to Cartesian coordinates. So, we set
\begin{align}
A_{\phi} &= \frac{\mu \varpi^2}{r^3}, \\
\end{align}
with $\mu = B_p R^3_{\rm NS} / 2$, $R_{\rm NS}$ is the neutron star radius, and $\varpi = \sqrt{x^2+y^2}$

```python
r     = rfm.xxSph[0]
varpi = sp.sqrt(rfm.xxCart[0]**2 + rfm.xxCart[1]**2)

mu = B_p_aligned_rotator * R_NS_aligned_rotator**3 / 2

ASphD = ixp.zerorank1()

ASphD[2] = mu * varpi**2 / (r**3) # The other components were already declared to be 0.
```

# Step 3: Use the Jacobian matrix to transform the vector to Cartesian coordinates \[Back to [top](#toc)\]
$$\label{jacobian}$$

Now, we will use the coordinate transformation definitions provided by [reference_metric.py](../edit/reference_metric.py) to build the Jacobian
$$
\frac{\partial x_{\rm Sph}^j}{\partial x_{\rm Cart}^i},
$$
where $x_{\rm Sph}^j \in \{r,\theta,\phi\}$ and $x_{\rm Cart}^i \in \{x,y,z\}$. We would normally compute its inverse, but since none of the quantities we need to transform have upper indices, it is not necessary. Then, since $A_i$ and has one lower index, it will need to be multiplied by the Jacobian:

$$
A_i^{\rm Cart} = A_j^{\rm Sph} \frac{\partial x_{\rm Sph}^j}{\partial x_{\rm Cart}^i},
$$

```python
# Step 3: Use the Jacobian matrix to transform the vectors to Cartesian coordinates.
drrefmetric__dx_0UDmatrix = sp.Matrix([[sp.diff(rfm.xxSph[0],rfm.xx[0]), sp.diff(rfm.xxSph[0],rfm.xx[1]), sp.diff(rfm.xxSph[0],rfm.xx[2])],
                                       [sp.diff(rfm.xxSph[1],rfm.xx[0]), sp.diff(rfm.xxSph[1],rfm.xx[1]), sp.diff(rfm.xxSph[1],rfm.xx[2])],
                                       [sp.diff(rfm.xxSph[2],rfm.xx[0]), sp.diff(rfm.xxSph[2],rfm.xx[1]), sp.diff(rfm.xxSph[2],rfm.xx[2])]])
#dx__drrefmetric_0UDmatrix = drrefmetric__dx_0UDmatrix.inv() # We don't actually need this in this case.

AD = ixp.zerorank1(DIM=3)
for i in range(3):
    for j in range(3):
        AD[i] = drrefmetric__dx_0UDmatrix[(j,i)]*ASphD[j]
```

# Step 4: Calculate $v^i$ \[Back to [top](#toc)\]
$$\label{vi}$$

Here, we will calculate the drift velocity $v^i = \Omega \textbf{e}_z \times \textbf{r} = [ijk] \Omega \textbf{e}^j_z x^k$, where $[ijk]$ is the Levi-Civita permutation symbol, $\Omega$ is the angular velocity of the "neutron star", and $\textbf{e}^i_z = (0,0,1)$. Conveniently, in flat space, the drift velocity reduces to the Valencia velocity because $\alpha = 1$ and $\beta^i = 0$.

However, we need to set the velocity to zero outside the neutron star; this is equivalent to only setting the non-zero velocity inside the star, $r \leq R_{\rm NS}$

```python
# Step 4: Calculate v^i
LeviCivitaSymbolDDD = ixp.LeviCivitaSymbol_dim3_rank3()

import Min_Max_and_Piecewise_Expressions as noif

unit_zU = ixp.zerorank1()
unit_zU[2] = sp.sympify(1)

global ValenciavU
ValenciavU = ixp.zerorank1()
for i in range(3):
    for j in range(3):
        for k in range(3):
            ValenciavU[i] += noif.coord_leq_bound(r,R_NS_aligned_rotator)*LeviCivitaSymbolDDD[i][j][k] * Omega_aligned_rotator * unit_zU[j] * rfm.xx[k]
```

# Step 5: Code Validation against `GiRaFFEfood_NRPy.GiRaFFEfood_NRPy_Aligned_Rotator` NRPy+ Module  \[Back to [top](#toc)\]
$$\label{code_validation}$$

Here, as a code validation check, we verify agreement in the SymPy expressions for the $\texttt{GiRaFFE}$ Aligned Rotator initial data equations we intend to use between

1. this tutorial and
2. the NRPy+ [GiRaFFEfood_NRPy.GiRaFFEfood_NRPy_Aligned_Rotator](../edit/GiRaFFEfood_NRPy/GiRaFFEfood_NRPy_Aligned_Rotator.py) module.

```python
import GiRaFFEfood_NRPy.GiRaFFEfood_NRPy_Aligned_Rotator as gfho
gfho.GiRaFFEfood_NRPy_Aligned_Rotator()

def consistency_check(quantity1,quantity2,string):
    if quantity1-quantity2==0:
        print(string+" is in agreement!")
    else:
        print(string+" does not agree!")
        sys.exit(1)

print("Consistency check between GiRaFFEfood_NRPy tutorial and NRPy+ module:")

for i in range(3):
    consistency_check(ValenciavU[i],gfho.ValenciavU[i],"ValenciavU"+str(i))
    consistency_check(AD[i],gfho.AD[i],"AD"+str(i))
```

    Consistency check between GiRaFFEfood_NRPy tutorial and NRPy+ module:
    ValenciavU0 is in agreement!
    AD0 is in agreement!
    ValenciavU1 is in agreement!
    AD1 is in agreement!
    ValenciavU2 is in agreement!
    AD2 is in agreement!

# Step 6: Output this notebook to $\LaTeX$-formatted PDF file \[Back to [top](#toc)\]
$$\label{latex_pdf_output}$$

The following code cell converts this Jupyter notebook into a proper, clickable $\LaTeX$-formatted PDF file. After the cell is successfully run, the generated PDF may be found in the root NRPy+ tutorial directory, with filename
[Tutorial-GiRaFFEfood_NRPy_Aligned_Rotator.pdf](Tutorial-GiRaFFEfood_NRPy_Aligned_Rotator.pdf) (Note that clicking on this link may not work; you may need to open the PDF file through another means.)

```python
import cmdline_helper as cmd    # NRPy+: Multi-platform Python command-line interface
cmd.output_Jupyter_notebook_to_LaTeXed_PDF("Tutorial-GiRaFFEfood_NRPy_Aligned_Rotator")
```

    Notebook output to PDF is only supported on Linux systems, with pdflatex installed.


# Matplotlib Random Walk project

### First we import the necessary libraries

```python
import matplotlib.pyplot as plt
from random import choice
%matplotlib inline
```

### Then we change the default display settings for our plots

```python
plt.rc('figure',figsize=(12,6))
```

### Now we create x and y values

```python
values = list(range(0,55,5))
values
```

    [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50]

### And we plot them

```python
plt.plot(values)
plt.show()
```

### We can label the x and y axis and add a title

```python
x_axis = [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]
```

```python
plt.plot(x_axis,values)
plt.xlim(0,1.0)
plt.ylim(0,50)
plt.title('Our Plot')
plt.xlabel('This is the horizantal axis')
plt.ylabel('Vertical axis')
plt.show()
```

### We can also change the style of the plot

```python
plt.plot(x_axis,values,'ko--')
plt.xlim(0,1.0)
plt.ylim(0,50)
plt.title('Our Plot')
plt.xlabel('This is the horizantal axis')
plt.ylabel('Vertical axis')
plt.show()
```

### Now let's create a Random Walk

```python
step_option = [1,-1]
step_choice = choice(step_option)
# step_choice
```

```python
walk = []
walk.append(step_choice)
for step in range(1,1000):
    next_step = walk[step-1] + choice(step_option)
    walk.append(next_step)
print(walk)
```

    [-1, -2, -3, -2, -3, -2, -1, 0, 1, 0, -1, 0, -1, -2, -3, -4, -5, -4, -3, -2, -3, -2, -1, -2, -1, 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 3, 4, 3, 4, 3, 2, 1, 0, 1, 2, 3, 2, 3, 2, 1, 2, 1, 0, -1, 0, 1, 0, -1, 0, -1, -2, -1, -2, -3, -2, -1, 0, 1, 0, -1, 0, -1, 0, -1, -2, -3, -4, -5, -6, -5, -4, -5, -4, -5, -4, -3, -4, -5, -6, -5, -4, -5, -6, -5, -6, -5, -6, -5, -4, -3, -2, -3, -4, -5, -4, -5, -6, -7, -6, -5, -6, -7, -8, -9, -8, -7, -6, -7, -8, -9, -8, -9, -10, -11, -10, -11, -12, -13, -14, -15, -14, -15, -14, -13, -14, -15, -16, -15, -14, -15, -14, -13, -14, -15, -16, -17, -16, -17, -16, -17, -16, -15, -16, -15, -14, -13, -12, -11, -10, -11, -10, -9, -10, -11, -12, -11, -10, -9, -8, -9, -10, -11, -12, -13, -14, -13, -12, -11, -10, -11, -12, -13, -12, -13, -14, -15, -16, -15, -14, -13, -14, -13, -14, -13, -12, -11, -12, -11, -10, -9, -10, -9, -10, -11, -10, -9, -8, -9, -8, -9, -8, -9, -10, -9, -8, -9, -10, -9, -8, -7, -8, -9, -10, -9, -8, -9, -8, -9, -10, -9, -10, -11, -10, -11, -12, -13, -14, -13, -14, -13, -14, -15, -16, -17, -18, -17, -16, -15, -14, -13, -14, -15, -14, -13, -14, -13, -14, -15, -14, -15, -14, -15, -16, -15, -14, -15, -16, -15, -14, -13, -14, -13, -12, -11, -10, -9, -10, -9, -8, -7, -6, -7, -8, -9, -10, -9, -8, -7, -6, -5, -6, -5, -4, -3, -2, -3, -4, -3, -2, -1, -2, -3, -4, -5, -6, -5, -6, -5, -6, -5, -4, -5, -4, -3, -4, -5, -6, -7, -6, -5, -4, -5, -6, -5, -4, -3, -2, -3, -4, -3, -2, -3, -4, -3, -4, -5, -6, -7, -6, -7, -6, -5, -4, -3, -4, -5, -6, -5, -6, -5, -6, -7, -8, -7, -6, -5, -6, -5, -4, -3, -2, -3, -2, -1, -2, -1, -2, -3, -4, -5, -4, -5, -4, -3, -2, -1, -2, -1, -2, -1, -2, -1, 0, 1, 2, 1, 2, 3, 4, 3, 2, 1, 2, 3, 4, 3, 2, 1, 0, -1, 0, 1, 2, 1, 0, 1, 2, 3, 2, 3, 2, 1, 0, -1, 0, -1, 0, 1, 2, 3, 2, 1, 2, 1, 0, -1, 0, -1, 0, 1, 0, -1, 0, -1, 0, 1, 2, 1, 2, 3, 2, 1, 2, 1, 0, 1, 0, 1, 2, 1, 0, -1, 0, -1, 0, 1, 2, 3, 2, 1, 2, 3, 2, 1, 0, -1, 0, 1, 0, 1, 2, 1, 2, 3, 4, 5, 6, 5, 4, 3, 2, 1, 0, 1, 0, 1, 0, -1, -2, -3, -4, -3, -2, -3, -4, -3, -4, -5, -4, -5, -6, -7, -8, -7, -6, -5, -6, -5, -6, -7, -6, -5, -6, -5, -6, -7, -8, -7, -8, -9, -8, -9, -8, -9, -10, -11, -12, -13, -14, -15, -14, -13, -12, -11, -10, -9, -8, -7, -8, -7, -8, -9, -10, -9, -10, -9, -10, -9, -8, -7, -6, -5, -4, -5, -6, -5, -4, -3, -4, -3, -2, -1, 0, -1, -2, -1, -2, -3, -2, -1, -2, -1, 0, 1, 0, -1, -2, -3, -2, -3, -2, -3, -4, -3, -4, -5, -4, -5, -6, -7, -8, -7, -8, -7, -6, -7, -8, -9, -8, -7, -8, -7, -8, -7, -6, -7, -6, -7, -8, -7, -8, -7, -6, -7, -6, -7, -6, -5, -4, -3, -4, -5, -6, -7, -6, -7, -6, -7, -8, -7, -8, -7, -6, -5, -6, -5, -4, -5, -6, -5, -4, -3, -4, -5, -6, -5, -4, -3, -4, -3, -2, -3, -2, -1, 0, -1, -2, -3, -4, -3, -4, -5, -6, -5, -6, -7, -6, -7, -8, -7, -6, -5, -6, -7, -6, -7, -8, -9, -8, -9, -8, -9, -8, -9, -10, -9, -10, -11, -10, -9, -10, -9, -8, -7, -6, -5, -4, -3, -2, -3, -4, -3, -2, -1, 0, -1, 0, -1, -2, -1, 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 5, 4, 5, 6, 7, 8, 9, 10, 11, 10, 9, 8, 9, 10, 11, 12, 11, 12, 11, 12, 13, 14, 13, 14, 13, 14, 15, 16, 17, 18, 17, 16, 15, 16, 15, 16, 17, 18, 17, 16, 15, 14, 13, 12, 11, 12, 11, 12, 11, 10, 9, 8, 9, 8, 7, 6, 7, 6, 7, 6, 5, 4, 5, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 13, 14, 15, 16, 17, 16, 15, 14, 15, 16, 17, 16, 17, 18, 19, 18, 17, 16, 15, 14, 13, 14, 13, 12, 13, 12, 11, 12, 11, 10, 11, 12, 11, 12, 13, 12, 11, 10, 9, 8, 7, 8, 9, 8, 9, 8, 7, 8, 7, 8, 7, 8, 9, 8, 7, 6, 7, 8, 9, 10, 11, 10, 11, 10, 11, 10, 9, 8, 7, 6, 5, 6, 7, 8, 7, 8, 9, 8, 7, 6, 5, 4, 3, 4, 5, 4, 5, 4, 5, 4, 3, 4, 5, 6, 7, 6, 5, 6, 7, 8, 9, 10, 9, 10, 11, 12, 11, 10, 9, 8, 7, 8, 7, 8, 7, 6, 5, 4, 3, 4, 5, 4, 5, 4, 5, 6, 5, 4, 3, 4, 3, 2, 1, 0, -1, -2, -1, 0, 1, 0, -1, 0, 1, 0, -1, 0, -1, 0, 1, 2, 3, 2, 1, 2, 3, 2, 3, 4, 5, 4, 3, 2, 1, 0, -1, 0, -1, 0, -1, 0, 1, 2, 1, 0, -1, -2, -3, -2, -1, 0, 1, 2, 3, 2, 1, 2, 1, 0, 1, 2, 3, 2]

```python
plt.plot(walk)
plt.show()
```

### Using functions would be better

```python
def rand_walk(step_num):
    walk = []
    step_choice = choice([1,-1])
    walk.append(step_choice)
    for i in range(1,step_num):
        step_choice_2 = choice([1,-1])
        next_step = walk[i-1] + step_choice_2
        walk.append(next_step)
    return walk
```

```python
random_w = rand_walk(100)
print(random_w)
```

    [-1, 0, 1, 2, 3, 4, 5, 4, 3, 4, 5, 6, 5, 6, 7, 6, 5, 4, 3, 2, 1, 0, 1, 0, -1, 0, 1, 2, 3, 2, 1, 2, 1, 2, 1, 2, 1, 0, -1, -2, -1, 0, 1, 2, 1, 0, -1, -2, -1, -2, -1, 0, -1, 0, 1, 0, 1, 2, 1, 0, -1, 0, 1, 2, 3, 4, 5, 4, 3, 2, 3, 2, 1, 0, -1, -2, -3, -4, -3, -2, -3, -2, -3, -4, -5, -4, -5, -6, -5, -6, -5, -4, -3, -4, -3, -4, -5, -6, -5, -4]

```python
def plot_walk(walk):

    plt.plot(walk)
    plt.xlabel('Number of Steps')
    plt.ylabel('Distance from Origin')
    plt.title('Our Random Walk')
    plt.show()
```

```python
plot_walk(rand_walk(1000))

```

### We can create a function to plot more than 1 random walk

```python
def plot_walks(num,walk_steps):

    lab_list = list(range(1,num+1))
    for i in range(0,num):

        x = list(range(1,walk_steps+1))

        plt.plot(x,rand_walk(walk_steps),label = 'Plot Number' + ' ' + str(lab_list[i]))
        plt.xlabel('Number of Steps')
        plt.ylabel('Distance from Origin')
        plt.title('Our Random Walk')
        plt.legend(loc='lower left')
    plt.show()
```

```python
plot_walks(5,100000)
```

# Sierpinski Triangle

First Transformation
\begin{equation}
x_{n+1} = 0.5x_n
\end{equation}
\begin{equation}
y_{n+1} = 0.5y_n
\end{equation}
Second Transformation
\begin{equation}
x_{n+1} = 0.5x_n + 0.5
\end{equation}
\begin{equation}
y_{n+1} = 0.5y_n + 0.5
\end{equation}
Third Transformation
\begin{equation}
x_{n+1} = 0.5x_n + 1
\end{equation}
\begin{equation}
y_{n+1} = 0.5y_n
\end{equation}

```python
def trans_1(p):
    x = p[0]
    y = p[1]
    x1 = 0.5 * x
    y1 = 0.5 * y
    return x1,y1

def trans_2(p):
    x = p[0]
    y = p[1]
    x1 = 0.5 * x + 0.5
    y1 = 0.5 * y + 0.5
    return x1,y1

def trans_3(p):
    x = p[0]
    y = p[1]
    x1 = 0.5 * x + 1
    y1 = 0.5 * y
    return x1,y1

transformations = [trans_1,trans_2,trans_3]
a1 = [0]
b1 = [0]
a,b = 0,0

for i in range(200000):
    trans = choice(transformations)
    a,b = trans((a,b))
    a1.append(a)
    b1.append(b)
```

```python
plt.rc('figure',figsize=(16,16))
```

```python
plt.plot(a1,b1,'o')
plt.savefig('my_figure.png')
```

```python

```


# 矩阵和张量
$$\LaTeX$$

```python
import numpy as np
import scipy
import sympy as sym
import matplotlib
import sklearn
sym.init_printing()

print("NumPy version:", np.__version__)
print("SciPy version:", scipy.__version__)
print("SymPy version:", sym.__version__)
print("Matplotlib version:", matplotlib.__version__)
print("SkLearn version:", sklearn.__version__)
```

    NumPy version: 1.13.1
    SciPy version: 0.19.1
    SymPy version: 1.1.1
    Matplotlib version: 2.0.2
    SkLearn version: 0.19.0

## 符号说明

以下是要使用符号:

符号$A \in R^{m×n}$表示一个m行n列的矩阵，并且矩阵A中的所有元素都是实数, 可以写作$A_{mn}$。
符号$x \in R^n$表示一个含有n个元素的向量。通常，我们把n维向量看成是一个n行1列矩阵，即列向量。如果我们想表示一个行向量（1行n列矩阵），我们通常写作$x^T$ (xT表示x的转置，后面会解释它的定义)。
一个向量x的第i个元素表示为{x_i}：

我们用$a_{ij}$ (或$A_{ij}$) 表示第i行第j列的元素：

## 矩阵基础运算

```python
x_11,x_12,x_13 = sym.var('x_11 x_12 x_13')
x_21,x_22,x_23 = sym.var('x_21 x_22 x_23')
x_31,x_32,x_33 = sym.var('x_31 x_32 x_33')
M = sym.Matrix([[x_11,x_12,x_13],[x_21,x_22,x_23],[x_31,x_32,x_33]])
M
```

$$\left[\begin{matrix}x_{11} & x_{12} & x_{13}\\x_{21} & x_{22} & x_{23}\\x_{31} & x_{32} & x_{33}\end{matrix}\right]$$

```python
v_1, v_2, v_3 = sym.var('v_1, v_2, v_3')
V = sym.Matrix([[v_1],[v_2],[v_3]])
M.dot(V)
```

### 矩阵运算性质
- 结合律即(AB)C = A(BC)
- 分配率即A(B + C) = AB + AC
- **矩阵乘法没有交换律**，即AB ≠BA.

结合律

```python
An=np.array([[1,1],[2,1]])
Bn=np.array([[2,2],[3,2]])
Cn=np.array([[3,3],[4,3]])
```

```python
np.dot(np.dot(An,Bn),Cn)
```

    array([[31, 27],
           [45, 39]])

```python
np.dot(An,np.dot(Bn,Cn))
```

    array([[31, 27],
           [45, 39]])

```python
np.dot(np.dot(Cn,Bn),An)
```

    array([[39, 27],
           [45, 31]])

分配律

```python
An=np.array([[1,1],[2,1]])
Bn=np.array([[2,2],[3,2]])
Cn=np.array([[3,3],[4,3]])
```

```python
np.dot(An,(Bn+Cn))
```

    array([[12, 10],
           [17, 15]])

```python
np.dot(An,Bn)+np.dot(An,Cn)
```

    array([[12, 10],
           [17, 15]])

```python
np.dot((Bn+Cn),An)
```

    array([[15, 10],
           [17, 12]])

## 公式推导工具
约定求和，用于张量运算

$$
A=a_{ij}e^i e^j\\
B=B_{km}e^k e^m\\
e^i\cdot e^j=\delta_{ij}\\
a_{ij} \delta_{ij}=a_{ii}
$$

$$
A\cdot B\\
\sum_k {a_{ik}b_{kj}} \triangleq a_{ik}b_{kj}
$$

$$
A\cdot B \cdot C=a_{ij} b_{jk} c_{km} e^i e^m
$$

## 单位矩阵对角矩阵

$$
I\cdot A = A\cdot I\\
I=\delta_{ij}
$$

```python
E=sym.eye(3)
E
```

$$\left[\begin{matrix}1 & 0 & 0\\0 & 1 & 0\\0 & 0 & 1\end{matrix}\right]$$

```python
M*E
```

$$\left[\begin{matrix}x_{11} & x_{12} & x_{13}\\x_{21} & x_{22} & x_{23}\\x_{31} & x_{32} & x_{33}\end{matrix}\right]$$

```python
E[1,2]=1
E[1,0]=0
E*M
```

$$\left[\begin{matrix}x_{11} & x_{12} & x_{13}\\x_{21} + x_{31} & x_{22} + x_{32} & x_{23} + x_{33}\\x_{31} & x_{32} & x_{33}\end{matrix}\right]$$

## 矩阵转置

$$
A=a_{ij}\\
A^T=a_{ji}
$$

```python
An=np.array([[1,1],[2,1]])
Bn=np.array([[2,2],[3,2]])
Cn=np.array([[3,3],[4,3]])
An
```

    array([[1, 1],
           [2, 1]])

```python
np.transpose(An)
```

    array([[1, 2],
           [1, 1]])

```python
np.dot(An, Bn)
```

    array([[5, 4],
           [7, 6]])

```python
np.dot(np.transpose(Bn), np.transpose(An))
```

    array([[5, 7],
           [4, 6]])

```python
np.dot(Bn, An)
```

    array([[6, 4],
           [7, 5]])

## 对称矩阵

$$
A^T=A
$$

```python
A1=(An+np.transpose(An))/2
A2=(An-np.transpose(An))/2
```

```python
A1
```

    array([[ 1. ,  1.5],
           [ 1.5,  1. ]])

```python
A2
```

    array([[ 0. , -0.5],
           [ 0.5,  0. ]])

## 矩阵迹

$$
tr(A)=a_{ii}
$$

```python
np.trace(np.dot(np.dot(An,Bn),Cn))
```

    70

```python
np.trace(np.dot(np.dot(Cn,Bn),An))
```

    70

```python
np.trace(np.dot(np.dot(Cn,An),Bn))
```

    70

## 范数

$$
||v||^1=\sum_i |x_i|\\
||v||^2=x_i x_i\\
\cdots
$$

## 矩阵行列式

```python
import numpy as np
mat = np.array([[1,1],[-1,1]])
det = np.linalg.det(mat)
det
```

    2.0

## 矩阵逆

```python
x_11,x_12,x_13 = sym.var('x_11 x_12 x_13')
x_21,x_22,x_23 = sym.var('x_21 x_22 x_23')
x_31,x_32,x_33 = sym.var('x_31 x_32 x_33')
M = sym.Matrix([[x_11,x_12,x_13],[x_21,x_22,x_23],[x_31,x_32,x_33]])
invM=M.inv()
invM
```

```python
M = sym.Matrix([[25, 15, -5], [15, 18, 0],[-5, 0, 11]])
M
```

```python
invM=M.inv()
invM
```

```python
M.dot(invM)
```

定义方程
$$2x+3y=13\\x+y=5$$
矩阵书写形式：
$$A\cdot x = b$$
求解可得：
$$x=A^{-1}\cdot b$$

```python
M=sym.Matrix([[2,3],[1,1]])
invM=M.inv()
b = sym.Matrix([[13],[5]])
invM.dot(b)
```

## 矩阵分解

## LU分解
求解方程：
$$A\cdot x=b$$
的过程中需要对其进行初等变换：
$$RA=\begin{bmatrix} 1&\cdots&0\\ \vdots&\ddots&\vdots \\r&\cdots&1 \end{bmatrix}\cdot A=U$$
$$U=\begin{bmatrix} u_{11}&\cdots&u_{nn}\\ \vdots&\ddots&\vdots \\0&\cdots&u_{1n} \end{bmatrix}$$

$$A=LU$$

```python
L, U, T = M.LUdecomposition()
```

```python
L
```

```python
U
```

```python
L*U
```

### QR分解

```python
M = sym.Matrix([[25, 15, -5], [15, 18, 0],[-5, 0, 11]])
Q, R = M.QRdecomposition()
M
```

```python
Q
```

```python
R
```

```python
Q*R
```

**两种分解可以用于求解线性方程组以及求解矩阵的特征值**

## 矩阵特征值分解
$$M=E\Lambda E^{-1}$$

```python
from numpy import linalg as LA
M = np.array([[25, 15, -5], [15, 18, 0],[-5, 0, 11]])
A, E = LA.eig(M)
print(M)
```

    [[25 15 -5]
     [15 18  0]
     [-5  0 11]]

```python
print(A)
```

    [ 37.48885386   4.4954625   12.01568365]

```python
print(E)
```

    [[-0.78373593 -0.59500318 -0.17812698]
     [-0.60321859  0.66089251  0.44648453]
     [ 0.14793693 -0.45737547  0.87688217]]

```python
ET=np.transpose(E)
np.dot(ET,E)
```

    array([[  1.00000000e+00,   1.99610479e-16,   1.72809357e-16],
           [  1.99610479e-16,   1.00000000e+00,   2.88286088e-16],
           [  1.72809357e-16,   2.88286088e-16,   1.00000000e+00]])

```python
Lambda=np.diag(A)
print(Lambda)
np.dot(E,np.dot(Lambda,ET))
```

    [[ 37.48885386   0.           0.        ]
     [  0.           4.4954625    0.        ]
     [  0.           0.          12.01568365]]

    array([[  2.50000000e+01,   1.50000000e+01,  -5.00000000e+00],
           [  1.50000000e+01,   1.80000000e+01,   2.42610070e-15],
           [ -5.00000000e+00,   2.52871842e-15,   1.10000000e+01]])

## SVD 分解

$$M = U\Lambda V$$

```python
import numpy as np
M = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
M
```

    array([[-1, -1],
           [-2, -1],
           [-3, -2],
           [ 1,  1],
           [ 2,  1],
           [ 3,  2]])

```python
U, A, V = np.linalg.svd(M, full_matrices=True)
```

```python
U
```

    array([[-0.21956688,  0.53396977, -0.48030985,  0.45219595,  0.02811389,
             0.48030985],
           [-0.35264795, -0.45713538, -0.30371038, -0.31508521,  0.61879559,
             0.30371038],
           [-0.57221483,  0.07683439,  0.75680405,  0.17257785,  0.0706181 ,
             0.24319595],
           [ 0.21956688, -0.53396977,  0.03329824,  0.79735166,  0.1693501 ,
            -0.03329824],
           [ 0.35264795,  0.45713538,  0.20989771,  0.03007049,  0.7600318 ,
            -0.20989771],
           [ 0.57221483, -0.07683439,  0.24319595, -0.17257785, -0.0706181 ,
             0.75680405]])

```python
m=np.shape(U)[0]
n=np.shape(V)[0]
mn = np.min([m, n])
Lambda = np.zeros([m, n])
print(m)
Lambda[:mn,:mn] = np.diag(A)
Lambda
```

    6

    array([[ 6.30061232,  0.        ],
           [ 0.        ,  0.54980396],
           [ 0.        ,  0.        ],
           [ 0.        ,  0.        ],
           [ 0.        ,  0.        ],
           [ 0.        ,  0.        ]])

```python
V
```

    array([[ 0.83849224,  0.54491354],
           [ 0.54491354, -0.83849224]])

```python
np.dot(U, np.dot(Lambda, V))
```

    array([[-1., -1.],
           [-2., -1.],
           [-3., -2.],
           [ 1.,  1.],
           [ 2.,  1.],
           [ 3.,  2.]])

## PCA method

```python
from sklearn.decomposition import PCA
import numpy as np
```

```python
M = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
pca = PCA(n_components=2)
pca.fit(M)
```

    PCA(copy=True, iterated_power='auto', n_components=2, random_state=None,
      svd_solver='auto', tol=0.0, whiten=False)

```python
print(pca.explained_variance_ratio_)
```

    [ 0.99244289  0.00755711]

```python
U, A, V = np.linalg.svd(M, full_matrices=True)
A2=A**2
SA2=np.sum(A2)
print(A2[0]/SA2,A2[1]/SA2)
```

    0.99244289009 0.00755710991019

```python
# PCA计算
import numpy as np
def pca(X):
    m,n = X.shape

    # 计算协方差
    C = 1.0/m*X.T.dot(X)

    # 特征值赋值给evalues，对应特征向量赋值给evectors
    evalues,evectors = np.linalg.eig(C)

    # 特征值从大到小排列，返回数组索引
    sorted_indices = np.argsort(-evalues)

    # 特征向量按特征值大小从左到右排列
    sorted_evectors = evectors[:,sorted_indices]
    sorted_evalues = evalues[sorted_indices]

    return sorted_evectors,np.diag(sorted_evalues),sorted_evalues
```

```python
pca(np.array([[1,2],[3,4],[5,6]]))
```

    (array([[-0.61962948, -0.78489445],
            [-0.78489445,  0.61962948]]), array([[ 30.24516497,   0.        ],
            [  0.        ,   0.08816836]]), array([ 30.24516497,   0.08816836]))

```python

```

math_slides/Chapter 4 - Matrix and tensor.ipynb


This notebook is part of https://github.com/AudioSceneDescriptionFormat/splines, see also https://splines.readthedocs.io/.

[back to overview](end-conditions.ipynb)

# Natural End Conditions

For the first and last segment, we assume that the inner tangent is known.
To find the outer tangent
according to *natural* end conditions,
the second derivative is set to $0$
at the beginning and end of the curve.

We are looking only at the non-uniform case here,
it's easy to get to the uniform case by setting $\Delta_i = 1$.

In case you were wondering,
natural end conditions are sometimes also called "relaxed" end conditions.

```python
import sympy as sp
sp.init_printing(order='grevlex')
```

As usual we are getting some help from [utility.py](utility.py):

```python
from utility import NamedExpression
```

```python
t = sp.symbols('t')
```

## Begin

We are starting with the first polynomial segment $\boldsymbol{p}_0(t)$,
with $t_0 \le t \le t_1$.

```python
t0, t1 = sp.symbols('t:2')
```

The coefficients ...

```python
a0, b0, c0, d0 = sp.symbols('a:dbm0')
```

... multiplied with the [monomial basis](polynomials.ipynb)
give us the uniform polynomial ...

```python
d0 * t**3 + c0 * t**2 + b0 * t + a0
```

... which we re-scale to the desired parameter range:

```python
p0 = NamedExpression('pbm0', _.subs(t, (t - t0) / (t1 - t0)))
p0
```

We need the first derivative
(a.k.a velocity, a.k.a. tangent vector):

```python
pd0 = p0.diff(t)
pd0
```

Similar to the
[notebook about non-uniform Hermite splines](hermite-non-uniform.ipynb),
we are interested in the function values and first derivatives
at the control points:

\begin{align*}
\boldsymbol{x}_0 &= \boldsymbol{p}_0(t_0)\\
\boldsymbol{x}_1 &= \boldsymbol{p}_0(t_1)\\
\boldsymbol{\dot{x}}_0 &= \boldsymbol{p}_0'(t_0)\\
\boldsymbol{\dot{x}}_1 &= \boldsymbol{p}_0'(t_1)
\end{align*}

```python
equations_begin = [
    p0.evaluated_at(t, t0).with_name('xbm0'),
    p0.evaluated_at(t, t1).with_name('xbm1'),
    pd0.evaluated_at(t, t0).with_name('xdotbm0'),
    pd0.evaluated_at(t, t1).with_name('xdotbm1'),
]
```

To get simpler equations,
we are substituting $\Delta_0 = t_1 - t_0$.
Note that this is only for display purposes,
the calculations are still done with $t_i$.

```python
delta_begin = [
    (t0, 0),
    (t1, sp.Symbol('Delta0')),
]
```

```python
for e in equations_begin:
    display(e.subs(delta_begin))
```

```python
coefficients_begin = sp.solve(equations_begin, [a0, b0, c0, d0])
```

```python
for c, e in coefficients_begin.items():
    display(NamedExpression(c, e.subs(delta_begin)))
```

The second derivative (a.k.a. acceleration) ...

```python
pdd0 = pd0.diff(t)
pdd0
```

... at the beginning of the curve ($t = t_0$) ...

```python
pdd0.evaluated_at(t, t0)
```

... is set to zero ...

```python
sp.Eq(_.expr, 0).subs(coefficients_begin)
```

... leading to an expression for the initial tangent vector:

```python
xd0 = NamedExpression.solve(_, 'xdotbm0')
xd0.subs(delta_begin)
```

## End

If a spline has $N$ vertices,
it has $N-1$ polynomial segments
and the last polynomial segment is
$\boldsymbol{p}_{N-2}(t)$, with $t_{N-2} \le t \le t_{N-1}$.
To simplify the notation a bit,
let's assume we have $N = 10$ vertices,
which makes $\boldsymbol{p}_8$ the last polynomial segment.
The following steps are very similar
to the above derivation of the start conditions.

```python
a8, b8, c8, d8 = sp.symbols('a:dbm8')
```

```python
t8, t9 = sp.symbols('t8:10')
```

```python
d8 * t**3 + c8 * t**2 + b8 * t + a8
```

```python
p8 = NamedExpression('pbm8', _.subs(t, (t - t8) / (t9 - t8)))
p8
```

```python
pd8 = p8.diff(t)
pd8
```

\begin{align*}
\boldsymbol{x}_{N-2} &= \boldsymbol{p}_{N-2}(t_{N-2})\\
\boldsymbol{x}_{N-1} &= \boldsymbol{p}_{N-2}(t_{N-1})\\
\boldsymbol{\dot{x}}_{N-2} &= \boldsymbol{p}_{N-2}'(t_{N-2})\\
\boldsymbol{\dot{x}}_{N-1} &= \boldsymbol{p}_{N-2}'(t_{N-1})
\end{align*}

```python
equations_end = [
    p8.evaluated_at(t, t8).with_name('xbm8'),
    p8.evaluated_at(t, t9).with_name('xbm9'),
    pd8.evaluated_at(t, t8).with_name('xdotbm8'),
    pd8.evaluated_at(t, t9).with_name('xdotbm9'),
]
```

We define $\Delta_8 = t_9 - t_8$:

```python
delta_end = [
    (t8, 0),
    (t9, sp.Symbol('Delta8')),
]
```

```python
for e in equations_end:
    display(e.subs(delta_end))
```

```python
coefficients_end = sp.solve(equations_end, [a8, b8, c8, d8])
```

```python
for c, e in coefficients_end.items():
    display(NamedExpression(c, e.subs(delta_end)))
```

This time,
the second derivative ...

```python
pdd8 = pd8.diff(t)
pdd8
```

... *at the end* of the last segment ($t = t_9$) ...

```python
pdd8.evaluated_at(t, t9)
```

... is set to zero ...

```python
sp.Eq(_.expr, 0).subs(coefficients_end)
```

... leading to an expression for the final tangent vector:

```python
xd9 = NamedExpression.solve(_, 'xdotbm9')
xd9.subs(delta_end)
```

Luckily, that's symmetric to the result we got above.

## Example

We are showing a one-dimensional example where 3 time/value pairs are given.
The slope for the middle value is given, the begin and end slopes are calculated using the "natural" end conditions as calculated above.

```python
values = 2, 2, 1
times = 0, 4, 5
slope = 2
```

We are using a few helper functions from [helper.py](helper.py) for plotting:

```python
from helper import plot_sympy, grid_lines
```

```python
begin = p0.subs(coefficients_begin).subs_symbols(xd0).subs({
    t0: times[0],
    t1: times[1],
    sp.Symbol('xbm0'): values[0],
    sp.Symbol('xbm1'): values[1],
    sp.Symbol('xdotbm1'): slope,
}).with_name(r'p_\text{begin}')
end = p8.subs(coefficients_end).subs_symbols(xd9).subs({
    t8: times[1],
    t9: times[2],
    sp.Symbol('xbm8'): values[1],
    sp.Symbol('xbm9'): values[2],
    sp.Symbol('xdotbm8'): slope,
}).with_name(r'p_\text{end}')
```

```python
plot_sympy(
    (begin.expr, (t, times[0], times[1])),
    (end.expr, (t, times[1], times[2])))
grid_lines(times, [1, 2])
```

```python
begin.diff(t).evaluated_at(t, times[0])
```

```python
end.diff(t).evaluated_at(t, times[-1])
```