# Building Acyclic Diagrams for the paper using daft

```python
from matplotlib import rc
import matplotlib.pyplot as plt
import sys
rc("font", family="serif", size=12)
rc("text", usetex=True)

import daft
```

## Gaia model

\begin{equation}
p(\theta_{\rm RC}, \varpi_{\rm zp}, L, \alpha | \mathcal{D})
\propto  p(\theta_{\rm RC}, \varpi_{\rm zp}, L)\ p(\hat{\varpi} | r, \varpi_{\rm zp}, \underline{\Sigma})\ \times
\prod_{i=1}^{\rm N} p(\hat{m}_i | \alpha_i, \sigma_{\hat{m}_i})\ p(\alpha_i | \theta_{\rm RC}, \varpi_{\rm zp}, L)
\end{equation}

where $\theta_{RC}$, $\varpi_{\rm zp}$ and $L$ are hyperparameters, $\alpha$ is the set of latent parameters (which includes the latent parameter $r$). Anything with a hat on it is our data $\mathcal{D}$. $\underline{\Sigma}$ is a covariance matrix. N is the number of data points.

```python
#Instantiate the PGM
pgm = daft.PGM(shape=[3.4, 3.8],
               origin=[0.0, 0.0],
               backset=.05,
              label_params={'fontsize':12})

#Hiearchical parameters
pgm.add_node(daft.Node("mu", r"$\mu_{RC}$", .27, 1.35))
pgm.add_node(daft.Node("sigma", r"$\sigma_{RC}$", .27, 2.))
pgm.add_node(daft.Node("Q", r"$Q$", .27, 2.65))
pgm.add_node(daft.Node("sigo", r"$\sigma_o$",.27,3.3))

pgm.add_node(daft.Node("L", r"$L$",2.075, 3.3))
pgm.add_node(daft.Node("oozp", r"$\varpi_{zp}$", 2.95, 2.4))

#Add a plate
pgm.add_plate(daft.Plate([.6, .2, 1.9, 2.7], label=r"$i = 1, \cdots, \rm N$",shift=-0.1))

#Latent varable.
pgm.add_node(daft.Node("Mi", r"$M_i$", 1.15, 2.4))
pgm.add_node(daft.Node("Ai", r"$A_i$", 2.075, 1.6))
pgm.add_node(daft.Node("ri", r"$r_i$", 2.075, 2.4))

#Data
pgm.add_node(daft.Node("mi", r"$\hat{m}_i$", 1.15, 1.6, observed=True))
pgm.add_node(daft.Node("sigmi", r"$\sigma_{\hat{m}_i}$", 1.15, .9, fixed=True, offset=(0,-24.)))
pgm.add_node(daft.Node("ooi", r"$\hat{\varpi}$",   2.95, 1.6, observed=True))
pgm.add_node(daft.Node("sigooi",r"$\underline{\Sigma}$", 2.95, .9, fixed=True, offset=(0,-24.)))
# pgm.add_node(daft.Node("rlebv", r"$R_{\lambda}E_{B-V}$", 2.075, .9, fixed=True, offset=(0,-22.)))
pgm.add_node(daft.Node("rlebv", r"$\hat{A}_i$", 2.075, .9, fixed=True, offset=(0,-24.)))

#Add in edges
pgm.add_edge("mu", "Mi")
pgm.add_edge("sigma", "Mi")
pgm.add_edge("Q", "Mi")
pgm.add_edge("sigo", "Mi")
pgm.add_edge("oozp", "ooi")
pgm.add_edge("L", "ri")
pgm.add_edge("Ai", "mi")
pgm.add_edge("Mi", "mi")
pgm.add_edge("ri", "mi")
pgm.add_edge("ri", "ooi")
pgm.add_edge("rlebv", "Ai")
pgm.add_edge("sigmi", "mi")
pgm.add_edge("sigooi", "ooi")
```

    <daft.Edge at 0x7f3ad1522dd0>

```python
pgm.render()
pgm.figure.savefig("Figures/gaiastan_alt.pdf")
pgm.figure.savefig("Figures/gaiastan_alt.png", dpi=300)
plt.show()
```

# Astero Model

\begin{equation}
p(\theta_{\rm RC} | \mathcal{D}) \propto  p(\theta_{\rm RC}) \prod_{i=1}^{\rm N} p(\mathcal{D}_i | \theta_{\rm RC})p(M_i | \theta_{\rm RC})
\end{equation}
where $\theta_{RC}$ are hyperparameters, and $M_i$ is our latent parameters. Anything with a hat on it is our data $\mathcal{D}$. N is the number of data points.

```python
#Instantiate the PGM
pgm = daft.PGM(shape=[3.,3.],
               origin=[0.0, 0.0],
               backset=0.05,
              label_params={'fontsize':13})

#Hiearchical parameters
pgm.add_node(daft.Node("mu", r"$\mu_{RC}$", .5, .6))
pgm.add_node(daft.Node("sigma", r"$\sigma_{RC}$", .5, 1.2))
pgm.add_node(daft.Node("Q", r"$Q$", .5, 1.8))
pgm.add_node(daft.Node("sigo", r"$\sigma_o$", .5, 2.4))

#Add a plate
pgm.add_plate(daft.Plate([1., .9, 1.9, 1.6], label=r"$i = 1, \cdots, \rm N$",shift=-0.1, position='bottom right'))

#Latent varable.
pgm.add_node(daft.Node("Mi", r"$M_i$", 1.5, 1.5))

#Data
pgm.add_node(daft.Node("Mhati", r"$\hat{M}_i$", 2.5, 1.5, observed=True))
pgm.add_node(daft.Node("sigMhati", r"$\sigma_{\hat{M}_i}$", 2.5, 2., fixed=True, offset=(0,2.)))

# #Add in edges
pgm.add_edge("mu", "Mi")
pgm.add_edge("sigma", "Mi")
pgm.add_edge("Q", "Mi")
pgm.add_edge("sigo", "Mi")
pgm.add_edge("Mi", "Mhati")
pgm.add_edge("sigMhati", "Mhati")
```

    <daft.Edge at 0x7f3ad15227d0>

```python
pgm.render()
plt.tight_layout()
pgm.figure.savefig("Figures/asterostan.pdf")
pgm.figure.savefig("Figures/asterostan.png", dpi=150)
plt.show()
```

# Old Gaia model

```python
#Instantiate the PGM
pgm = daft.PGM(shape=[3.4, 3.8],
               origin=[0.0, 0.0],
               backset=.05,
              label_params={'fontsize':12})

#Hiearchical parameters
pgm.add_node(daft.Node("mu", r"$\mu_{RC}$", .95, 3.3))
pgm.add_node(daft.Node("sigma", r"$\sigma_{RC}$", 1.6, 3.3))
pgm.add_node(daft.Node("Q", r"$Q$", 2.25, 3.3))
pgm.add_node(daft.Node("sigo", r"$\sigma_o$",2.9,3.3))

pgm.add_node(daft.Node("L", r"$L$",.27, .8))
pgm.add_node(daft.Node("oozp", r"$\varpi_{zp}$", .27, 1.7))

#Add a plate
pgm.add_plate(daft.Plate([.6, .2, 2.7, 2.7], label=r"$i = 1, \cdots, \rm N$",shift=-0.1))

#Latent varable.
pgm.add_node(daft.Node("Mi", r"$M_i$", 2.1, 2.5))
pgm.add_node(daft.Node("Ai", r"$A_i$", 2.10, .8))
pgm.add_node(daft.Node("ri", r"$r_i$", 1.15, .8))

#Data
pgm.add_node(daft.Node("mi", r"$\hat{m}_i$", 2.1, 1.7, observed=True))
pgm.add_node(daft.Node("sigmi", r"$\sigma_{\hat{m}_i}$", 2.85, 1.7, fixed=True, offset=(0,5.)))
pgm.add_node(daft.Node("ooi", r"$\hat{\varpi}_i$",  1.15, 1.7, observed=True))
# pgm.add_node(daft.Node("sigooi",r"$\sigma_{\hat{\varpi}_i}$", 1.15, 2.2, fixed=True, offset=(0,5.)))
pgm.add_node(daft.Node("sigooi",r"$\underline{\Sigma}$", 1.15, 2.2, fixed=True, offset=(0,5.)))
pgm.add_node(daft.Node("rlebv", r"$R_{\lambda}E_{B-V}$", 2.85, .8, fixed=True, offset=(0,5.)))

# #Add in edges
pgm.add_edge("mu", "Mi")
pgm.add_edge("sigma", "Mi")
pgm.add_edge("Q", "Mi")
pgm.add_edge("sigo", "Mi")
pgm.add_edge("oozp", "ooi")
pgm.add_edge("L", "ri")
pgm.add_edge("Ai", "mi")
pgm.add_edge("Mi", "mi")
pgm.add_edge("ri", "mi")
pgm.add_edge("ri", "ooi")
pgm.add_edge("rlebv", "Ai")
pgm.add_edge("sigmi", "mi")
pgm.add_edge("sigooi", "ooi")
```

    <daft.Edge at 0x7f3accc45290>

```python
pgm.render()
pgm.figure.savefig("Figures/gaiastan.pdf")
pgm.figure.savefig("Figures/gaiastan.png", dpi=150)
plt.show()
```


# Tutorial 2: Convolutional Neural Networks
**Week 2, Day 1: Deep Learning**

**By Neuromatch Academy**

**Content creators**: Jorge A. Menendez, Carsen Stringer

**Content reviewers**: Roozbeh Farhoodi,  Madineh Sarvestani, Kshitij Dwivedi, Spiros Chavlis, Ella Batty, Michael Waskom

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives

*Estimated timing of tutorial: 40 minutes*

In this short tutorial, we'll go through an introduction to 2D convolutions and apply a convolutional network to an image to prepare for creating normative models in tutorial 3.

In this tutorial, we will
* Understand the basics of 2D convolution
* Build a convolutional layer using PyTorch
* Visualize and analyze its outputs

```python
# @title Tutorial slides

# @markdown These are the slides for all videos in this tutorial.
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/s59jy/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

---
# Setup

```python
# Imports

import os
import numpy as np
import torch
from torch import nn
from torch import optim
from matplotlib import pyplot as plt
import matplotlib as mpl
```

```python
#@title Figure settings
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
# @title Data retrieval and loading

import hashlib
import requests

fname = "W3D4_stringer_oribinned6_split.npz"
url = "https://osf.io/p3aeb/download"
expected_md5 = "b3f7245c6221234a676b71a1f43c3bb5"

if not os.path.isfile(fname):
  try:
    r = requests.get(url)
  except requests.ConnectionError:
    print("!!! Failed to download data !!!")
  else:
    if r.status_code != requests.codes.ok:
      print("!!! Failed to download data !!!")
    elif hashlib.md5(r.content).hexdigest() != expected_md5:
      print("!!! Data download appears corrupted !!!")
    else:
      with open(fname, "wb") as fid:
        fid.write(r.content)
```

```python
# @title Plotting Functions

def show_stimulus(img, ax=None):
  """Visualize a stimulus"""
  if ax is None:
    ax = plt.gca()
  ax.imshow(img+0.5, cmap=mpl.cm.binary)
  ax.set_xticks([])
  ax.set_yticks([])
  ax.spines['left'].set_visible(False)
  ax.spines['bottom'].set_visible(False)

def plot_weights(weights, channels=[0], colorbar=True):
  """ plot convolutional channel weights
  Args:
        weights: weights of convolutional filters (conv_channels x K x K)
        channels: which conv channels to plot
  """
  wmax = torch.abs(weights).max()
  fig, axs = plt.subplots(1,len(channels), figsize=(12,2.5))
  for i, channel in enumerate(channels):
    im = axs[i].imshow(weights[channel,0], vmin=-wmax, vmax=wmax, cmap='bwr')
    axs[i].set_title('channel %d'%channel)

  if colorbar:
    ax = fig.add_axes([1, 0.1, 0.05, 0.8])
    plt.colorbar(im, ax=ax)
    ax.axis('off')

def plot_example_activations(stimuli, act, channels=[0]):
  """ plot activations act and corresponding stimulus
  Args:
        stimuli: stimulus input to convolutional layer (n x h x w) or (h x w)
        act: activations of convolutional layer (n_bins x conv_channels x n_bins)
        channels: which conv channels to plot
  """
  if stimuli.ndim>2:
    n_stimuli = stimuli.shape[0]
  else:
    stimuli = stimuli.unsqueeze(0)
    n_stimuli = 1

  fig, axs = plt.subplots(n_stimuli,1+len(channels),figsize=(12,12))

  # plot stimulus
  for i in range(n_stimuli):
    show_stimulus(stimuli[i].squeeze(), ax=axs[i, 0])
    axs[i, 0].set_title('stimulus')

    # plot example activations
    for k, (channel, ax) in enumerate(zip(channels, axs[i][1:])):
      img=ax.imshow(act[i,channel], vmin=-3, vmax=3, cmap='bwr')
      ax.set_xlabel('x-pos')
      ax.set_ylabel('y-pos')
      ax.set_title('channel %d'%channel)
  ax = fig.add_axes([1.05,0.8,0.01,0.1])
  plt.colorbar(img, cax=ax)
  ax.set_title('activation\n strength')
```

```python
# @title Helper Functions

def load_data_split(data_name=fname):
  """Load mouse V1 data from Stringer et al. (2019)

  Data from study reported in this preprint:
  https://www.biorxiv.org/content/10.1101/679324v2.abstract

  These data comprise time-averaged responses of ~20,000 neurons
  to ~4,000 stimulus gratings of different orientations, recorded
  through Calcium imaginge. The responses have been normalized by
  spontaneous levels of activity and then z-scored over stimuli, so
  expect negative numbers. The repsonses were split into train and
  test and then each set were averaged in bins of 6 degrees.

  This function returns the relevant data (neural responses and
  stimulus orientations) in a torch.Tensor of data type torch.float32
  in order to match the default data type for nn.Parameters in
  Google Colab.

  It will hold out some of the trials when averaging to allow us to have test
  tuning curves.

  Args:
    data_name (str): filename to load

  Returns:
    resp_train (torch.Tensor): n_stimuli x n_neurons matrix of neural responses,
        each row contains the responses of each neuron to a given stimulus.
        As mentioned above, neural "response" is actually an average over
        responses to stimuli with similar angles falling within specified bins.
    resp_test (torch.Tensor): n_stimuli x n_neurons matrix of neural responses,
        each row contains the responses of each neuron to a given stimulus.
        As mentioned above, neural "response" is actually an average over
        responses to stimuli with similar angles falling within specified bins
    stimuli: (torch.Tensor): n_stimuli x 1 column vector with orientation
        of each stimulus, in degrees. This is actually the mean orientation
        of all stimuli in each bin.

  """
  with np.load(data_name) as dobj:
    data = dict(**dobj)
  resp_train = data['resp_train']
  resp_test = data['resp_test']
  stimuli = data['stimuli']

  # Return as torch.Tensor
  resp_train_tensor = torch.tensor(resp_train, dtype=torch.float32)
  resp_test_tensor = torch.tensor(resp_test, dtype=torch.float32)
  stimuli_tensor = torch.tensor(stimuli, dtype=torch.float32)

  return resp_train_tensor, resp_test_tensor, stimuli_tensor

def filters(out_channels=6, K=7):
  """ make example filters, some center-surround and gabors
  Returns:
      filters: out_channels x K x K
  """
  grid = np.linspace(-K/2, K/2, K).astype(np.float32)
  xx,yy = np.meshgrid(grid, grid, indexing='ij')

  # create center-surround filters
  sigma = 1.1
  gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*sigma**2))
  wide_gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*(sigma*2)**2))
  center_surround = gaussian - 0.5 * wide_gaussian

  # create gabor filters
  thetas = np.linspace(0, 180, out_channels-2+1)[:-1] * np.pi/180
  gabors = np.zeros((len(thetas), K, K), np.float32)
  lam = 10
  phi = np.pi/2
  gaussian = np.exp(-(xx**2 + yy**2)**0.5/(2*(sigma*0.4)**2))
  for i,theta in enumerate(thetas):
    x = xx*np.cos(theta) + yy*np.sin(theta)
    gabors[i] = gaussian * np.cos(2*np.pi*x/lam + phi)

  filters = np.concatenate((center_surround[np.newaxis,:,:],
                            -1*center_surround[np.newaxis,:,:],
                            gabors),
                           axis=0)
  filters /= np.abs(filters).max(axis=(1,2))[:,np.newaxis,np.newaxis]
  filters -= filters.mean(axis=(1,2))[:,np.newaxis,np.newaxis]
  # convert to torch
  filters = torch.from_numpy(filters)
  # add channel axis
  filters = filters.unsqueeze(1)

  return filters

def grating(angle, sf=1 / 28, res=0.1, patch=False):
  """Generate oriented grating stimulus

  Args:
    angle (float): orientation of grating (angle from vertical), in degrees
    sf (float): controls spatial frequency of the grating
    res (float): resolution of image. Smaller values will make the image
      smaller in terms of pixels. res=1.0 corresponds to 640 x 480 pixels.
    patch (boolean): set to True to make the grating a localized
      patch on the left side of the image. If False, then the
      grating occupies the full image.

  Returns:
    torch.Tensor: (res * 480) x (res * 640) pixel oriented grating image

  """

  angle = np.deg2rad(angle)  # transform to radians

  wpix, hpix = 640, 480  # width and height of image in pixels for res=1.0

  xx, yy = np.meshgrid(sf * np.arange(0, wpix * res) / res, sf * np.arange(0, hpix * res) / res)

  if patch:
    gratings = np.cos(xx * np.cos(angle + .1) + yy * np.sin(angle + .1))  # phase shift to make it better fit within patch
    gratings[gratings < 0] = 0
    gratings[gratings > 0] = 1
    xcent = gratings.shape[1] * .75
    ycent = gratings.shape[0] / 2
    xxc, yyc = np.meshgrid(np.arange(0, gratings.shape[1]), np.arange(0, gratings.shape[0]))
    icirc = ((xxc - xcent) ** 2 + (yyc - ycent) ** 2) ** 0.5 < wpix / 3 / 2 * res
    gratings[~icirc] = 0.5

  else:
    gratings = np.cos(xx * np.cos(angle) + yy * np.sin(angle))
    gratings[gratings < 0] = 0
    gratings[gratings > 0] = 1

  gratings -= 0.5

  # Return torch tensor
  return torch.tensor(gratings, dtype=torch.float32)
```

---
# Section 1: Introduction to 2D convolutions

## Section 1.1: What is a 2D convolution?

A 2D convolution is an integral of the product of a filter $f$ and an input image $I$ computed at various positions as the filter is slid across the input. The output of the convolution operation at position $(x,y)$ can be written as follows, where the filter $f$ is size $(K,K)$:
\begin{equation}
C(x,y) = \sum_{k_x=-K/2}^{K/2} \sum_{k_y=-K/2}^{K/2} f(k_x,k_y) I(x+k_x,y+k_y)
\end{equation}
This **convolutional filter** is often called a **kernel**.

Here is an illustration of a 2D convolution from this [article](https://towardsdatascience.com/light-on-math-machine-learning-intuitive-guide-to-convolution-neural-networks-e3f054dd5daa):

```python
#@title
#@markdown Execute this cell to view convolution gif

from IPython.display import Image
Image(url='https://miro.medium.com/max/700/1*5BwZUqAqFFP5f3wKYQ6wJg.gif')
```

## Section 1.2: 2D convolutions in deep learning

*Estimated timing to here from start of tutorial: 6 min*

```python
# @title Video 1: 2D Convolutions

from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1jw411d7Kg", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="zgO9rHYbDxE", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

This video covers convolutions and how to implement them in Pytorch.

 Click here for text recap of video

Recall Aude Oliva’s discussion of convolutions in the [intro](https://www.youtube.com/watch?v=IZvcy0Myb3M). Convolutional neural networks with several layers revolutionized the deep learning field, and in particular AlexNet, depicted here, was the first deep neural network to excel on the ImageNet classification task. The first layer in the network takes as input an image, runs convolutional filters on the image, rectifies the output, them downsamples the output (the pooling layer). The next layers repeat this process, and then at the end fully connected linear layers are attached which output a label for the image.

The main advantages of convolutional layers over fully connected layers are the reduction in parameters through weight-sharing, which we will get into shortly, and also the fact that the units have local receptive fields. These local receptive fields allow the network to pool over units in spatial proximity and helps the network learn translation invariant representations.

A convolution is the integral of the product of two functions, one of which is a stimulus and the other which is a filter. This integral is computed at all positions by sliding the filter weights across the stimulus. If you want to perform a convolution and get the same output as the input you need to pad the input by half the filter size on each side. This is called “same” padding. Another parameter of this convolution computation is the stride -- how often the convolution is computed along the stimulus dimension. In this case, we use a stride of 1, but we can increase the stride and in turn have fewer units.

All the units of this filter are called a single output **channel**. A convolutional layer often consists of multiple output channels each with their own filter weights. We call the number of output convolutional channels $C_{out}$.

We will implement this convolutional layer in pytorch. We will create a convolutional layer `ConvolutionalLayer` which takes as input a stimulus, which in our case are the gratings images. The convolutional layer is initialized with a few different parameters - first is the # of input channels $C_{in}$, which is 1 in our case. Next the number of convolutional channels which we’ll call $C_{out}$ which we can set to 6. Then the size of the filter $K$ which we set by default to 7. There’s also an optional `filters` input which we use to initialize the convolutional weights.  We set them as the weights of the conv layer we just created, and set the bias terms for the conv layer to zero.

We declare an `nn.conv2d` variable to be an attribute of the class `ConvolutionalLayer` called `conv`. For this convolutional layer we set the padding to half the filter size and the stride to 1 to get the same size output as the input.

What are some example filters we might use? One is a center-surround filter. It is positive in the middle and negative in the surround. Another is a gabor filter, which has a positive section next to a negative section. Look at the responses of these filters given the image. Both of these filter types are inspired by neurons recorded in the brain.

In fact, convolutional neural networks are inspired by the brain. In the retina there are a variety of cell types which we can think of as filters and each of these cells tile the entire visual space. The picture here shows the part of visual space each one of these cells responds to. In barrel cortex we have a similar situation where each whisker’s activation corresponds to a single cortical column of activity and the functions computed in each of these columns are similar.

Object recognition was essentially an unsolved problem in machine learning until the [advent](https://en.wikipedia.org/wiki/AlexNet) of techniques for effectively training *deep* convolutional neural networks. See Bonus Section 1 for more information on why we use CNNs to model the visual system.

Convolutional neural networks consist of 2D convolutional layers, RELU non-linearities, 2D pooling layers, and at the output, a fully connected layer. We will see an example network with all these components in tutorial 3.

A 2D convolutional layer has multiple output channels. Each output **channel** is the result of a 2D convolutional filter applied to the input. In the gif below, the input is in blue, the filter is in gray, and the output is in green. The number of units in the output channel depends on the *stride* you set. In the gif below, the stride is 1 because the input image is sampled at each position, a stride of 2 would mean skipping over input positions. In most applications, especially with small filter sizes, a stride of 1 is used.

(*Technical note*: if filter size *K* is odd and you set the *pad=K//2* and *stride=1* (as is shown below), you get a **channel** of units that is the same size as the input. See a more detailed explanation of strides and pads [here](https://theano-pymc.readthedocs.io/en/latest/tutorial/conv_arithmetic.html) if interested).

```python
#@title
#@markdown Execute this cell to view convolution gif

from IPython.display import Image
Image(url='https://miro.medium.com/max/790/1*1okwhewf5KCtIPaFib4XaA.gif')
```

## Section 1.3: 2D convolutions in Pytorch

*Estimated timing to here from start of tutorial: 18 min*

In Tutorial 1, fully connected linear layers were used to decode stimuli from neural activity. Convolutional layers are different from their fully connected counterparts in two ways:
  * In a fully connected layer, each unit computes a weighted sum over all the input units. In a convolutional layer, on the other hand, each unit computes a weighted sum over only a small patch of the input, referred to as the unit's **receptive field**. When the input is an image, the receptive field can be thought of as a local patch of pixels.
  * In a fully connected layer, each unit uses its own independent set of weights to compute the weighted sum. In a convolutional layer, all the units (within the same channel) **share the same weights**. This set of shared weights is called the **convolutional filter or kernel**. The result of this computation is a convolution, where each unit has computed the same weighted sum over a different part of the input. This reduces the number of parameters in the network substantially.

We will compute the difference in the number of weights for a fully connected layer versus a convolutional layer in the Think exercise below.

First, let's visualize the stimuli in the dataset from tutorial 1. During the neural recordings from [Stringer et al 2019](https://www.biorxiv.org/content/10.1101/679324v2.abstract), mice were presented oriented gratings:

```python
# @markdown Execute this cell to plot example stimuli

orientations = np.linspace(-90, 90, 5)

h = 3
n_col = len(orientations)
fig, axs = plt.subplots(1, n_col, figsize=(h * n_col, h))

h, w  = grating(0).shape  # height and width of stimulus
print('stimulus size: %i x %i' % (h, w))

for i, ori in enumerate(orientations):
  stimulus = grating(ori)
  axs[i].set_title(f'{ori: .0f}$^o$')
  show_stimulus(stimulus, axs[i])
```

Now let's implement 2D convolutional operations. We will use multiple convolutional channels and implement this operation efficiently using pytorch. A *layer* of convolutional channels can be implemented with one line of code using the PyTorch class `nn.Conv2d()`, which requires the following arguments for initialization (see full documentation [here](https://pytorch.org/docs/master/generated/torch.nn.Conv2d.html)):
  * $C^{in}$: the number of input channels
  * $C^{out}$: the number of output channels (number of different convolutional filters)
  * $K$: the size of the $C^{out}$ different convolutional filters

When you run the network, you can input a stimulus of arbitrary size $(H^{in}, W^{in})$, but it needs to be shaped as a 4D input $(N, C^{in}, H^{in}, W^{in})$, where $N$ is the number of images. In our case, $C^{in}=1$ because there is only one color channel (our images are grayscale, but often $C^{in}=3$ in image processing).

```python
class ConvolutionalLayer(nn.Module):
  """Deep network with one convolutional layer
     Attributes: conv (nn.Conv2d): convolutional layer
  """
  def __init__(self, c_in=1, c_out=6, K=7, filters=None):
    """Initialize layer

    Args:
        c_in: number of input stimulus channels
        c_out: number of output convolutional channels
        K: size of each convolutional filter
        filters: (optional) initialize the convolutional weights

    """
    super().__init__()
    self.conv = nn.Conv2d(c_in, c_out, kernel_size=K,
                          padding=K//2, stride=1)
    if filters is not None:
      self.conv.weight = nn.Parameter(filters)
      self.conv.bias = nn.Parameter(torch.zeros((c_out,), dtype=torch.float32))

  def forward(self, s):
    """Run stimulus through convolutional layer

    Args:
        s (torch.Tensor): n_stimuli x c_in x h x w tensor with stimuli

    Returns:
        (torch.Tensor): n_stimuli x c_out x h x w tensor with convolutional layer unit activations.

    """
    a = self.conv(s)  # output of convolutional layer

    return a
```

See that `ConvolutionalLayer` takes as input `filters`. We have predesigned some filters that you can use by calling the `filters` function below. These are similar to filters we think are implemented in biological circuits such as the retina and the visual cortex. Some of them are **center-surround** filters and some of them are **gabor** filters. Check out this [website](http://www.cns.nyu.edu/~david/courses/perception/lecturenotes/ganglion/ganglion.html) for more details on center-surround filters, and this [website](https://en.wikipedia.org/wiki/Gabor_filter) for more details on gabor filters, if you're interested.

```python
# @markdown Execute this cell to create and visualize filters

example_filters = filters(out_channels=6, K=7)

plt.figure(figsize=(8,4))
plt.subplot(1,2,1)
plt.imshow(example_filters[0,0], vmin=-1, vmax=1, cmap='bwr')
plt.title('center-surround filter')
plt.axis('off')
plt.subplot(1,2,2)
plt.imshow(example_filters[4,0], vmin=-1, vmax=1, cmap='bwr')
plt.title('gabor filter')
plt.axis('off');
```

### Coding Exercise 1.3: 2D convolution in pytorch

We will now run the convolutional layer on our stimulus. We will use gratings stimuli made using the function `grating`, which returns a stimulus which is 48 x 64.

Reminder, `nn.Conv2d` takes in a tensor of size $(N, C^{in}, H^{in}, W^{in}$) where $N$ is the number of stimuli, $C^{in}$ is the number of input channels, and $(H^{in}, W^{in})$ is the size of the stimulus. We will need to add these first two dimensions to our stimulus, then input it to the convolutional layer.

We will plot the outputs of the convolution. `convout` is a tensor of size $(N, C^{out}, H^{in}, W^{in})$ where $N$ is the number of examples and $C^{out}$ are the number of convolutional channels. It is the same size as the input because we used a stride of 1 and padding that is half the kernel size.

```python
# Stimulus parameters
in_channels = 1 # how many input channels in our images
h = 48 # height of images
w = 64 # width of images

# Convolution layer parameters
K = 7 # filter size
out_channels = 6 # how many convolutional channels to have in our layer
example_filters = filters(out_channels, K) # create filters to use

convout = np.zeros(0) # assign convolutional activations to convout

################################################################################
## TODO for students: create convolutional layer in pytorch
# Complete and uncomment
#raise NotImplementedError("Student exercise: create convolutional layer")
################################################################################

# Initialize conv layer and add weights from function filters
# you need to specify :
# * the number of input channels c_in
# * the number of output channels c_out
# * the filter size K
convLayer = ConvolutionalLayer(c_in=in_channels, c_out=out_channels, K=K, filters=example_filters) #

# Create stimuli (H_in, W_in)
orientations = [-90, -45, 0, 45, 90]
stimuli = torch.zeros((len(orientations), in_channels, h, w), dtype=torch.float32)
for i,ori in enumerate(orientations):
  stimuli[i,0] = grating(ori)

convout = convLayer(stimuli) #
convout = convout.detach() # detach gradients

plot_example_activations(stimuli, convout, channels=np.arange(0, out_channels))
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_712d995b.py)

*Example output:*

### Think! 1.3: Output and weight shapes in a convolutional layer

Let's think about the shape of the weights and outputs of `convLayer`:
  - How many convolutional activations are there in each channel, and why is it this size?
  - How many weights does `convLayer` have?
  - How many weights would it have if it were a fully connected layer?

Additionally, let's think about why the activations look the way they do. It seems like for all channels the activations are only non-zero for edges of the gratings (where the grating goes from white-to-black and from black-to-white).
  -  Channel 0 and 1 seem to respond to every edge regardless of the orientation, but their signs are different. What type of filter might produce these types of responses?
  - Channels 2-5 seem to respond differently depending on the orientation of the stimulus. What type of filter might produce these types of responses?

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_f48d6ecf.py)

Please see Bonus Section 2 to visualize the convolutional filter weights. See the Bonus Tutorial to use CNNs as encoding models of neurons (by fitting directly to neural responses).

---
# Summary

*Estimated timing of tutorial: 40 minutes*

In this notebook, we built a 2D convolutional layer which is meant to represent the responses of neurons in mouse visual cortex, or the responses of neurons
which are inputs to mouse visual cortex.

In Tutorial 3, we will add to this 2D convolutional layer a fully-connected layer and train this model to predict whether an orientation is left or right. We will see if the convolutional filters it learns are similar to mouse visual cortex. See Section 3 in the Bonus Tutorial to fit convolutional neural networks directly to neural activity.

---
# Bonus

## Bonus Section 1: Why CNN's?

CNN models are particularly [well-suited](https://www.nature.com/articles/nn.4244) to modeling the visual system for a number of reasons:

1. **Distributed computation**: like any other neural network, CNN's use distributed representations to compute -- much like the brain seems to do. Such models, therefore, provide us with a vocabulary with which to talk and think about such distributed representations. Because we know the exact function the model is built to perform (e.g. orientation discrimination), we can analyze its internal representations with respect to this function and begin to interpret why the representations look the way they do. Most importantly, we can then use these insights to analyze the structure of neural representations observed in recorded population activity. We can qualitatively and quantitatively compare the representations we see in the model and in a given population of real neurons to hopefully tease out the computations it performs.

2. **Hierarchical architecture**: like in any other deep learning architecture, each layer of a deep CNN comprises a non-linear transformation of the previous layer. Thus, there is a natural hierarchy whereby layers closer to the network output represent increasingly more abstract information about the input image. For example, in a network trained to do object recognition, the early layers might represent information about edges in the image, whereas later layers closer to the output might represent various object categories. This resembles the [hierarchical structure of the visual system](https://pubmed.ncbi.nlm.nih.gov/1822724/), where [lower-level areas](https://www.jneurosci.org/content/25/46/10577.short) (e.g. retina, V1) represent visual features of the sensory input and [higher-level areas](https://www.sciencedirect.com/science/article/pii/S089662731200092X) (e.g. V4, IT) represent properties of objects in the visual scene. We can then naturally use a single CNN to model multiple visual areas, using early CNN layers to model lower-level visual areas and late CNN layers to model higher-level visual areas.

  Relative to fully connected networks, CNN's, in fact, have further hierarchical structure built-in through the max pooling layers. Recall that each output of a convolution + pooling block is the result of processing a local patch of the inputs to that block. If we stack such blocks in a sequence, then the outputs of each block will be sensitive to increasingly larger regions of the initial raw input to the network: an output from the first block is sensitive to a single patch of these inputs, corresponding to its "receptive field"; an output from the second block is sensitive to a patch of outputs from the first block, which together are sensitive to a larger patch of raw inputs comprising the union of their receptive fields. Receptive fields thus get larger for deeper layers (see [here](http://colah.github.io/posts/2014-07-Conv-Nets-Modular/) for a nice visual depiction of this). This resembles primate visual systems, where neurons in higher-level visual areas respond to stimuli in wider regions of the visual field than neurons in lower-level visual areas.

3. **Convolutional layers**: through the weight sharing constraint, the outputs of each channel of a convolutional layer process different parts of the input image in exactly the same way. This architectural constraint effectively builds into the network the assumption that objects in the world typically look the same regardless of where they are in space. This is useful for modeling the visual system for two (largely separate) reasons:
  * Firstly, this assumption is generally valid in mammalian visual systems, since mammals tend to view the same object from many perspectives. Two neurons at a similar hierarchy in the visual system with different receptive fields could thus end up receiving statistically similar synaptic inputs, so that the synaptic weights developed over time may end up being similar as well.
  * Secondly, this architecture significantly improves object recognition ability. Object recognition was essentially an unsolved problem in machine learning until the [advent](https://en.wikipedia.org/wiki/AlexNet) of techniques for effectively training *deep* convolutional neural networks. Fully connected networks on their own can't achieve object recognition abilities anywhere close to human levels, making them bad models of human object recognition. Indeed, it is generally the case that [the better a neural network model is at object recognition, the closer the match between its representations and those observed in the brain](https://www.pnas.org/content/111/23/8619.short). That said, it is worth noting that our much simpler orientation discrimination task (in Tutorial 3) can be solved by relatively simple networks.

---
## Bonus Section 2: Understanding activations from weight

### Bonus Coding Exercise 2: Visualizing convolutional filter weights

Why do the activations look the way they do? Let's look at the weights (`convLayer.conv.weight.detach()`) of the convolutional filters and try to interpret them.

```python
################################################################################
## TODO for students: get weights
# Complete and uncomment
raise NotImplementedError("Student exercise: get weights")
################################################################################

# get weights of conv layer in convLayer
weights = ...
print(weights.shape) # can you identify what each of the dimensions are?

plot_weights(weights, channels=np.arange(0, out_channels))
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_20c66403.py)

*Example output:*

In the function `filters` we premade center-surround filters and Gabor filters of various orientations. [Gabor filters](https://en.wikipedia.org/wiki/Gabor_filter) have a positive region next to a negative region and the orientation of these regions of the filter determine the orientation of edges to which they respond.

In visual cortex, Hubel and Wiesel discovered simple cells, which would correspond to a unit in the channels 2-5 above with Gabor filters. There are also some neurons with activity that resembles center-surround filters, which would correspond to the first two convolutional channels above.

There were additional cells discovered by Hubel and Wiesel - complex cells - that respond to an oriented grating regardless of where the bars are exactly (note that the responses we see are specific to where the bars are). These cells therefore have some level of translation invariance. This is something that convolutional neural networks try to replicate -- e.g. a grating is still oriented horizontal even if it moves slightly, and a cat is still a cat even if it's in a different position in the image.

### Bonus Think! 2
  - How might you create a complex cell and have responses that are translation invariant?
  - How might you create a cell that responds to multiple orientations?

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D1_DeepLearning/solutions/W2D1_Tutorial2_Solution_d290c523.py)


# メモ

python で graph theory のグラフをどう描くか、試行錯誤。 実験

参考
1. networkx 本家 tutorial
https://networkx.org/documentation/stable/tutorial.html
1. グラフ理論
Lecture 3: Graphs, Relations and a Matrix
http://schmidt.ucg.ie/cs4423/networks03.pdf
1. sagemath での描き方 sage: g=graphs.GrotzschGraph()
1. https://qiita.com/kzm4269/items/081ff2fdb8a6b0a6112f <=qiita

1. https://www.analyticsvidhya.com/blog/2018/09/introduction-graph-theory-applications-python/
1. https://www.python-course.eu/networkx.php
1. https://igraph.org/python/doc/tutorial/tutorial.html igraph tutorial
1. plotly, pypi

networkx で作ったグラフ理論のグラフを colab (jupyter) で表示するのに

1. matplotlib.pyplot の show を使う方法
1. networkx の draw を使う方法
1. IPython.display の SVG と Image を使う方法
1. pygraphviz を使う方法

の 4通りある。

基本的に matplotlib の show を使う。

networkx の draw は裏で matplotlib を使っているので、出力は同じと思われる。

pygraphviz はツールのインストールが必要なので colab で使うのは現実的でない。

SVG に変換する networkx の関数は裏で pygraphiviz を使っていて、pygraphviz のインストールが必要である。 なので同じ。

```python
import networkx as nx
nx.__version__
```

    '2.5'

```python
# 参考
# pygraphviz を使いたい場合次のようにすれば colab (jupyter) で描ける
# !apt install libgraphviz-dev
# !pip install pygraphviz
#
# としておいて別のコードセルで
#
# import pygraphviz
# import networkx as nx
# from IPython.display import Image
# G = nx.DiGraph()
# data_array = [1,2,3,4,5,6,7,8]
# nx.add_cycle(G, data_array)
# Image(nx.nx_agraph.view_pygraphviz(G, prog='fdp')[0])
#
# nx.nx_agraph.view_pygraphviz(G, prog='fdp') の結果が (png, object) のタプルなので [0] で png を取り出している
# png の表示方法に IPython.display のt Image を使ったがもっと一般的な方法があるかもしれない
```

# グラフを描く

```python
# グラフを描く
import networkx as nx
G = nx.Graph()
data_array =['A','B','C']
nx.add_cycle(G, data_array)
nx.draw_networkx(G)
plt.show (G)
```

```python
# 実験 数字と混ぜられるか
import networkx as nx
G = nx.Graph()
data_array =['A','B','C']
nx.add_cycle(G, data_array)
G.add_nodes_from([2, 3])
nx.draw_networkx(G)
plt.show (G)
```

```python
# 実験 数字と混ぜられるか
import networkx as nx
G = nx.Graph()
data_array =['A','B','C',2,3]
nx.add_cycle(G, data_array)
nx.draw_networkx(G)
plt.show (G)
```

# 有向グラフ DiGraph

```python
import networkx as nx
import matplotlib.pyplot as plt

G = nx.DiGraph()  # 有向グラフ (Directed Graph)
data_array = [1,2,3,4,5,6,7,8]

nx.add_cycle(G, data_array)

nx.draw_networkx(G)
plt.show (G)
```

```python
# 上のセルと同じものを SVG にしてから出力
# pygraphviz をインストールしないとエラーになる
# import networkx as nx
# from IPython.display import SVG

# G = nx.DiGraph()  # 有向グラフ (Directed Graph)
# data_array = [1,2,3,4,5,6,7,8]

# nx.add_cycle(G, data_array)

# display(SVG(nx.nx_agraph.to_agraph(G).draw(prog='fdp', format='svg')))
```

```python
import networkx as nx
import matplotlib.pyplot as plt

G = nx.DiGraph()  # 有向グラフ (Directed Graph)

# 頂点の追加
G.add_node(1)
G.add_nodes_from([3, 4, 5])
plt.show (nx.draw_networkx(G))

# 辺の追加 (頂点も必要に応じて追加される)
G.add_edge(1, 2)
G.add_edges_from([(1, 3), (2, 5), (3, 4), (4, 5)])
plt.show (nx.draw_networkx(G))

# 辺の削除
G.remove_edge(3, 4)
G.remove_edges_from([(1, 3), (2, 5)])
plt.show (nx.draw_networkx(G))

# 頂点の削除 (削除された頂点に接続されている辺も削除される)
G.remove_node(5)
G.remove_nodes_from([3, 4])
plt.show (nx.draw_networkx(G))

# 指定したパス上の頂点と辺を追加
nx.add_path(G, [1, 2, 3, 4, 5])  # 1 → 2 → 3 → 4 → 5
plt.show (nx.draw_networkx(G))

# 指定した閉路上の頂点と辺を追加
nx.add_cycle(G, [1, 2, 3, 4, 5])  # 1 → 2 → 3 → 4 → 5 → 1
plt.show (nx.draw_networkx(G))

# 放射状に頂点と辺を追加
nx.add_star(G, [1, 2, 3, 4, 5])  # 1 → 2, 1 → 3, 1 → 4, 1 → 5
plt.show (nx.draw_networkx(G))

```

```python
# 実験
import matplotlib.pyplot as plt
import networkx as nx

G = nx.DiGraph()
nx.add_path(G, [3, 5, 4, 1, 0, 2, 7, 8, 9, 6])
nx.add_path(G, [3, 0, 6, 4, 2, 7, 1, 9, 8, 5])

nx.draw_networkx(G)
plt.show()
```

```python
# 実験 頂点の名前を変えるにはどうしたらよいか
# 以下の方法では変わらない
import matplotlib.pyplot as plt
import networkx as nx

G = nx.DiGraph()
G.add_edges_from([(1, 2), (1, 3), (2, 3)])

# G.nodes[頂点][属性キー] = 属性値
G.nodes[1]['a'] = 'Alice'
nx.draw_networkx(G)
plt.show()

# G.edges[辺][属性キー] = 属性値
G.edges[1, 2]['b'] = 'Bob'

# G.succ[始点][終点][属性キー] = 属性値
G.succ[2][3]['c'] = 'Carol'

# G.pred[終点][始点][属性キー] = 属性値
G.pred[3][1]['d'] = 'Dave'

print(dict(G.nodes))
# {1: {'a': 'Alice'}, 2: {}, 3: {}}

print(dict(G.edges))
# {(1, 2): {'b': 'Bob'}, (1, 3): {'d': 'Dave'}, (2, 3): {'c': 'Carol'}}

print(G.succ)
# {1: {2: {'b': 'Bob'}, 3: {'d': 'Dave'}}, 2: {3: {'c': 'Carol'}}, 3: {}}

print(G.pred)
# {1: {2: {'b': 'Bob'}, 3: {'d': 'Dave'}}, 2: {3: {'c': 'Carol'}}, 3: {}}
nx.draw_networkx(G)
plt.show()
```

```python
# 実験 node の名前が長かった場合
import networkx as nx
G = nx.Graph()
G.add_node("Allways at the Center")
nx.draw_networkx(G)
plt.show (G)
```

```python
# node だけを追加する
import networkx as nx
G = nx.Graph()
G.add_node("A")
G.add_nodes_from([2, 3])
nx.draw_networkx(G)
plt.show (G)
```

```python
import networkx as nx
import matplotlib.pyplot as plt
G = nx.Graph()
G.add_node("A")
G.add_nodes_from([2, 3])
G.add_nodes_from([
    (4, {"node_color": "red"}), # ここで色を指定しても反映しない
    (5, {"color": "green"}),
])
nx.draw_networkx(G, node_color='grey')
plt.show (G)

# node の属性はどうしたら反映されるか。 とりあえず色が変わらない。
```

```python
# 2つのグラフの node を足す
import networkx as nx
import matplotlib.pyplot as plt
G = nx.Graph()
G.add_node("A")
G.add_node("B")
H = nx.path_graph(10)
nx.draw_networkx(H)
plt.show (H)

G.add_nodes_from(H)

nx.draw_networkx(G)
plt.show (G)

```

```python
# 辺を追加する add_edge(,), add_edges_from([()])
import networkx as nx
import matplotlib.pyplot as plt
G = nx.Graph()
G.add_edge(1, 2)
e = (2, 3)
G.add_edge(*e)  # unpack edge tuple*
G.add_edges_from([(4,1),(4,2)])
nx.draw_networkx(G)
plt.show (G)
```

# グラフ理論と線形代数と集合論

graph theory , linear algebra, matrix multiplication , set theory

```python
import networkx as nx
import numpy as np
import matplotlib.pyplot as plt
G = nx.Graph(["AB","BC","BD","CD","DE"])
opts = {"with_labels": True, "node_color": 'yellow'}
nx.draw(G, **opts)
nx.draw_networkx(G, node_color='red')
plt.show(G)
```

# 隣接行列 adjacency matrix

グラフ理論および計算機科学において、隣接行列は、有限グラフを表わすために使われる正方行列である。

$
A = \begin{pmatrix}0 & 1 & 0 & 0 & 0\\1 & 0 & 1 & 1 & 0\\0 & 1 & 0 & 1 & 0\\0 & 1 & 1 & 0 & 1\\0 & 0 & 0 & 1 & 0\end{pmatrix}
$

行と列はグラフの頂点に対応し、対応する頂点がエッジで結合されているかどうかに関係なく、行と列がある。

行や列が頂点に対応し、その中のエントリが繋がりを表すことによってエッジを示す。

グラフ G の隣接行列 A は
```
A=nx.adjacency_matrix(G)
```
で算出される。

```python
import networkx as nx
from sympy import *
init_printing()
G = nx.Graph(["AB","BC","BD","CD","DE"])
A=nx.adjacency_matrix(G)
print(A)
print(type(A))
print(type(A.toarray()))
print(A.toarray())
display(Matrix(A.toarray()))
print(latex(Matrix(A.toarray())))
```

      (0, 1)	1
      (1, 0)	1
      (1, 2)	1
      (1, 3)	1
      (2, 1)	1
      (2, 3)	1
      (3, 1)	1
      (3, 2)	1
      (3, 4)	1
      (4, 3)	1
    <class 'scipy.sparse.csr.csr_matrix'>
    <class 'numpy.ndarray'>
    [[0 1 0 0 0]
     [1 0 1 1 0]
     [0 1 0 1 0]
     [0 1 1 0 1]
     [0 0 0 1 0]]

$\displaystyle \left[\begin{matrix}0 & 1 & 0 & 0 & 0\\1 & 0 & 1 & 1 & 0\\0 & 1 & 0 & 1 & 0\\0 & 1 & 1 & 0 & 1\\0 & 0 & 0 & 1 & 0\end{matrix}\right]$

    \left[\begin{matrix}0 & 1 & 0 & 0 & 0\\1 & 0 & 1 & 1 & 0\\0 & 1 & 0 & 1 & 0\\0 & 1 & 1 & 0 & 1\\0 & 0 & 0 & 1 & 0\end{matrix}\right]

この行列 $A=(a_ij)$ は対角線について対称である。

すべての $i,j$ について $a_ij = a_ji$ である。

頂点からそれ自身への辺 loop を許容しない場合、対角要素は全て 0 になる。

頂点のラベル名を無視すれば、隣接行列からグラフを再現できる。

```python
import matplotlib.pyplot as plt
# H=nx.from_numpy_matrix(A.toarray())
H=nx.from_numpy_array(A.toarray())
opts = {"with_labels": True, "node_color": 'green'}
nx.draw(H, **opts)
opts = {"with_labels": True, "node_color": 'red'}
nx.draw_networkx(H, **opts)
plt.show (H)
print((A**0).toarray())
print((A*A).toarray())
print((A.toarray()**0))
print((A.toarray()*A.toarray()))
```

# いまここ

# 次数 degree

単純グラフの頂点 $x$ が接続している他の頂点の数を次数と呼ぶ。 ネットワークにおけるノードの重要度を測る簡単な手法となる。

次数の平均は、そのグラフの頂点の数 $n$ と 辺の数 $m$ のみで決まる。

# いまここ最後


```python
from sympy import *
VxC, VxW, VxE, VxS, VxN, VySW, VySE, VyNW, VyNE = symbols('VxC, VxW, VxE, VxS, VxN, VySW, VySE, VyNW, VyNE')
etaW, etaE, etaS, etaN = symbols('etaW, etaE, etaS, etaN')
dx, dy = symbols('dx, dy')
PW, PE = symbols('PW, PE')
Dir_N, r = symbols('Dir_N, r')

divW = (VxC-VxW)/dx + (VyNW-VySW)/dy
ExxW = (VxC-VxW)/dx - 1.0/3.0*divW
divE = (VxE-VxC)/dx + (VyNE-VySE)/dy
ExxE = (VxE-VxC)/dx - 1.0/3.0*divE

ExyS = 1.0/2.0*( (VxC-VxS)/dy + (VySE-VySW)/dx)
ExyN = 1.0/2.0*( ((2*Dir_N*r-VxC)-VxC)/dy + (VyNE-VyNW)/dx)
# ExyN = 1.0/2.0*( (VxN-VxC)/dy + (VyNE-VyNW)/dx)

SxxW = -PW + 2.0*etaW*ExxW
SxxE = -PE + 2.0*etaE*ExxE
SxyS =       2.0*etaS*ExyS
SxyN =       2.0*etaN*ExyN

Fx = - ( (SxxE-SxxW)/dx + (SxyN-SxyS)/dy )
print("cVxC  = " + octave_code(Fx.diff(VxC)).replace('1.33333333333333', '4/3'))
print("cVxW  = " + octave_code(Fx.diff(VxW)).replace('1.33333333333333', '4/3'))
print("cVxE  = " + octave_code(Fx.diff(VxE)).replace('1.33333333333333', '4/3'))
print("cVxS  = " + octave_code(Fx.diff(VxS)).replace('1.33333333333333', '4/3'))
print("cVxN  = " + octave_code(Fx.diff(VxN)).replace('1.33333333333333', '4/3'))

print("cVySW = " + octave_code(Fx.diff(VySW)).replace('0.666666666666667', '2/3'))
print("cVySE = " + octave_code(Fx.diff(VySE)).replace('0.666666666666667', '2/3'))
print("cVyNW = " + octave_code(Fx.diff(VyNW)).replace('0.666666666666667', '2/3'))
print("cVyNE = " + octave_code(Fx.diff(VyNE)).replace('0.666666666666667', '2/3'))

print("cPW   = " + octave_code(Fx.diff(PW)))
print("cPE   = " + octave_code(Fx.diff(PE)))

print("r   = " + octave_code(-Fx.diff(r)))

```

    cVxC  = -(-2.0*etaN./dy - 1.0*etaS./dy)./dy - (-4/3*etaE./dx - 4/3*etaW./dx)./dx
    cVxW  = -4/3*etaW./dx.^2
    cVxE  = -4/3*etaE./dx.^2
    cVxS  = -1.0*etaS./dy.^2
    cVxN  = 0
    cVySW = -1.0*etaS./(dx.*dy) + 2/3*etaW./(dx.*dy)
    cVySE = -2/3*etaE./(dx.*dy) + 1.0*etaS./(dx.*dy)
    cVyNW = 1.0*etaN./(dx.*dy) - 2/3*etaW./(dx.*dy)
    cVyNE = 2/3*etaE./(dx.*dy) - 1.0*etaN./(dx.*dy)
    cPW   = -1./dx
    cPE   = 1./dx
    r   = 2.0*Dir_N.*etaN./dy.^2

```python
from sympy import *
VyC, VyW, VyE, VyS, VyN, VxSW, VxSE, VxNW, VxNE = symbols('VyC, VyW, VyE, VyS, VyN, VxSW, VxSE, VxNW, VxNE')
etaW, etaE, etaS, etaN = symbols('etaW, etaE, etaS, etaN')
dx, dy = symbols('dx, dy')
PS, PN = symbols('PS, PN')

divS = (VyC-VyS)/dy + (VxSE-VxSW)/dx
EyyS = (VyC-VyS)/dy - 1.0/3.0*divS
divN = (VyN-VyC)/dy + (VxNE-VxNW)/dx
EyyN = (VyN-VyC)/dy - 1.0/3.0*divN

ExyW = 1.0/2.0*( (VyC-VyW)/dx + (VxNW-VxSW)/dy)
ExyE = 1.0/2.0*( (VyE-VyC)/dx + (VxNE-VxSE)/dy)

SyyS = -PS + 2.0*etaS*EyyS
SyyN = -PN + 2.0*etaN*EyyN
SxyW =       2.0*etaW*ExyW
SxyE =       2.0*etaE*ExyE

Fy = - ( (SyyN-SyyS)/dy + (SxyE-SxyW)/dx )
print("cVyC  = " + octave_code(Fy.diff(VyC)).replace('1.33333333333333', '4/3'))
print("cVyW  = " + octave_code(Fy.diff(VyW)).replace('1.33333333333333', '4/3'))
print("cVyE  = " + octave_code(Fy.diff(VyE)).replace('1.33333333333333', '4/3'))
print("cVyS  = " + octave_code(Fy.diff(VyS)).replace('1.33333333333333', '4/3'))
print("cVyN  = " + octave_code(Fy.diff(VyN)).replace('1.33333333333333', '4/3'))

print("cVxSW = " + octave_code(Fy.diff(VxSW)).replace('0.666666666666667', '2/3'))
print("cVxSE = " + octave_code(Fy.diff(VxSE)).replace('0.666666666666667', '2/3'))
print("cVxNW = " + octave_code(Fy.diff(VxNW)).replace('0.666666666666667', '2/3'))
print("cVxNE = " + octave_code(Fy.diff(VxNE)).replace('0.666666666666667', '2/3'))

print("cPS   = " + octave_code(Fy.diff(PS)))
print("cPN   = " + octave_code(Fy.diff(PN)))
```

    cVyC  = -(-4/3*etaN./dy - 4/3*etaS./dy)./dy - (-1.0*etaE./dx - 1.0*etaW./dx)./dx
    cVyW  = -1.0*etaW./dx.^2
    cVyE  = -1.0*etaE./dx.^2
    cVyS  = -4/3*etaS./dy.^2
    cVyN  = -4/3*etaN./dy.^2
    cVxSW = 2/3*etaS./(dx.*dy) - 1.0*etaW./(dx.*dy)
    cVxSE = 1.0*etaE./(dx.*dy) - 2/3*etaS./(dx.*dy)
    cVxNW = -2/3*etaN./(dx.*dy) + 1.0*etaW./(dx.*dy)
    cVxNE = -1.0*etaE./(dx.*dy) + 2/3*etaN./(dx.*dy)
    cPS   = -1./dy
    cPN   = 1./dy

```python

```


###### Content under Creative Commons Attribution license CC-BY 4.0, code under MIT license (c)2014 L.A. Barba, C.D. Cooper, G.F. Forsyth.

# Riding the wave

## Convection problems

Welcome to *Riding the wave: Convection problems*, the third module of  ["Practical Numerical Methods with Python"](http://openedx.seas.gwu.edu/courses/GW/MAE6286/2014_fall/about).

In the [first module](https://github.com/numerical-mooc/numerical-mooc/tree/master/lessons/01_phugoid), we learned about numerical integration methods for the solution of ordinary differential equations (ODEs). The [second module](https://github.com/numerical-mooc/numerical-mooc/tree/master/lessons/02_spacetime) introduced the finite difference method for numerical solution of partial differential equations (PDEs), where we need to discretize both *space* and *time*.

This module explores the convection equation in more depth, applied to a traffic-flow problem.  We already introduced convection in [Lesson 1 of Module 2](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/master/lessons/02_spacetime/02_01_1DConvection.ipynb). This hyperbolic equation is very interesting because the solution can develop *shocks*, or regions with very high gradient, which are difficult to resolve well with numerical methods.

We will start by introducing the concept of a conservation law, closely related to the convection equation.  Then we'll explore different numerical schemes and how they perform when shocks are present.

## Conservation laws

You know from (non relativistic) physics that mass is _conserved_. This is one example of a conserved quantity, but there are others (like momentum and energy) and they all obey a _conservation law_. Let's start with the more intuitive case of conservation of mass.

### Conservation of mass

In any closed system, we know that the mass $M$ in the system does not change, which we can write: $\frac{D\,M}{Dt} =0$. When we change the point of view from a closed system to what engineers call a _control volume_, mass can move in and out of the volume and conservation of mass is now expressed by:

Let's imagine the control volume as a tiny cylinder of cross-section dA and length dx, like in the sketch below.

#### Figure 1. Tiny control volume in the shape of a cylinder.

If we represent the mass density by $\rho$, then mass is equal to $\rho\times$ volume. For simplicity, let's assume that mass flows in or out of the control volume only in one direction, say, the $x$-direction. Express the 1D velocity component by $u$, and conservation of mass for the control volume is translated to a mathematical expression as follows:

\begin{equation}
\frac{\partial}{\partial t}\int_{\text{cv}}\rho \, dV + \int_{\text{cs}}\rho \, u\, dA =0
\end{equation}

where "cv" stands for control volume and "cs" stands for control surface. The first term represents the rate of change of mass in the control volume, and the second term is the rate of flow of mass, with velocity $u$, accross the control surface.

Since the control volume is very small, we can take, to leading order, $\rho$ as a uniform quantity inside it, and the first term in equation (1) can be simplified to the time derivative of density multiplied by the volume of the tiny cylinder, $dAdx$:

$$\frac{\partial}{\partial t}\int_{\text{cv}}\rho \, dV \rightarrow \frac{\partial \rho}{\partial t} dAdx$$

Now, for the second term in equation (1), we have to do a little more work. The quantity inside the integral is now $\rho u$ and, to leading order, we have to take into consideration that this quantity can change in the distance $dx$. Take $\rho u$ to be the value in the center of the cylinder. Then the flow of mass on each side is illustrated in the figure below, where we use a Taylor expansion of the quantity $\rho u$ around the center of the control volume (to first order).

#### Figure 2. Flux terms on the control surfaces.

Subtracting the negative flux on the left to the positive flux on the right, we arrive at the total flux of mass accross the control surfaces, the second term in equation (1):

$$ \int_{\text{cs}}\rho \, u\, dA \rightarrow \frac{\partial}{\partial x}(\rho u) dAdx$$

We can now put together the equation of conservation of mass for the tiny cylindrical control volume, which after diving by $dAdx$ is:

\begin{equation}
\frac{\partial \rho}{\partial t} + \frac{\partial}{\partial x}(\rho u)=0
\end{equation}

This is the 1D mass conservation equation in differential form. If we take $u$ to be a constant and take it out of the spatial derivative this equation looks the same as the first PDE we studied: the linear convection equation in [Lesson 1 of Module 2](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/master/lessons/02_spacetime/02_01_1DConvection.ipynb).
But in the form shown above, it is a typical _conservation law_. The term under the spatial derivative is called the _flux_, for reasons that should be clear from our discussion above: it represents amounts of the conserved quantity flowing across the boundary of the control volume.

##### Dig deeper

You can follow the derivation of the full three-dimensional equation of conservation of mass for a flow on this screencast by Prof. Barba (duration 12:47).

```python
from IPython.display import YouTubeVideo
YouTubeVideo('35unQgSaT88')
```

### General conservation laws

All conservation laws express the same idea: the variation of a conserved quantity inside a control volume is due to the total flux of that quantity crossing the boundary surface (plus possibly the effect of any sources inside the volume, but let's ignore those for now).

The _flux_ is a fundamental concept in conservation laws: it represents the amount of the quantity that crosses a surface per unit time. Our discussion above was limited to flow in one dimension, but in general the flux has any direction and is a vector quantity. Think about this: if the direction of flow is parallel to the surface, then no quantity comes in or out. We really only care about the component of flux perpendicular to the surface. Mathematically, for a vector flux $\vec{F}$, the amount of the conserved quantity crossing a small surface element is:

$$\vec{F}\cdot d\vec{A}$$

where $d\vec{A}$ points in the direction of the outward normal to the surface. A general conservation law for a quantity $e$ is thus (still ignoring possible sources):

\begin{equation}
\frac{\partial}{\partial t}\int_{\text{cv}}e \, dV + \oint_{\text{cs}}\vec{F}\cdot d\vec{A} =0
\end{equation}

To obtain a differential form of this conservation equation, we can apply the theorem of Gauss to the second integral, which brings the gradient of $\vec{F}$ into play. One way to recognize a conservation law in differential form is that the _fluxes appear only under the gradient operator_.

Recall the non-linear convection equation from [Lesson 1 of Module 2](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/master/lessons/02_spacetime/02_01_1DConvection.ipynb). It was:

\begin{equation}\frac{\partial u}{\partial t} + u \frac{\partial u}{\partial x} = 0\end{equation}

If we look closely at the spatial derivative, we can rewrite this equation as

\begin{equation}\frac{\partial u}{\partial t} + \frac{\partial}{\partial x} \left(\frac{u^2}{2} \right) = 0, \end{equation}

which is the *conservation form* of the non-linear convection equation, with flux $F=\frac{u^2}{2}$.

## Traffic flow model

We've all experienced it: as rush hour approaches certain roads in or out of city centers start getting full of cars, and the speed of travel can reduce to a crawl. Sometimes, the cars stop altogether. If you're a driver, you know that the more cars on the road, the slower your trip will flow.

Traffic flow models seek to describe these everyday experiences with mathematics, to help engineers design better road systems.

Let's review the [Lighthill-Whitham-Richards](http://en.wikipedia.org/wiki/Macroscopic_traffic_flow_model) traffic model that was offered as an exercise at the end of Module 2. This model considers cars with a continuous *traffic density* (average number of cars per unit length of road) rather than keeping track of them individually. If $\rho(x)=0$, there are no cars at that point $x$ of the road.  If $\rho(x) = \rho_{\rm max}$, traffic is literally bumper to bumper.

If the number of cars on a bounded stretch of road changes, it means that cars are entering or leaving the road somehow. _Traffic density obeys a conservation law_ (!) where the flux is the number of cars leaving the road per unit time. It is given by $F=\rho u$—as with mass conservation, flux equals density times velocity. But don't forget your experience on the road: the speed of travel depends on the car density. Here, $u$ refers not to the speed of each individual car, but to the _traffic speed_ at a given point of the road.

You know from experience that with more cars on the road, the speed of travel decreases. It is also true that if you are traveling at fast speed, you are advised to leave a larger gap with cars ahead. These two considerations lead us to propose a monotonically decreasing $u=u(\rho)$ function. As a first approximation, we may consider the linear function:

\begin{equation}u(\rho) = u_{\rm max} \left(1-\frac{\rho}{\rho_{\rm max}}\right)\end{equation}

#### Figure 3. Traffic speed vs. traffic density.

The linear model of the behavior of drivers satisfies these experimental observations:
1.  All drivers will approach a maximum velocity $u_{max}$ when the road is empty.
2.  If the road is completely jampacked ($\rho \rightarrow \rho_{max}$), velocity goes to zero.

That seems like a reasonable approximation of reality!

Applying a conservation law to the vehicle traffic, the traffic density will obey the following transport equation:

\begin{equation} \frac{\partial \rho}{\partial t} + \frac{\partial F}{\partial x} = 0 \end{equation}

where $F$ is the *traffic flux*, which in the linear traffic-speed model is given by:

\begin{equation} F = \rho u_{\rm max} \left(1-\frac{\rho}{\rho_{\rm max}}\right).
\end{equation}

We can now use our numerical kung-fu to solve some interesting traffic situations, and check if our simple model gives realistic results!

### Green light!

Let's say that we are examining a road of length $4$ where the speed limit is $u_{\rm max}=1$, fitting $10$ cars per unit length $(\rho_{\rm max}=10)$. Now, imagine we have an intersection with a red light at $x=2$.  At the stoplight, traffic is bumper-to-bumper, and the traffic density decreases linearly to zero as we approach the beginning of our road. Ahead of the stoplight, the road is clear.

Mathematically, we can represent this situation with the following initial condition:

\begin{equation}\rho(x,0) = \left\{ \begin{array}{cc}
\rho_{\rm max}\frac{x}{2} & 0 \leq x < 2  \\
0 & 2 \leq x \leq 4 \\ \end{array} \right.\end{equation}

Let's see what a plot of that looks like.

```python
%matplotlib inline
import numpy
from matplotlib import pyplot
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

```python
def rho_green_light(x, rho_light):
    """Computes "green light" initial condition with shock, and linear distribution behind

    Parameters
    ----------
    x         : array of floats
        Locations on the road
    rho_light : float
        Density of cars at stoplight

    Returns
    -------
    rho: array of floats
        Array with initial values of density
    """
    rho = numpy.zeros_like(x)
    mask = numpy.where(x < 2.0)
    rho[mask] = rho_light * x[mask] / 2.0

    return rho
```

```python
#Basic initial condition parameters
#defining grid size, time steps
nx = 81
nt = 30
dx = 4.0/(nx-1)

x = numpy.linspace(0,4,nx)

rho_max = 10.
u_max = 1.
rho_light = 10.

rho = rho_green_light(x, rho_light)
```

```python
pyplot.plot(x, rho, color='#003366', ls='-', lw=3)
pyplot.ylabel('Traffic density')
pyplot.xlabel('Distance')
pyplot.ylim(-0.5,11.);
```

**How does the traffic behave once the light turns green?** Cars should slowly start moving forward: the density profile should move to the right. Let's see if the numerical solution agrees with that!

Before we start, let's define a function to calculate the traffic flux. We'll use it in each time step of our numerical solution.

```python
def computeF(u_max, rho_max, rho):
    """Computes flux F=V*rho

    Parameters
    ----------
    u_max  : float
        Maximum allowed velocity
    rho    : array of floats
        Array with density of cars at every point x
    rho_max: float
        Maximum allowed car density

    Returns
    -------
    F : array
        Array with flux at every point x
    """
    return u_max*rho*(1-rho/rho_max)
```

### Forward-time/backward-space

Start by using a forward-time, backward-space scheme, like you used in Module 2. The discretized form of our traffic model is:

\begin{equation}
\frac{\rho^{n+1}_i- \rho^n_{i}}{\Delta t}+ \frac{F^n_{i}-F^n_{i-1}}{\Delta x}=0
\end{equation}

Like before, we'll step in time via a for-loop, and we'll operate on all spatial points simultaneously via array operations. In each time step, we also need to call the function that computes the flux. Here is a function that implements in code the forward-time/backward-space difference scheme:

```python
def ftbs(rho, nt, dt, dx, rho_max, u_max):
    """ Computes the solution with forward in time, backward in space

    Parameters
    ----------
    rho    : array of floats
            Density at current time-step
    nt     : int
            Number of time steps
    dt     : float
            Time-step size
    dx     : float
            Mesh spacing
    rho_max: float
            Maximum allowed car density
    u_max  : float
            Speed limit

    Returns
    -------
    rho_n : array of floats
            Density after nt time steps at every point x
    """

    #initialize our results array with dimensions nt by nx
    rho_n = numpy.zeros((nt,len(rho)))
    #copy the initial u array into each row of our new array
    rho_n[0,:] = rho.copy()

    for t in range(1,nt):
        F = computeF(u_max, rho_max, rho)
        rho_n[t,1:] = rho[1:] - dt/dx*(F[1:]-F[:-1])
        rho_n[t,0] = rho[0]
        rho = rho_n[t].copy()

    return rho_n
```

We're all good to go!

**Note:** The code above saves the complete traffic density at each time-step—we'll use that in a second to create animations with our results.

Running the numerical solution is easy now: we just need to call the function for evolving the initial condition with the forward-time/backward-space scheme.

```python
sigma = 1.
dt = sigma*dx

rho_n = ftbs(rho, nt, dt, dx, rho_max, u_max)
```

Let's see how that looks.

Below is another way to use the `matplotlib` animation routines.  Instead of computing and animating at the same time, we can pass the results of our calculation, stored in the array `rho_n`, as individual frames.

It doesn't make much of a difference for a computationally light example like this one, but if you want to animate more complicated problems, it's nice to separate the computation from the visualization.  You don't want to re-run your entire simulation just so you can update the line style!

```python
from matplotlib import animation
from IPython.display import HTML
```

```python
fig = pyplot.figure();
ax = pyplot.axes(xlim=(0,4),ylim=(-.5,11.5),xlabel=('Distance'),ylabel=('Traffic density'));
line, = ax.plot([],[],color='#003366', lw=2);

def animate(data):
    x = numpy.linspace(0,4,nx)
    y = data
    line.set_data(x,y)
    return line,

anim = animation.FuncAnimation(fig, animate, frames=rho_n, interval=50)
```

```python
HTML(anim.to_html5_video())
```

**Yikes! The solution is blowing up.** This didn't happen in your traffic-flow exercise (coding assignment) for Module 2! (Thankfully.) What is going on? Is there a bug in the code?

No need to panic. Let's take a closer look at the equation we are solving:

\begin{equation}\frac{\partial \rho}{\partial t} + \frac{\partial F}{\partial x} = 0\end{equation}

Using the chain rule of calculus, rewrite is as follows:

\begin{equation} \frac{\partial \rho}{\partial t} + \frac{\partial F}{\partial \rho} \frac{\partial \rho}{\partial x} = 0\end{equation}

This form of the equation looks like the nonlinear convection equation from [Lesson 1 of Module 2](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/master/lessons/02_spacetime/02_01_1DConvection.ipynb), right? This is a wave equation where the wave speed is $u_{\rm wave} = \frac{\partial F}{\partial\rho}$. That term is:

\begin{equation}u_{\rm wave} = \frac{\partial F}{\partial \rho} = u_{\rm max} \left( 1-2\frac{\rho}{\rho_{\rm max}} \right).\end{equation}

See how the wave speed changes sign at $\rho = \rho_{\rm max}/2$? That means that for the initial conditions given for the green-light problem, the part of the wave under $\rho = \rho_{\rm max}/2$ will want to move right, whereas the part of the wave over this mark, will move left!

There is no real problem with that in terms of the model, but a scheme that is backward in space is *unstable* for negative values of the wave speed.

## Upwind schemes

Maybe you noticed that the backward-space discretization is spatially biased: we include the points $i$ and $i-1$ in the formula. Look again at the stencil and you'll see what we mean.

#### Figure 4. Stencil of forward-time/backward-space.

In fact, the spatial bias was meant to be in the direction of propagation of the wave—this was true when we solved the convection equation (with positive wave speed $c$), but now we have some problems. Discretization schemes that are biased in the direction that information propagates are called _upwind schemes_.

Remember when we discussed the characteristic lines for the linear convection equation in [lesson 1 of the previous module](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/wave_dev/lessons/02_spacetime/02_01_1DConvection.ipynb)? Compare the sketch of the characteristic lines with the stencil above. The point is that there is an inherent directionality in the physics, and we want the numerical scheme to have the same directionality. This is one example of _choosing an appropriate scheme_ for the physical problem.

If we wanted to solve the convection equation with negative wave speed, $c<0$, we would need a spatial bias "slanting left," which we would obtain by using the points $i$ and $i+1$ in the formula.

But if we have waves traveling in both directions, we are in a bit of a bind. One way to avoid this problem with our traffic flow model is to simply use an initial condition that doesn't produce negative speed. This should work. But later we will learn about other numerical schemes that are able to handle waves in both directions.

Just for a sanity check, let's try the forward-time/backward-space scheme with the initial conditions

\begin{equation}\rho(x,0) = \left\{ \begin{array}{cc}
2.5 x & 0 \leq x < 2  \\
0 & 2 \leq x \leq 4 \\ \end{array} \right.\end{equation}

If all values of $\rho \leq \rho_{\rm max}/2$, then $\frac{\partial F}{\partial \rho}$ is positive everywhere. For these conditions, our forward-time/backward-space scheme shouldn't have any trouble, as all wave speeds are positive.

```python
rho_light = 5.
nt = 40
rho = rho_green_light(x, rho_light)
rho_n = ftbs(rho, nt, dt, dx, rho_max, u_max)

anim = animation.FuncAnimation(fig, animate, frames=rho_n, interval=50)
HTML(anim.to_html5_video())
```

Phew!  It works! Try this out yourself with different initial conditions.

Also, you can easily create a new function `ftfs` to do a forward-time/forward-space scheme, which is stable for negative wave speeds. Unfortunately, forward in space is unstable for positive wave speeds.  If you don't want it blowing up, make sure the wave speed is negative everywhere: $u_{\rm wave} = \frac{\partial F}{\partial \rho} < 0 \ \forall \ x$.

Look at that solution again, and you'll get some nice insights of the real physical problem. See how on the trailing edge, a shock is developing? In the context of the traffic flow problem, a shock is a sign of a traffic jam: a region where traffic is heavy and slow next to a region that is free of cars. In the initial condition, the cars in the rear end of the triangle see a mostly empty road (traffic density is low!).  They see an empty road and speed up, accordingly.  The cars in the peak of the triangle are moving pretty slowly because traffic density is higher there.  Eventually the cars that started in the rear will catch up with the rest and form a traffic jam.

## Beware the CFL!

[Lesson 2 of Module 2](http://nbviewer.ipython.org/github/numerical-mooc/numerical-mooc/blob/wave_dev/lessons/02_spacetime/02_02_CFLCondition.ipynb) discusses the CFL condition for the linear convection equation. To refresh your memory, for a constant wave speed $u_{\rm wave} = c$:

\begin{equation} \sigma = c\frac{\Delta t}{\Delta x} < 1.\end{equation}

What happens for non-linear equations? The wave speed is space- and time-dependent, $u_{\rm wave} = u_{\rm wave}(x,t)$, and the CFL condition needs to apply for every point in space, at every instant of time. We just need $\sigma>1$ in one spot, for the whole solution to blow up!

Let's generalize the CFL condition to

\begin{equation} \sigma = \max\left[ \left| u_{\rm wave} \right| \frac{\Delta t}{\Delta x} \right] < 1.\end{equation}

which in our case is

\begin{equation} \sigma = \max\left[ u_{\rm max} \left| 1-\frac{2 \rho}{\rho_{\rm max}} \right|  \frac{\Delta t}{\Delta x} \right] < 1.\end{equation}

Here, the closer $\rho$ is to zero, the more likely it is to be unstable.

### Green light and CFL

We know that the green-light problem with density at the stop light $\rho = \rho_{\rm light} = 4$ is stable using a forward-time/backward -space scheme. Earlier, we used $u_{\rm max} = 1$, and $\Delta t/\Delta x=1$, which gives a CFL $= 1$, when $\rho = 0$.

What if we change the conditions slightly, say $u_{\rm max} = 1.1$?

```python
rho_light = 4.
u_max = 1.1
nt = 40
rho = rho_green_light(x, rho_light)
rho_n = ftbs(rho, nt, dt, dx, rho_max, u_max)

anim = animation.FuncAnimation(fig, animate, frames=rho_n, interval=50)
HTML(anim.to_html5_video())
```

That failed miserably! Only by changing $u_{\rm max}$ to $1.1$, even an algorithm that we know is stable for this problem, fails. Since we kept $\Delta t/\Delta x=1$, the CFL number for $\rho=0$ is $1.1$. See where the instability begins? Beware the CFL!

## References

* Neville D. Fowkes and John J. Mahony, *"An Introduction to Mathematical Modelling,"* Wiley & Sons, 1994. Chapter 14: Traffic Flow.

* M. J. Lighthill and G. B. Whitham (1955), On kinematic waves. II. Theory of traffic flow and long crowded roads, _Proc. Roy. Soc. A_, Vol. 229, pp. 317–345. [PDF from amath.colorado.edu](https://amath.colorado.edu/sites/default/files/2013/09/1710796241/PRSA_Lighthill_1955.pdf), checked Oct. 14, 2014. [Original source](http://rspa.royalsocietypublishing.org/content/229/1178/317.short) on the Royal Society site.

---

###### The cell below loads the style of the notebook.

```python
from IPython.core.display import HTML
css_file = '../../styles/numericalmoocstyle.css'
HTML(open(css_file, "r").read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>
<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro' rel='stylesheet' type='text/css'>
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 750px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
    margin-top:0.8em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Alegreya Sans' sans-serif;
    line-height: 140%;
    font-size: 125%;
    font-weight: 400;
    width:600px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Nixie One', serif;
    font-style:regular;
    font-weight: 400;
    font-size: 45pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}

.text_cell_render h2 {
    font-family: 'Nixie One', serif;
    font-weight: 400;
    font-size: 30pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.1em;
    margin-top: 0.3em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Nixie One', serif;
    margin-top:16px;
    font-size: 22pt;
    font-weight: 600;
    margin-bottom: 3px;
    font-style: regular;
    color: rgb(102,102,0);
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Nixie One', serif;
    font-size: 14pt;
    text-align: center;
    margin-top: 0em;
    margin-bottom: 2em;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Nixie One', sans-serif;
    font-weight: 400;
    font-size: 16pt;
    color: rgb(163,0,0);
    font-style: italic;
    margin-bottom: .1em;
    margin-top: 0.8em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'PT Mono', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

.CodeMirror{
    font-family: "Source Code Pro";
    font-size: 90%;
}

.alert-box {
    padding:10px 10px 10px 36px;
    margin:5px;
}

.success {
    color:#666600;
    background:rgb(240,242,229);
}
</style>


#  Muon Differential Cross-Sections
#### Roland Grinis  -  Researcher at MIPT Nuclear Physics Methods lab  -  CTO at GrinisRIT (grinisrit.com)
#### Danila Riazanov - Student at MIPT,  JetBrains Research trainee

Code available within `NOA` [github.com/grinisrit/noa](https://github.com/grinisrit/noa) - Differentiable Programming Algorithms in `C++17` over [LibTorch](https://pytorch.org/cppdocs)

## Installation

The `conda` environment provided with the repository has all the required dependencies. For this particular tutorial we will need the following `python` packages:

```python
import torch
from torch.utils.cpp_extension import load
import matplotlib.pyplot as plt
```

Now we need to build and load `C++17/CUDA` extensions for `PyTorch`, set up the locations:

```python
!mkdir -p build
noa_location = '../..'
```

If you are running this on Google Colab, you need to clone `NOA` and set `noa_location` accordingly:
```python
!git clone https://github.com/grinisrit/noa.git
noa_location = 'noa'
```

Also, make sure that `ninja` and `g++-9` or higher are available. The following commands will do that for you:
```python
!pip install Ninja
!add-apt-repository ppa:ubuntu-toolchain-r/test -y
!apt update
!apt upgrade -y
!apt install gcc-9 g++-9
!update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9
```

If you are on Datalore:
```python
!pip install Ninja
!sudo apt-get update && sudo apt-get install cuda-nvcc-11-2 -y
```

Check the installation:

```python
!gcc --version
!g++ --version
!nvcc --version
```

    gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0
    Copyright (C) 2019 Free Software Foundation, Inc.
    This is free software; see the source for copying conditions.  There is NO
    warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

    g++ (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0
    Copyright (C) 2019 Free Software Foundation, Inc.
    This is free software; see the source for copying conditions.  There is NO
    warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

    nvcc: NVIDIA (R) Cuda compiler driver
    Copyright (c) 2005-2021 NVIDIA Corporation
    Built on Mon_Sep_13_19:13:29_PDT_2021
    Cuda compilation tools, release 11.5, V11.5.50
    Build cuda_11.5.r11.5/compiler.30411180_0

Finally, you get the extensions into `python` by calling `load`:

```python
muons = load(name='muons',
             build_directory='./build',
             sources=[f'{noa_location}/docs/pms/muons_dcs.cc'],
             extra_include_paths=[f'{noa_location}/include'],
             extra_cflags=['-Wall -Wextra -Wpedantic -O3 -std=c++17'],
             verbose=False)
```

```python
muons_cuda = load(name='muons_cuda',
             build_directory='./build',
             sources=[f'{noa_location}/docs/pms/muons_dcs.cu'],
             extra_include_paths=[f'{noa_location}/include'],
             extra_cflags=['-Wall -Wextra -Wpedantic -O3 -std=c++17'],
             extra_cuda_cflags=['-std=c++17 --extended-lambda'],
             verbose=False) if torch.cuda.is_available() else None
```

## Energy loss
Differential cross-sections (DCS) are implemented in `<noa/pms/dcs.hh>` for `CPU` and `<noa/pms/dcs.cuh>` for `CUDA` within the namespace `noa::pms::dcs`.

Here, we demonstrate the calculations for muons passing through the standard rock. In `<noa/pms/physics.hh>` you will find:
```cpp
constexpr ParticleMass MUON_MASS = 0.10565839;     // GeV/c^2

constexpr AtomicElement<Scalar> STANDARD_ROCK =
            AtomicElement<Scalar>{
                    22.,       // Atomic mass in g/mol
                    0.1364E-6, // Mean Excitation in GeV
                    11 // Atomic number
    };
```

Let's get a range of kinetic and recoil energies:

```python
kinetic_energies = torch.linspace(1e-3, 1e6, 10000).double()
recoil_energies = 0.0505 * kinetic_energies
```

```python
kinetic_energies_gpu = kinetic_energies.cuda()
recoil_energies_gpu = recoil_energies.cuda()
```

From now on, we shall get ourselves into the namespace:
```cpp
using namespace noa::pms;
```
On `CPU` the DCS computation kernel can be mapped on a tensor via the utility `dcs::vmap`. The user is expected to provide a `result` tensor with the same options as `kinetic_energies` which will get populated with the calculation values:
```cpp
const auto result = torch::zeros_like(kinetic_energies);
```

### Bremsstrahlung

Bremsstrahlung process corresponds to radiation due to deceleration when two charged particles interact. We will consider here the example of muons. At high energies ($E \geq 1$
TeV) this process contributes to about 40% of the average muon energy loss.

* D. Groom et. al. [Muon stopping power and range tables 10 MeV - 100 TeV](https://pdg.lbl.gov/2014/AtomicNuclearProperties/adndt.pdf)

##### Input:
   * $K$ - The projectile initial kinetic energy, in GeV
   * $q$ - The kinetic energy lost to the photon, in GeV
   * $A$ - The mass number of the target atom, in g/mol
   * $Z$ - The charge number of the target atom.
   * $m_\mu$ - muon rest mass $0.10565839$ GeV

##### Output
   The DCS (differential cross-section) in $\text{m}^{2}$/kg:

\begin{equation}
\frac{\text{d}\sigma}{\text{d}q} = \alpha Z(2\frac{m_e}{m_\mu})^{2}(\frac{4}{3}(\frac{1}{\nu} - 1) + \nu)(\Phi_\text{in}(\delta) + Z\Phi_n(\delta) ) \frac{N_a}{A \cdot 10^{-3}}
\end{equation}

Here:

\begin{equation}
E = K + m_\mu
\end{equation}

and:

* $\nu = \frac{q}{E}$ is the fraction of the muon's energy transferred to the photon
* $N_a$ the Avogadro number $6.02214199 \cdot 10^{23} \text{mol}^{-1}$
* $m_e$ electron rest mass $0.51099891003 \cdot 10^{-3}$ GeV
* $\alpha$ fine structure constant $1/137.03599976$

We have the contribution from (screened) nucleus:

\begin{equation}
\Phi_n(\delta) = \ln \left( \frac{BZ^{\frac{-1}{3}}(m_\mu + \delta(D_n\sqrt{e} - 2))}{(m_e + \delta \sqrt{e}BZ^{\frac{-1}{3}})D_n} \right),
\end{equation}

where $D_n = 1.54A^{0.27}$ , $B = 182.7$ ($B = 202.4$ for hygrogen), the exponential $e = 2.7181...$

and the contributions from atomic electrons:
\begin{equation}
\Phi_\text{in}(\delta) = \ln \left( \frac{m_\mu BZ^{-2/3} \sqrt{e}}{(m_e + \delta  BZ^{-2/3} \sqrt{e})(\frac{m_\mu \delta}{m_e^{2}} + \sqrt{e})} \right),
\end{equation}

where $B = 1429$ ($B = 446$ for hygrogen).

Both are evaluated at:

\begin{equation}
\delta = \frac{m_\mu^{2}\nu}{2E(1 - \nu)}
\end{equation}

For `CPU` we have:
```cpp
dcs::vmap(dcs::pumas::bremsstrahlung)(
    result, kinetic_energies, recoil_energies, STANDARD_ROCK, MUON_MASS);
```
For `CUDA` we create the lambda function directly ourselves:
```cpp
dcs::pumas::cuda::vmap_bremsstrahlung(
    result, kinetic_energies, recoil_energies, STANDARD_ROCK, MUON_MASS);
```

```python
brems = muons.bremsstrahlung(kinetic_energies, recoil_energies)
brems[:5]
```

    tensor([3.5293e-04, 3.9395e-06, 4.0777e-06, 4.1341e-06, 4.1650e-06],
           dtype=torch.float64)

```python
brems_gpu = muons_cuda.bremsstrahlung(kinetic_energies_gpu, recoil_energies_gpu);
brems_gpu[:5]
```

    tensor([3.5293e-04, 3.9395e-06, 4.0777e-06, 4.1341e-06, 4.1650e-06],
           device='cuda:0', dtype=torch.float64)

```python
(brems - brems_gpu.cpu()).abs().sum()
```

    tensor(3.2941e-18, dtype=torch.float64)

```python
%timeit muons_cuda.bremsstrahlung(kinetic_energies_gpu, recoil_energies_gpu);
```

    155 µs ± 2.49 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)

```python
%timeit muons.bremsstrahlung(kinetic_energies, recoil_energies);
```

    305 µs ± 4.68 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

### Pair Production

Direct electron pair production is one of the most important muon interaction processes. At TeV muon energies,
the pair production cross section exceeds those of other muon interaction processes over a range of energy transfers
between 100 MeV and 0.1𝐸𝜇. The average energy loss for pair production increases linearly with muon energy, and
in the TeV region this process contributes more than half the total energy loss rate.

To adequately describe the number of pairs produced, the average energy loss and the stochastic energy loss distribution,
the differential cross section behavior over an energy transfer range of 5 MeV ≤ 𝜖 ≤ 0.1 ·𝐸𝜇 must be accurately
reproduced. This is is because the main contribution to the total cross section is given by transferred energies 5 MeV
≤ 𝜖 ≤ 0.01 ·𝐸𝜇, and because the contribution to the average muon energy loss is determined mostly in the region
0.001 · 𝐸𝜇 ≤ 𝜖 ≤ 0.1 ·𝐸𝜇 .

##### Input:
   * Z - The charge number of the target atom.
   * A - The mass number of the target atom.
   * mass - The projectile rest mass, in GeV
   * K - The projectile initial kinetic energy.
   * q - The kinetic energy lost to the photon ($E - E^{\text{'}}$).

##### Output:
    DCS in m^2/kg.

#### Definitions and Applicability

Theory from [Geant4](https://geant4-userdoc.web.cern.ch/UsersGuides/PhysicsReferenceManual/fo/PhysicsReferenceManual.pdf) (Page 151) using [Gaussian quadrature](https://pomax.github.io/bezierinfo/legendre-gauss.html).

The formula for the differential cross section applies when:
* $E_\mu \gg \mu $ $ ( E \geq 2 - 5 GeV)$ and $E_\mu \leq 10^{15} - 10^{17} eV $. If muon energies exceed this limit, the LPM (Landau Pomeranchuk Migdal) effect may become important, depending on the material.
*The muon energy transfer $q$ lies between $q_\text{min} = 4m_e$ and $q_\text{max} = E_\mu - \frac{3\sqrt{e}}{4} M_\mu Z^{1/3}$, although the formal lower limit is $q \gg 2m_e$, and the formal upper limit requires $E_\mu ^\text{'} \gg \mu$.
*$Z \leq 40 - 50$. For higher Z, the Coulomb correction is important but has not been sufficiently studied theoretically.

##### Formulae

\begin{equation}
\frac{\text{d}\sigma}{\text{d}q}(Z, A, E, q) = \frac{4}{3\pi} \frac{Z(Z + \xi(Z))}{A} N_A {(\alpha r_e)}^{2} (\frac{1 - \nu}{q}) \int_{p}[\Phi_e + (m_e/M_\mu)^{2}\Phi_\mu]dp,
\end{equation}

where $\Phi_{e,\mu} = B_{e,\mu} {L_{e,\mu}}^{'}$  and  $\Phi_{e,\mu} = 0 \text{  whenever  } \Phi_{e,\mu}<0$.

$B_e$ and $B_\mu$ do not depend on Z, A, and are given by

\begin{equation}
B_e = [(2 + \rho^{2})(1 + \beta) + \xi(3 + \rho^{2})]\ln(1 + \frac{1}{\xi}) + \frac{1 - \rho^{2} - \beta}{1 + \xi} - (3 + \rho^{2})
\approx \frac{1}{2\xi}[(3 - \rho^{2}) + 2\beta(z + \rho^{2})] \text{ for } \xi \geq 10 ^{3}
\end{equation}

\begin{equation}
B_\mu = [(1 + \rho^{2})(1 + \frac{3\beta}{2}) - \frac{1}{\xi}(1 + 2\beta)(1 - \rho^{2})]\ln(1 + \xi) + \frac{\xi(1 - \rho{2} - \beta)}{1 + \xi} + (1 + 2\beta)(1 - \rho^{2})
\approx \frac{\xi}{2}[(5 - \rho^{2}) + \beta(3 + \rho^{2})] \text{ for } \xi \leq 10^{-3}
\end{equation}

Also,

\begin{equation}
\xi = \frac{\mu^{2}\nu^{2}}{4m^{2}} \frac{(1 - \rho^{2})}{1 - \nu}; \text{ }
\beta = \frac{\nu^{2}}{2(1 - \nu)}
\end{equation}

\begin{equation}
L_e^\text{'} = \ln(\frac{A^{*}Z^{-1/3} \sqrt{(1 + \xi)(1 + Y_e)}}{1 + \frac{2m\sqrt{e}A*Z^{-1/3}(1 + \xi)(1 + Y_\mu)}{E\nu(1 - \rho^{2})}})  - \frac{1}{2}ln([1 + (\frac{3mZ^{1/3}}{2\mu})^{2}(1 + \xi)(1 + Y_e)])
\end{equation}

\begin{equation}
L_\mu ^\text{'} = \ln(\frac{(\frac{M_\mu}{m_e})A^{*}Z^{-1/3} \sqrt{(1 + \frac{1}{\xi})(1 + Y_e)}}{1 + \frac{2m\sqrt{e}A*Z^{-1/3}(1 + \xi)(1 + Y_\mu)}{E\nu(1 - \rho^{2})}})  - ln(\frac{3}{2} Z^{1/3}\sqrt{(1 + \frac{1}{\xi})(1 + Y_\mu)})
\end{equation}

For faster computing, the expressions for $L_{e, \mu} ^\text{'}$ are further algebraically transformed. The function $L_{e, \mu} ^\text{'}$ include the nuclear size correction in comparison with parameterization.

\begin{equation}
Y_e = \frac{5 - \rho^2 + 4\beta(1 + \rho^2)}{2(1 + 3\beta)\ln(3 + 1/\xi)) - \rho^2 - 2\beta(2 - \rho^2)}
\end{equation}

\begin{equation}
Y_\mu = \frac{4 + \rho^2 + 3\beta(1 + \rho^2)}{(1 + \rho^2)(\frac{3}{2} + 2\beta)\ln(3 + \xi) + 1 - \frac{3}{2}\rho^2}
\end{equation}

\begin{equation}
\rho_{max} = 1 - \frac{6M_\mu^2}{E^2(1 - \nu)})\sqrt{1 - \frac{4m_e}{E\nu}}
\end{equation}

```cpp
dcs::vmap(dcs::pumas::pair_production)(
            result, kinetic_energies, recoil_energies, STANDARD_ROCK, MUON_MASS);
```

```python
ppair = muons.pair_production(kinetic_energies, recoil_energies)
ppair[:5]
```

    tensor([0.0000e+00, 6.5366e-06, 7.3699e-06, 7.7919e-06, 8.0572e-06],
           dtype=torch.float64)

### Photonuclear

Input:
   * Z - The charge number of the target atom.
   * A - The mass number of the target atom.
   * ml - The projectile rest mass, in GeV
   * K - The projectile initial kinetic energy.
   * q - The kinetic energy lost to the photon.

Output:
    DCS in m^2/kg.

#### Definitions and Applicability

Theory from [Dutta et. al.](https://arxiv.org/pdf/hep-ph/0012350.pdf) and [Abramowicz et. al.](https://arxiv.org/pdf/hep-ph/9712415.pdf).

The inelastic interaction of muons with nuclei is important at high muon energies (𝐸 ≥ 10 GeV), and at relatively high
energy transfers 𝜈 (𝜈/𝐸 ≥ 10−2). It is especially important for light materials and for the study of detector response
to high energy muons, muon propagation and muon-induced hadronic background. The average energy loss for this
process increases almost linearly with energy, and at TeV muon energies constitutes about 10% of the energy loss rate.

The main contribution to the cross section 𝜎(𝐸, 𝜈) and energy loss comes from the low 𝑄2–region ( 𝑄2 ≪ 1 GeV2).
In this domain, many simplifications can be made in the theoretical consideration of the process in order to obtain
convenient and simple formulae for the cross section. Most widely used are the expressions given by Borog and
Petrukhin [BP75], and Bezrukov and Bugaev [BB81]. Results from these authors agree within 10% for the differential
cross section and within about 5% for the average energy loss, provided the same photonuclear cross section, 𝜎𝛾𝑁, is
used in the calculations.

##### Formulae

The differential cross section can be written in the form

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{4\pi\alpha^2}{Q^4}\frac{F_2(x, Q^2)}{x}(1-y-\frac{Mxy}{2E} + (1 - \frac{2m_l^2}{Q^2})\frac{y^2(1 + \frac{4M^2x^2}{Q^2})}{2(1 + R(x, Q^2))})
\end{equation}

Where $F_2$ - a nucleon structure function

\begin{equation}
F_2(x, Q^2) = \frac{Q^2}{Q^2 + m_0^2} (F_2^P(x, Q^2) + F_2^R(x, Q^2))
\end{equation}

\begin{equation}
F_2 ^{R}(x, Q^{2}) = c_R(t)x_R^{a_R(t)}(1 - x)^{b_R(t)}
= c_R(t)e^{a_R(t)\ln x_R+ b_R(t)\ln(1 - x)}
\end{equation}

\begin{equation}
F_2 ^{P}(x, Q^{2}) = c_P(t)x_P^{a_P(t)}(1 - x)^{b_P(t)}
= c_P(t)e^{\ln(x_P^{a_P(t)}(1 - x)^{b_P(t)})}
= c_P(t)e^{\ln x_P^{a_P(t)}+ \ln(1 - x)^{b_P(t)}}
= c_P(t)e^{a_P(t)\ln x_P+ b_P(t)\ln(1 - x)}
\end{equation}

Take $x$ as $x = \frac{Q^2}{2Mq}$

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{4\pi\alpha^2 F_2}{Q^4} \frac{2Mq}{Q^2}(1 - y - \frac{My}{2E}\frac{Q^2}{2Mq} + (1 - \frac{2m_l^2}{Q^2})\frac{y^2(1 + (\frac{2MQ^2}{2MqQ})^2)}{2(1 + R)})
\end{equation}

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{8\pi\alpha^2F_2Mq}{Q^6}(1 - y - \frac{Q^2y}{4Eq} + (1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + (y\frac{Q}{q})^2}{2(1 + R)})
\end{equation}

Take into account, that $y = \frac{q}{E}$

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{8\pi\alpha^2F_2Mq}{Q^6}(1 - y - \frac{Q^2}{4E^2} + (1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{2(1 + R)})
\end{equation}

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{8\pi\alpha^2F_2Mq}{Q^6}(1 - y  + (1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{2(1 + R)}) - \frac{8\pi\alpha^2F_2Mq}{Q^6}\frac{Q^2}{4E^2}
\end{equation}

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{8\pi\alpha^2F_2Mq}{Q^6}(1 - y  + (1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{2(1 + R)}) - \frac{8\pi\alpha^2F_2Mq}{4Q^4E^2}
\end{equation}

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}x} = \frac{8\pi\alpha^2F_2Mq}{Q^2}((\frac{1 - y  + \frac{1}{2}(1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{1 + R}}{Q^4}) - \frac{1}{4Q^2E^2})
\end{equation}

Let's see on $x = \frac{Q^2}{2Mq}$

\begin{equation}
\frac{\text{d}x}{\text{d}q} = \frac{-Q^2}{2Mq^2}
\end{equation}

And
\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}q}\frac{2Mq^2}{-Q^2} = \frac{8\pi\alpha^2F_2Mq}{Q^2}((\frac{1 - y  + \frac{1}{2}(1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{1 + R}}{Q^4}) - \frac{1}{4Q^2E^2})
\end{equation}

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}q} = \frac{-4\pi\alpha^2F_2}{q}((\frac{1 - y  + \frac{1}{2}(1 - \frac{2m_l^2}{Q^2}) \frac{y^2 + \frac{Q^2}{E^2}}{1 + R}}{Q^4}) - \frac{1}{4Q^2E^2})
\end{equation}

So it's not enough correct equation. We should add some approximation factors. And finally we have:

\begin{equation}
\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}q} = \frac{c_f F_2}{q} (\frac{1-y+\frac{1}{2}(1 - \frac{2m_l^2}{Q^2}) \frac {y^2 + \frac{Q^2}{E^2}}{1 + R}}{Q^4} - \frac{1}{4E^2Q^2})
\end{equation}

with a constant factor $c_f = 2.603096 \cdot 10^{-35}$. Then integrate this equation using Gaussian Quadrature from $Q_\text{min}^2 = \frac{m_l^2y^2}{1 - y} $ to $Q_\text{max}^2 = 2MEy - ((M + m_\pi)^2 - M^2) $

\begin{equation}
\frac{\text{d}\sigma}{\text{d}q} = \int{\frac{\text{d}\sigma(x, Q^2)}{\text{d}Q^2\text{d}q}}\text{d}Q^2
\end{equation}

```cpp
dcs::vmap(dcs::pumas::photonuclear)(
            result, kinetic_energies, recoil_energies, STANDARD_ROCK, MUON_MASS);
```

```python
photonuc = muons.photonuclear(kinetic_energies, recoil_energies)
photonuc[:5]
```

    tensor([0.0000e+00, 2.2912e-06, 2.1304e-06, 2.0719e-06, 2.0427e-06],
           dtype=torch.float64)

### Ionisation

Input:
   * A - The charge number of the target atom.
   * I - The mean excitation of the target atom.
   * Z - The mass number of the target atom.
   * mu - The projectile rest mass, in GeV
   * K - The projectile initial kinetic energy.
   * q - The kinetic energy lost to the photon.

Output:
    DCS in m^2/kg.

### Definitions and Applicability

Theory from Salvat et al., NIMB316 (2013) 144-159, and [Sokalski et al.](https://arxiv.org/pdf/hep-ph/0106011.pdf).

The differential cross section for ionisation is computed following Salvat et al., NIMB316 (2013) 144-159, considering only close interactions for DELs. In addition a radiative correction is applied according toSokalski et al., Phys.Rev.D64 (2001) 074015 (MUM).

\begin{equation}
\frac{\text{d}\sigma}{\text{d}q} = \frac{CEZ}{A(\frac{1}{2P_2} + \frac{P_2W_\text{max}}{E^2W_\text{max} - qP_2})}(1 + \Delta_{e\gamma}),
\end{equation}

where $P_2 = E^2 - M_\mu^2$, $W_{max} = \frac{2m_e P_2}{M_\mu^2 + m_e(m_e + 2E)}$ and $C = 1.535336*10^{-5}$

$\Delta_{e\gamma}$ - Radiative correction

\begin{equation}
\Delta_{e\gamma} = \frac{\alpha}{2\pi}\ln(1 + \frac{2\nu E}{m_e})(\ln(\frac{4E^2(1 - \nu)}{M_\mu^2}) - \ln(1 + \frac{2\nu E}{m_e}))
\end{equation}

If we take into account, that $\nu = \frac{q}{E}$

\begin{equation}
\Delta_{e\gamma} = \frac{\alpha}{2\pi}\ln(1 + \frac{2q}{m_e})(\ln(\frac{4E(E - q)}{M_\mu^2}) - \ln(1 + \frac{2q}{m_e}))
\end{equation}

```cpp
dcs::vmap(dcs::pumas::ionisation)(
            result, kinetic_energies, recoil_energies, STANDARD_ROCK, MUON_MASS);
```

```python
ionis = muons.ionisation(kinetic_energies, recoil_energies)
ionis[:5]
```

    tensor([0.0000e+00, 3.0168e-05, 1.5300e-05, 1.0284e-05, 7.7575e-06],
           dtype=torch.float64)

```python
def plot_recoil_energy(E, sample = 1000):
    recoil_energies = torch.zeros(sample).double()
    kinetic_energies = torch.full((sample,), E).double() #tensor with one energy level

    for i in range(sample):
        recoil_energies[i] = E*(0.05 + i*(1 - 0.05)/sample)

    brems = muons.bremsstrahlung(kinetic_energies, recoil_energies)
    ppair = muons.pair_production(kinetic_energies, recoil_energies)
    photonuc = muons.photonuclear(kinetic_energies, recoil_energies)
    ionis = muons.ionisation(kinetic_energies, recoil_energies)

    fig, ax = plt.subplots()
    ax.plot(recoil_energies.numpy(), brems.numpy(), label = 'bremsstrahlung')
    ax.plot(recoil_energies.numpy(), ppair.numpy(), label = 'pair production')
    ax.plot(recoil_energies.numpy(), photonuc, label = 'photonuclear')
    ax.plot(recoil_energies.numpy(), ionis.numpy(), label = 'ionisation')

    ax.set_xlabel('The kinetic energy lost to the photon, GeV', fontsize = 20)
    ax.set_ylabel('DCS, m^2/kg', fontsize = 20)
    ax.set_title(f"Energy loss at {E} GeV")

    fig.set_figwidth(20)
    fig.set_figheight(10)
    ax.legend()
    plt.show()
```

```python
plot_recoil_energy(1e-3)
```

```python
plot_recoil_energy(10.0)
```

```python
plot_recoil_energy(1e3)
```

```python
plot_recoil_energy(1e6)
```

### Modelling high energy DCS

An interpolation model in $\tau = q/K$ is used for $K \geq 10$ GeV. For Bremsstrahlung we re-write:

\begin{equation}
\frac{K}{E} \frac{\text{d}\sigma}{\text{d}q} = \frac{K \alpha Z}{q}(2\frac{m_e}{m_\mu})^{2}(\frac{4}{3}(1 - \nu) + \nu^2)(\Phi_\text{in}(\delta) + Z\Phi_n(\delta) ) \frac{N_a}{A \cdot 10^{-3}}
\end{equation}

Setting $X=\ln(\tau)$ and $Y=\ln(1-\tau)$, we fit the model:
\begin{equation}
\ln \left( \frac{K}{E} \frac{\text{d}\sigma}{\text{d}q} \right) \approx \sum_{i=0}^{6} a_i X^i + b_1 Y + b_2 Y^2
\end{equation}