# Statistics

Some matplotlib and pandas experiments based on "Head First Statistics" from O'Reilly Media, Inc.

```
from pandas import *
```

## 1. First Impression

### Why learn statistics? (p. 3)

```
profit_data = [("jul", 2.0), ("aug", 2.1), ("sep", 2.2), ("okt", 2.1), ("nov", 2.3), ("dez", 2.4)]
profit_data = DataFrame(profit_data, columns=["month", "profit"])
profit_data
```

      month
      profit

      0
       jul
       2.0

      1
       aug
       2.1

      2
       sep
       2.2

      3
       okt
       2.1

      4
       nov
       2.3

      5
       dez
       2.4

```
profit_fig, profit_axes = plt.subplots(nrows=1, ncols=2)
profit_fig.set_size_inches(15,5)
profit_axes[0].set_ylim(0, 2.5)
profit_data.plot(ax = profit_axes[0], x="month").set_title("Stagnating profits.")
profit_axes[1].set_ylim(2.0, 2.5)
profit_data.plot(ax = profit_axes[1], x="month").set_title("Awesome profits!");
```

The figures use different scaling and offset of the x axes, thus giving a different impression of the profit.

### Manic Mango's Diagrams

```
mango_data = [("sports", 27500, 0.99), ("strategy", 11500, 0.9), ("action", 6000, 0.85), ("shooter", 3500, 0.95), ("others", 1500, 0.85)]
mango_data = DataFrame(mango_data, columns=["type", "sales", "satisfaction"])
mango_data
```

      type
      sales
      satisfaction

      0
         sports
       27500
       0.99

      1
       strategy
       11500
       0.90

      2
         action
        6000
       0.85

      3
        shooter
        3500
       0.95

      4
         others
        1500
       0.85

```
mango_colors=["#F3E761", "#F0F0F0", "#97C2B9", "#E9ABC7", "#CAA882"]
mango_fig, mango_axes = subplots(nrows=1, ncols=2)
mango_fig.set_size_inches(16,7)
_mango_sales = mango_data.sort("sales")
mango_axes[0].pie(_mango_sales.sales, labels=_mango_sales.type + "\n" + _mango_sales.sales.astype(str), startangle=90, colors=mango_colors) # autopct='%1.1f%%'
mango_axes[0].set_title("sales by type");
mango_axes[1].pie(_mango_sales.satisfaction, labels=_mango_sales.type + "\n" + (_mango_sales.satisfaction*100).astype(str) + "%", startangle=90, colors=mango_colors)
mango_axes[1].set_title("satisfied players by type");

```

Pie charts are only useful if the values are not all similar, and if they are all parts of a common whole.

```
mango_regions = [("A", 1000), ("B", 5000), ("C", 7500), ("D", 8000), ("E", 9500)]
mango_regions = DataFrame(mango_regions, columns=["region", "sales"])
mango_regions
```

      region
      sales

      0
       A
       1000

      1
       B
       5000

      2
       C
       7500

      3
       D
       8000

      4
       E
       9500

```
fig, axes = subplots(nrows=2, ncols=2)
fig.set_size_inches(16,11)
mango_regions_index = arange(len(mango_regions))
mango_width=0.8
axes[0,0].bar(mango_regions_index - mango_width/2, mango_regions.sales, width=mango_width, color="#cccccc")
axes[0,0].set_xticks(mango_regions_index)
axes[0,0].set_xticklabels(mango_regions.region);
axes[0,0].set_title("sales by region");
mango_data_index = arange(len(mango_data))

axes[0,1].barh(mango_data_index - mango_width/2, list(reversed(list(100*mango_data.satisfaction))), height=mango_width, color="#cccccc")
axes[0,1].set_yticks(mango_data_index)
axes[0,1].set_yticklabels(list(reversed(list(mango_data.type))))
axes[0,1].set_title("satisfaction by type");

mango_data_sat = list(reversed(list(mango_data.satisfaction*mango_data.sales)))
mango_data_notsat = list(reversed(list((1-mango_data.satisfaction)*mango_data.sales)))
mango_b1 = axes[1,0].barh(mango_data_index, mango_data_sat, height=mango_width/2, color="#8888ff")
mango_b2 = axes[1,0].barh(mango_data_index - mango_width/2, mango_data_notsat, height=mango_width/2, color="#88ff88")
axes[1,0].set_yticks(mango_data_index)
axes[1,0].set_yticklabels(list(reversed(list(mango_data.type))))
axes[1,0].set_title("satisfaction by type (absolute)");
axes[1,0].legend([mango_b1, mango_b2], ["satisfied", "unsatisfied"], loc="lower right");

mango_b3 = axes[1,1].barh(mango_data_index - mango_width/2, mango_data_sat, height=mango_width, color="#8888ff")
mango_b4 = axes[1,1].barh(mango_data_index - mango_width/2, mango_data_notsat, left=mango_data_sat, height=mango_width, color="#88ff88")
axes[1,1].set_yticks(mango_data_index)
axes[1,1].set_yticklabels(list(reversed(list(mango_data.type))))
axes[1,1].set_title("satisfaction by type (absolute)");
axes[1,1].legend([mango_b3, mango_b4], ["satisfied", "unsatisfied"], loc="lower right");

```

### Histogram

```
score_data = [(0,199,5), (200,399,29), (400,599, 56), (600, 799, 17), (800,999,3)]
score_data = DataFrame(score_data, columns=("points_from", "points_to", "frequency"))
score_data
```

      points_from
      points_to
      frequency

      0
         0
       199
        5

      1
       200
       399
       29

      2
       400
       599
       56

      3
       600
       799
       17

      4
       800
       999
        3

```
score_fig, score_axes = subplots(nrows=1, ncols=2)
score_fig.set_size_inches(16,4)
score_axes[0].bar(score_data.points_from, score_data.frequency, width=score_data.points_to-score_data.points_from+1, color="#cccccc");
score_axes[0].set_title("Frequency of games by points");
score_axes[0].set_xlabel("Points");
score_axes[0].set_ylabel("Frequency");
score_list = (score_data.points_to+score_data.points_from)/2
score_scores = []
for score in score_list:
    score_scores.extend(repeat(score, score_data[(score_data.points_from <= score) & (score_data.points_to >= score)].frequency))
score_axes[1].hist(score_scores, bins=sorted(set(list(score_data.points_from) + list(score_data.points_to+1))), color="#cccccc")
score_axes[1].set_title("Frequency of games by points");
score_axes[1].set_xlabel("Points");
score_axes[1].set_ylabel("Frequency");
```

```
gamelen_data = [(0,1,4300), (1,3,6900), (3,5,4900), (5,10,2000), (10,24,2100)]
gamelen_data = DataFrame(gamelen_data, columns=("len_from", "len_to", "frequency"))
gamelen_data
```

      len_from
      len_to
      frequency

      0
        0
        1
       4300

      1
        1
        3
       6900

      2
        3
        5
       4900

      3
        5
       10
       2000

      4
       10
       24
       2100

```
gamelen_fig, gamelen_axis = subplots(nrows=1, ncols=2)
gamelen_fig.set_size_inches(16,4)
gamelen_axis[0].bar(gamelen_data.len_from, gamelen_data.frequency, width=gamelen_data.len_to-gamelen_data.len_from, color="#cccccc");
gamelen_axis[0].set_title("Frequency of games by length");
gamelen_axis[0].set_xlabel("Length/h");
gamelen_axis[0].set_ylabel("Frequency");
gamelen_axis[0].spines['right'].set_color('none')
gamelen_axis[0].spines['top'].set_color('none')
gamelen_axis[0].xaxis.set_ticks_position('bottom')
gamelen_axis[0].yaxis.set_ticks_position('left')
gamelen_list = (gamelen_data.len_to+gamelen_data.len_from)/2
gamelen_list2 = []
for gamelen in gamelen_list:
    gamelen_list2.extend(repeat(gamelen, gamelen_data[(gamelen_data.len_from <= gamelen) & (gamelen_data.len_to >= gamelen)].frequency))
gamelen_axis[1].hist(gamelen_list2, bins=sorted(set(list(gamelen_data.len_from) + list(gamelen_data.len_to))), color="#cccccc", normed=True)
gamelen_axis[1].set_title("Frequency of games by length (normalized)");
gamelen_axis[1].set_xlabel("Length/h");
gamelen_axis[1].set_ylabel("Frequency");
gamelen_axis[1].spines['right'].set_color('none')
gamelen_axis[1].spines['top'].set_color('none')
gamelen_axis[1].xaxis.set_ticks_position('bottom')
gamelen_axis[1].yaxis.set_ticks_position('left')

```

```
gamelen_fig, gamelen_axis = subplots(nrows=1, ncols=1)
gamelen_fig.set_size_inches(7,4)
gamelen_axis.plot([0] + list(gamelen_data.len_to), [0] + list(gamelen_data.frequency.cumsum()));
gamelen_axis.set_title("Cumulative sum of games by length");
gamelen_axis.set_xlabel("Length/h");
gamelen_axis.set_ylabel("Cumulative sum");
gamelen_axis.spines['right'].set_color('none')
gamelen_axis.spines['top'].set_color('none')
gamelen_axis.xaxis.set_ticks_position('bottom')
gamelen_axis.yaxis.set_ticks_position('left')
gamelen_axis.set_xlim(0)
gamelen_axis.set_ylim(0, 22500);
gamelen_axis.yaxis.set_major_locator(MultipleLocator(2500.0))
```

## 2. Averages

### Arithmetic Mean

```
power_ages = DataFrame([19,20,20,20,21], columns=["age"])
power_ages
```

      age

      0
       19

      1
       20

      2
       20

      3
       20

      4
       21

Arithmetic mean:
\begin{equation}
\textrm{mean}(x_1,\...,x_n)=\overline x={1\over n}\sum_{i=1}^n x_i
\end{equation}

```
mean(power_ages.age)
```

    20.0

```
power_agefreqs = DataFrame([(19, 1), (20, 3), (21, 1)], columns=["age", "frequency"])
power_agefreqs
```

      age
      frequency

      0
       19
       1

      1
       20
       3

      2
       21
       1

```
def avg(frame):
    return sum(1. * frame.age * frame.frequency) / sum(frame.frequency)
avg(power_agefreqs)
```

    20.0

```
kungfu_agefreqs = DataFrame([(19, 3), (20, 5), (21, 3), (136, 1), (138, 1)], columns=["age", "frequency"])
kungfu_agefreqs
```

      age
      frequency

      0
        19
       3

      1
        20
       5

      2
        21
       3

      3
       136
       1

      4
       138
       1

```
avg(kungfu_agefreqs)
```

    38.0

```
agefreq_fig, agefreq_axes = subplots(nrows=1, ncols=2)
agefreq_fig.set_size_inches(16,4)
agefreq_axes[0].set_title("Age in Power-Workout");
agefreq_axes[0].bar(power_agefreqs.age, power_agefreqs.frequency, width=1, color="#cccccc");
agefreq_axes[0].set_xlim(18,24)
agefreq_axes[0].set_ylim(0, 5)
agefreq_axes[1].set_title("Age in Kung-Fu");
agefreq_axes[1].get_xaxis().set_visible(False)
agefreq_axes[1].get_yaxis().set_visible(False)
agefreq_axes[1].spines['bottom'].set_visible(False)
agefreq_axes[1].spines['top'].set_visible(False)
# Subdivide the right axes vertically: 1 row, 4 columns, select 3rd and 4th.
agefreq1 = agefreq_fig.add_subplot(143)
agefreq2 = agefreq_fig.add_subplot(144)
agefreq1.set_ylim(0, 10)
agefreq2.set_ylim(0, 10)
agefreq1.set_xlim(18,24)
agefreq2.set_xlim(135,140)
agefreq1.bar(kungfu_agefreqs.age, kungfu_agefreqs.frequency, width=1, color="#cccccc");
agefreq2.bar(kungfu_agefreqs.age, kungfu_agefreqs.frequency, width=1, color="#cccccc");
agefreq1.spines['right'].set_visible(False)
agefreq1.get_yaxis().tick_left()
agefreq2.spines['left'].set_visible(False)
agefreq2.get_yaxis().tick_right()
agefreq2.tick_params(labelright='off')
```

### Median

Note that the mean may not occur in the data.  The median has the advantage that it exists as a data point:

```
kungfu_ages = []
for index, data in kungfu_agefreqs.iterrows():
    kungfu_ages.extend(repeat(data["age"], data["frequency"]))
mean(kungfu_ages), median(kungfu_ages)
```

    (38.0, 20.0)

However, if the number of data points is even, the media is the value halfway between the two values around the middle:

```
median([10,20])
```

    15.0

```
skew_fig, skew_axes = subplots(nrows=1, ncols=3)
skew_fig.set_size_inches(17,4)
skew1 = randn(1000000)
skew2 = exp(1+0.4*skew1)
skew3 = -skew2
skew_axes[0].set_title("Unskewed normal, mean = median")
skew_axes[0].hist(skew1, bins=30, normed=True, color="#cccccc");
skew_axes[0].plot(mean(skew1), 0.2, 'go');
skew_axes[0].plot(median(skew1), 0.15, 'rD');
skew_axes[0].set_xlim(-4,4)
skew_axes[1].set_title("Right-skewed log-normal, median < mean");
skew_axes[1].hist(skew2, bins=30, normed=True, color="#cccccc");
skew_axes[1].plot(mean(skew2), 0.2, 'go');
skew_axes[1].plot(median(skew2), 0.15, 'rD');
skew_axes[1].set_xlim(0,10)
skew_axes[2].set_title("Left-skewed (mirrored) log-normal, median > mean");
skew_axes[2].hist(skew3, bins=30, normed=True, color="#cccccc");
skew_axes[2].plot(mean(skew3), 0.2, 'go');
skew_axes[2].plot(median(skew3), 0.15, 'rD');
skew_axes[2].set_xlim(-10,0);

```

### Modal/Mode

```
swim_agefreqs = DataFrame([(1, 3), (2, 4), (3, 2), (31, 2), (32, 4), (33, 3)], columns=["age", "frequency"])
swim_agefreqs
```

      age
      frequency

      0
        1
       3

      1
        2
       4

      2
        3
       2

      3
       31
       2

      4
       32
       4

      5
       33
       3

The mean and median are both misleading in this case:

```
swim_ages = []
for index, data in swim_agefreqs.iterrows():
    swim_ages.extend(repeat(data["age"], data["frequency"]))
mean(swim_ages), median(swim_ages)
```

    (17.0, 17.0)

```
import scipy.stats
val, cnt = scipy.stats.mstats.mode(array(swim_ages))
val
```

    array([ 2.])

Note that 32 is another mode.

## 3. Variance and Deviation

### Span, Interquantildistance, Percentils

```
player1 = DataFrame([(7,1),(8,1),(9,2),(10,3),(11,2),(12,1),(13,1)], columns=("points", "frequency"))
player2 = DataFrame([(7,1),(9,2),(10,5),(11,2),(13,1)], columns=("points", "frequency"))
player3 = DataFrame([(3,2),(6,1),(7,2),(10,3),(11,1),(13,1),(30,1)], columns=("points", "frequency"))
player1["player"] = 1
player2["player"] = 2
player3["player"] = 3
player_stats = pandas.concat([player1, player2, player3])
player_stats
```

      points
      frequency
      player

      0
        7
       1
       1

      1
        8
       1
       1

      2
        9
       2
       1

      3
       10
       3
       1

      4
       11
       2
       1

      5
       12
       1
       1

      6
       13
       1
       1

      0
        7
       1
       2

      1
        9
       2
       2

      2
       10
       5
       2

      3
       11
       2
       2

      4
       13
       1
       2

      0
        3
       2
       3

      1
        6
       1
       3

      2
        7
       2
       3

      3
       10
       3
       3

      4
       11
       1
       3

      5
       13
       1
       3

      6
       30
       1
       3

```
player_data = []

def get_stats(pstats):
    player, stats = pstats
    vals = []
    for index, data in stats.iterrows():
        vals.extend(repeat(data["points"], data["frequency"]))
    vals = array(vals)
    player_data.append(vals)
    return player, mean(vals), median(vals), scipy.stats.mstats.mode(vals)[0][0], amin(vals), amax(vals), percentile(vals,25), percentile(vals,75)

player_avgs = DataFrame(map(get_stats, player_stats.groupby(player_stats.player)),
                        columns=("player", "mean", "median", "mode", "minpoints", "maxpoints", "q1","q3"))
player_avgs
```

      player
      mean
      median
      mode
      minpoints
      maxpoints
      q1
      q3

      0
       1
       10
       10
       10
       7
       13
       9.0
       11.0

      1
       2
       10
       10
       10
       7
       13
       9.5
       10.5

      2
       3
       10
       10
       10
       3
       30
       6.5
       10.5

```
player_avgs['span'] = player_avgs.maxpoints - player_avgs.minpoints
player_avgs['interq'] = player_avgs.q3 - player_avgs.q1
player_avgs[['player', 'span', 'interq']]
```

      player
      span
      interq

      0
       1
        6
       2

      1
       2
        6
       1

      2
       3
       27
       4

```
player_fig, player_axes = subplots()
player_fig.set_size_inches(7,4)
player_axes.boxplot(player_data, vert=False, whis=1.5);
player_axes.set_title("Results of basketball players");
player_axes.set_xlim(0,33)
```

### Variance, Standard Deviation

\begin{equation}
V(x) = {1 \over n} \sum_i(x_i-\overline x)^2
\end{equation}

\begin{equation}
\sigma(x) = \sqrt{V(x)}
\end{equation}

A discussion for possible reasons why we prefer the mean square error can be found here: http://stats.stackexchange.com/questions/118/why-square-the-difference-instead-of-taking-the-absolute-value-in-standard-devia

```
player_avgs["var"] = var(player_data, axis=1)
player_avgs["std"] = std(player_data, axis=1)
player_avgs[["player", "var", "std"]]
```

      player
      var
      std

      0
       1
        2.727273
       1.651446

      1
       2
        2.000000
       1.414214

      2
       3
       49.272727
       7.019453

### Standard Score (z-score)

\begin{equation}
z(x) = \frac{ x - \overline{x} }{ \sigma(x) }
\end{equation}

```
scipy.stats.mstats.zscore(player_data[0]);
player_points = player_avgs["mean"] + player_avgs["std"]
array([scipy.stats.zmap([10, 11, player_points[0], 12, 30],
 player_data[0]), scipy.stats.zmap([10, 11, 12, 30],
 player_data[1]), scipy.stats.zmap([10, 11, 12, 30], player_data[2])])
```

    array([ array([  0.        ,   0.60553007,   1.        ,   1.21106014,  12.11060142]),
           array([  0.        ,   0.70710678,   1.41421356,  14.14213562]),
           array([ 0.        ,  0.14246123,  0.28492247,  2.84922466])], dtype=object)

```

```


# 03 - Full-Waveform Inversion (FWI)

This notebook is the third in a series of tutorial highlighting various aspects of seismic inversion based on Devito operators. In this second example we aim to highlight the core ideas behind seismic inversion, where we create an image of the subsurface from field recorded data. This tutorial follows on the modelling tutorial and will reuse the modelling and velocity model.

## Inversion requirement

Seismic inversion relies on two known parameters:

- **Field data** - or also called **recorded data**. This is a shot record corresponding to the true velocity model. In practice this data is acquired as described in the first tutorial. In order to simplify this tutorial we will fake field data by modelling it with the true velocity model.

- **Initial velocity model**. This is a velocity model that has been obtained by processing the field data. This model is a rough and very smooth estimate of the velocity as an initial estimate for the inversion. This is a necessary requirement for any optimization (method).

## Inversion computational setup

In this tutorial, we will introduce the gradient operator. This operator corresponds to the imaging condition introduced in the previous tutorial with some minor modifications that are defined by the objective function (also referred to in the tutorial series as the *functional*, *f*) and its gradient, *g*. We will define these two terms in the tutorial too.

## Notes on the operators

As we have already described the creation of a forward modelling operator, we will only call a wrapper function here. This wrapper already contains all the necessary operators for seismic modeling, imaging and inversion. Operators introduced for the first time in this tutorial will be properly described.

```python
import numpy as np
%matplotlib inline

from devito import configuration
configuration['log-level'] = 'WARNING'
```

## Computational considerations

As we will see, FWI is computationally extremely demanding, even more than RTM. To keep this tutorial as lightwight as possible we therefore again use a very small demonstration model. We also define here a few parameters for the final example runs that can be changed to modify the overall runtime of the tutorial.

```python
nshots = 9  # Number of shots to create gradient from
nreceivers = 101  # Number of receiver locations per shot
fwi_iterations = 5  # Number of outer FWI iterations
```

# True and smooth velocity models

We will use a very simple model domain, consisting of a circle within a 2D domain. We will again use the "true" model to generate our synthetic shot data and use a "smooth" model as our initial guess. In this case the smooth model is very smooth indeed - it is simply a constant background velocity without any features.

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import demo_model, plot_velocity, plot_perturbation

# Define true and initial model
shape = (101, 101)  # Number of grid point (nx, nz)
spacing = (10., 10.)  # Grid spacing in m. The domain size is now 1km by 1km
origin = (0., 0.)  # Need origin to define relative source and receiver locations

model = demo_model('circle-isotropic', vp_circle=3.0, vp_background=2.5,
                    origin=origin, shape=shape, spacing=spacing, nbl=40)

model0 = demo_model('circle-isotropic', vp_circle=2.5, vp_background=2.5,
                     origin=origin, shape=shape, spacing=spacing, nbl=40,
                     grid = model.grid)

plot_velocity(model)
plot_velocity(model0)
plot_perturbation(model0, model)
```

```python
assert model.grid == model0.grid
assert model.vp.grid == model0.vp.grid
```

## Acquisition geometry

In this tutorial, we will use the easiest case for inversion, namely a transmission experiment. The sources are located on one side of the model and the receivers on the other side. This allows to record most of the information necessary for inversion, as reflections usually lead to poor inversion results.

```python
#NBVAL_IGNORE_OUTPUT
# Define acquisition geometry: source
from examples.seismic import AcquisitionGeometry

t0 = 0.
tn = 1000.
f0 = 0.010
# First, position source centrally in all dimensions, then set depth
src_coordinates = np.empty((1, 2))
src_coordinates[0, :] = np.array(model.domain_size) * .5
src_coordinates[0, 0] = 20.  # Depth is 20m

# Define acquisition geometry: receivers

# Initialize receivers for synthetic and imaging data
rec_coordinates = np.empty((nreceivers, 2))
rec_coordinates[:, 1] = np.linspace(0, model.domain_size[0], num=nreceivers)
rec_coordinates[:, 0] = 980.

# Geometry

geometry = AcquisitionGeometry(model, rec_coordinates, src_coordinates, t0, tn, f0=f0, src_type='Ricker')
# We can plot the time signature to see the wavelet
geometry.src.show()
```

```python
#NBVAL_IGNORE_OUTPUT
# Plot acquisition geometry
plot_velocity(model, source=geometry.src_positions,
              receiver=geometry.rec_positions[::4, :])
```

## True and smooth data

We can generate shot records for the true and smoothed initial velocity models, since the difference between them will again form the basis of our imaging procedure.

```python
# Compute synthetic data with forward operator
from examples.seismic.acoustic import AcousticWaveSolver

solver = AcousticWaveSolver(model, geometry, space_order=4)
true_d, _, _ = solver.forward(vp=model.vp)
```

```python
# Compute initial data with forward operator
smooth_d, _, _ = solver.forward(vp=model0.vp)
```

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import plot_shotrecord

# Plot shot record for true and smooth velocity model and the difference
plot_shotrecord(true_d.data, model, t0, tn)
plot_shotrecord(smooth_d.data, model, t0, tn)
plot_shotrecord(smooth_d.data - true_d.data, model, t0, tn)
```

# Full-Waveform Inversion

## Formulation

Full-waveform inversion (FWI) aims to invert an accurate model of the discrete wave velocity, $\mathbf{c}$, or equivalently the square slowness of the wave, $\mathbf{m} = \frac{1}{\mathbf{c}^2}$, from a given set of measurements of the pressure wavefield $\mathbf{u}$. This can be expressed as the following optimization problem [1, 2]:

\begin{aligned}
	\mathop{\hbox{minimize}}_{\mathbf{m}} \Phi_s(\mathbf{m})&=\frac{1}{2}\left\lVert\mathbf{P}_r
	\mathbf{u} - \mathbf{d}\right\rVert_2^2 \\
	\mathbf{u} &= \mathbf{A}(\mathbf{m})^{-1} \mathbf{P}_s^T \mathbf{q}_s,
\end{aligned}

where $\mathbf{P}_r$ is the sampling operator at the receiver locations, $\mathbf{P}_s^T$ is the injection operator at the source locations, $\mathbf{A}(\mathbf{m})$ is the operator representing the discretized wave equation matrix, $\mathbf{u}$ is the discrete synthetic pressure wavefield, $\mathbf{q}_s$ is the corresponding pressure source and $\mathbf{d}$ is the measured data. It is worth noting that $\mathbf{m}$ is the unknown in this formulation and that multiple implementations of the wave equation operator $\mathbf{A}(\mathbf{m})$ are possible.

We have already defined a concrete solver scheme for $\mathbf{A}(\mathbf{m})$ in the first tutorial, including appropriate implementations of the sampling operator $\mathbf{P}_r$ and source term $\mathbf{q}_s$.

To solve this optimization problem using a gradient-based method, we use the
adjoint-state method to evaluate the gradient $\nabla\Phi_s(\mathbf{m})$:

\begin{align}
 \nabla\Phi_s(\mathbf{m})=\sum_{\mathbf{t} =1}^{n_t}\mathbf{u}[\mathbf{t}] \mathbf{v}_{tt}[\mathbf{t}] =\mathbf{J}^T\delta\mathbf{d}_s,
\end{align}

where $n_t$ is the number of computational time steps, $\delta\mathbf{d}_s = \left(\mathbf{P}_r \mathbf{u} - \mathbf{d} \right)$ is the data residual (difference between the measured data and the modelled data), $\mathbf{J}$ is the Jacobian operator and $\mathbf{v}_{tt}$ is the second-order time derivative of the adjoint wavefield solving:

\begin{align}
 \mathbf{A}^T(\mathbf{m}) \mathbf{v} = \mathbf{P}_r^T \delta\mathbf{d}.
\end{align}

We see that the gradient of the FWI function is the previously defined imaging condition with an extra second-order time derivative. We will therefore reuse the operators defined previously inside a Devito wrapper.

## FWI gradient operator

To compute a single gradient $\nabla\Phi_s(\mathbf{m})$ in our optimization workflow we again use `solver.forward` to compute the entire forward wavefield $\mathbf{u}$ and a similar pre-defined gradient operator to compute the adjoint wavefield `v`. The gradient operator provided by our `solver` utility also computes the correlation between the wavefields, allowing us to encode a similar procedure to the previous imaging tutorial as our gradient calculation:

- Simulate the forward wavefield with the background velocity model to get the synthetic data and save the full wavefield $\mathbf{u}$
- Compute the data residual
- Back-propagate the data residual and compute on the fly the gradient contribution at each time step.

This procedure is applied to multiple source positions and summed to obtain a gradient image of the subsurface. We again prepare the source locations for each shot and visualize them, before defining a single gradient computation over a number of shots as a single function.

```python
#NBVAL_IGNORE_OUTPUT

# Prepare the varying source locations sources
source_locations = np.empty((nshots, 2), dtype=np.float32)
source_locations[:, 0] = 30.
source_locations[:, 1] = np.linspace(0., 1000, num=nshots)

plot_velocity(model, source=source_locations)
```

```python
from devito import Eq, Operator

# Computes the residual between observed and synthetic data into the residual
def compute_residual(residual, dobs, dsyn):
    if residual.grid.distributor.is_parallel:
        # If we run with MPI, we have to compute the residual via an operator
        # First make sure we can take the difference and that receivers are at the
        # same position
        assert np.allclose(dobs.coordinates.data[:], dsyn.coordinates.data)
        assert np.allclose(residual.coordinates.data[:], dsyn.coordinates.data)
        # Create a difference operator
        diff_eq = Eq(residual, dsyn.subs({dsyn.dimensions[-1]: residual.dimensions[-1]}) -
                               dobs.subs({dobs.dimensions[-1]: residual.dimensions[-1]}))
        Operator(diff_eq)()
    else:
        # A simple data difference is enough in serial
        residual.data[:] = dsyn.data[:] - dobs.data[:]

    return residual
```

```python
# Create FWI gradient kernel
from devito import Function, TimeFunction, norm
from examples.seismic import Receiver

import scipy
def fwi_gradient(vp_in):
    # Create symbols to hold the gradient
    grad = Function(name="grad", grid=model.grid)
    # Create placeholders for the data residual and data
    residual = Receiver(name='residual', grid=model.grid,
                        time_range=geometry.time_axis,
                        coordinates=geometry.rec_positions)
    d_obs = Receiver(name='d_obs', grid=model.grid,
                     time_range=geometry.time_axis,
                     coordinates=geometry.rec_positions)
    d_syn = Receiver(name='d_syn', grid=model.grid,
                     time_range=geometry.time_axis,
                     coordinates=geometry.rec_positions)
    objective = 0.
    for i in range(nshots):
        # Update source location
        geometry.src_positions[0, :] = source_locations[i, :]

        # Generate synthetic data from true model
        _, _, _ = solver.forward(vp=model.vp, rec=d_obs)

        # Compute smooth data and full forward wavefield u0
        _, u0, _ = solver.forward(vp=vp_in, save=True, rec=d_syn)

        # Compute gradient from data residual and update objective function
        compute_residual(residual, d_obs, d_syn)

        objective += .5*norm(residual)**2
        solver.gradient(rec=residual, u=u0, vp=vp_in, grad=grad)

    return objective, grad
```

Having defined our FWI gradient procedure we can compute the initial iteration from our starting model. This allows us to visualize the gradient alongside the model perturbation and the effect of the gradient update on the model.

```python
# Compute gradient of initial model
ff, update = fwi_gradient(model0.vp)
assert np.isclose(ff, 57283, rtol=1e0)
```

```python
#NBVAL_IGNORE_OUTPUT
from devito import mmax
from examples.seismic import plot_image

# Plot the FWI gradient
plot_image(-update.data, vmin=-1e4, vmax=1e4, cmap="jet")

# Plot the difference between the true and initial model.
# This is not known in practice as only the initial model is provided.
plot_image(model0.vp.data - model.vp.data, vmin=-1e-1, vmax=1e-1, cmap="jet")

# Show what the update does to the model
alpha = .5 / mmax(update)
plot_image(model0.vp.data + alpha*update.data, vmin=2.5, vmax=3.0, cmap="jet")
```

We see that the gradient and the true perturbation have the same sign, therefore, with an appropriate scaling factor, we will update the model in the correct direction.

```python
from devito import Min, Max
# Define bounding box constraints on the solution.
def update_with_box(vp, alpha, dm, vmin=2.0, vmax=3.5):
    """
    Apply gradient update in-place to vp with box constraint

    Notes:
    ------
    For more advanced algorithm, one will need to gather the non-distributed
    velocity array to apply constrains and such.
    """
    update = vp + alpha * dm
    update_eq = Eq(vp, Max(Min(update, vmax), vmin))
    Operator(update_eq)()
```

```python
#NBVAL_SKIP

from devito import mmax

# Run FWI with gradient descent
history = np.zeros((fwi_iterations, 1))
for i in range(0, fwi_iterations):
    # Compute the functional value and gradient for the current
    # model estimate
    phi, direction = fwi_gradient(model0.vp)

    # Store the history of the functional values
    history[i] = phi

    # Artificial Step length for gradient descent
    # In practice this would be replaced by a Linesearch (Wolfe, ...)
    # that would guarantee functional decrease Phi(m-alpha g) <= epsilon Phi(m)
    # where epsilon is a minimum decrease constant
    alpha = .05 / mmax(direction)

    # Update the model estimate and enforce minimum/maximum values
    update_with_box(model0.vp , alpha , direction)

    # Log the progress made
    print('Objective value is %f at iteration %d' % (phi, i+1))
```

    Objective value is 39292.605833 at iteration 1
    Objective value is 24506.628229 at iteration 2
    Objective value is 14386.573665 at iteration 3
    Objective value is 7907.616850 at iteration 4
    Objective value is 3960.106497 at iteration 5

```python
#NBVAL_IGNORE_OUTPUT

# Plot inverted velocity model
plot_velocity(model0)
```

```python
#NBVAL_SKIP
import matplotlib.pyplot as plt

# Plot objective function decrease
plt.figure()
plt.loglog(history)
plt.xlabel('Iteration number')
plt.ylabel('Misift value Phi')
plt.title('Convergence')
plt.show()
```

## References

[1] _Virieux, J. and Operto, S.: An overview of full-waveform inversion in exploration geophysics, GEOPHYSICS, 74, WCC1–WCC26, doi:10.1190/1.3238367, http://library.seg.org/doi/abs/10.1190/1.3238367, 2009._

[2] _Haber, E., Chung, M., and Herrmann, F. J.: An effective method for parameter estimation with PDE constraints with multiple right hand sides, SIAM Journal on Optimization, 22, http://dx.doi.org/10.1137/11081126X, 2012._

This notebook is part of the tutorial "Optimised Symbolic Finite Difference Computation with Devito" presented at the Intel® HPC Developer Conference 2017.


# Hello, I am Richard Otis from Prof. Zi-Kui Liu's research group.

# Tonight I present pycalphad, a free and open-source library for CALPHAD computations. It works on Mac, Windows, and Linux.

# Phase Equilibria Computation
## Al-Fe (M.Seiersten et al., 1991)

```python
%matplotlib inline
import matplotlib.pyplot as plt
from pycalphad import Database, binplot
db_alfe = Database('alfe_sei.TDB')
my_phases_alfe = ['LIQUID', 'B2_BCC', 'FCC_A1', 'HCP_A3', 'AL5FE2', 'AL2FE', 'AL13FE4', 'AL5FE4']

fig = plt.figure(figsize=(9,6))
pdens = [{'B2_BCC': 20000}, 2000]
%time binplot(db_alfe, ['AL', 'FE', 'VA'] , my_phases_alfe, 'X(AL)', 300, 2000, steps=100, \
              pdens=pdens, ax=fig.gca(), mode='numexpr')
plt.show()
```

## Energy Surfaces of Al-Ni (N. Dupin et al., 2001)

```python
%matplotlib inline
from pycalphad import energy_surf, Database
from pycalphad.plot.utils import phase_legend
import numpy as np
import matplotlib.pyplot as plt
db_alni = Database('NI_AL_DUPIN_2001.TDB')
my_phases_alni = ['LIQUID', 'FCC_L12', 'BCC_B2', 'AL3NI5', 'AL3NI2', 'AL3NI1']

fig = plt.figure(figsize=(9,6))
ax = fig.gca()
%time nrg_df = energy_surf(db_alni, ['AL', 'NI', 'VA'], my_phases_alni, T=1000.0)

legend_handles, colorlist = phase_legend(my_phases_alni)
group_df = nrg_df.groupby('Phase')
for phase, phase_df in group_df:
    ax = phase_df.plot(kind='scatter', x='X(AL)', y='GM', ax=ax, color=colorlist[phase.upper()], s=1, xlim=(0,1), ylim=(-11e4, -3e4))
ax = ax.legend(handles=legend_handles, loc='center left', bbox_to_anchor=(1, 0.6))
plt.title('Al-Ni Gibbs Energy at 1000 K', fontsize=20)
plt.show()
```

```python
nrg_df
```

      GM
      Phase
      T
      X(AL)
      X(NI)
      Y(AL3NI1,0,AL)
      Y(AL3NI1,1,NI)
      Y(AL3NI2,0,AL)
      Y(AL3NI2,1,AL)
      Y(AL3NI2,1,NI)
      ...
      Y(BCC_B2,1,NI)
      Y(BCC_B2,1,VA)
      Y(BCC_B2,2,VA)
      Y(FCC_L12,0,AL)
      Y(FCC_L12,0,NI)
      Y(FCC_L12,1,AL)
      Y(FCC_L12,1,NI)
      Y(FCC_L12,2,VA)
      Y(LIQUID,0,AL)
      Y(LIQUID,0,NI)

      0
      -78820.96
      AL3NI1
      1000
      0.750000
      0.250000
      1
      1
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      1
      -42406.27
      AL3NI2
      1000
      0.833333
      0.166667
      NaN
      NaN
      1
      1.000000
      0.000000
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      2
      -31245.28
      AL3NI2
      1000
      1.000000
      0.000000
      NaN
      NaN
      1
      1.000000
      0.000000
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      3
      -96138.68
      AL3NI2
      1000
      0.500000
      0.500000
      NaN
      NaN
      1
      0.000000
      1.000000
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      4
      -95724.16
      AL3NI2
      1000
      0.600000
      0.400000
      NaN
      NaN
      1
      0.000000
      1.000000
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      5
      -78441.97
      AL3NI2
      1000
      0.699594
      0.300406
      NaN
      NaN
      1
      0.405684
      0.594316
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      6
      -83711.23
      AL3NI2
      1000
      0.666261
      0.333739
      NaN
      NaN
      1
      0.306762
      0.693238
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      7
      -52357.6
      AL3NI2
      1000
      0.856900
      0.143100
      NaN
      NaN
      1
      0.811368
      0.188632
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      8
      -53907.05
      AL3NI2
      1000
      0.852927
      0.147073
      NaN
      NaN
      1
      0.784209
      0.215791
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      9
      -94605.5
      AL3NI2
      1000
      0.593467
      0.406533
      NaN
      NaN
      1
      0.072421
      0.927579
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      10
      -70472.13
      AL3NI2
      1000
      0.773870
      0.226130
      NaN
      NaN
      1
      0.513128
      0.486872
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      11
      -78455.38
      AL3NI2
      1000
      0.650357
      0.349643
      NaN
      NaN
      1
      0.417235
      0.582765
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      12
      -88702.35
      AL3NI2
      1000
      0.584153
      0.415847
      NaN
      NaN
      1
      0.208809
      0.791191
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      13
      -45764.46
      AL3NI2
      1000
      0.836801
      0.163199
      NaN
      NaN
      1
      0.949757
      0.050243
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      14
      -85180.4
      AL3NI2
      1000
      0.604807
      0.395193
      NaN
      NaN
      1
      0.282256
      0.717744
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      15
      -86633.83
      AL3NI2
      1000
      0.673257
      0.326743
      NaN
      NaN
      1
      0.216330
      0.783670
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      16
      -55505.88
      AL3NI2
      1000
      0.868494
      0.131506
      NaN
      NaN
      1
      0.722351
      0.277649
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      17
      -59649.33
      AL3NI2
      1000
      0.849447
      0.150553
      NaN
      NaN
      1
      0.654595
      0.345405
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      18
      -67201.6
      AL3NI2
      1000
      0.740282
      0.259718
      NaN
      NaN
      1
      0.615686
      0.384314
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      19
      -79900.42
      AL3NI2
      1000
      0.670879
      0.329121
      NaN
      NaN
      1
      0.389007
      0.610993
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      20
      -83727.73
      AL3NI2
      1000
      0.649001
      0.350999
      NaN
      NaN
      1
      0.314701
      0.685299
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      21
      -93610.27
      AL3NI2
      1000
      0.577309
      0.422691
      NaN
      NaN
      1
      0.105298
      0.894702
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      22
      -48055.96
      AL3NI2
      1000
      0.863983
      0.136017
      NaN
      NaN
      1
      0.887928
      0.112072
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      23
      -45576.63
      AL3NI2
      1000
      0.881772
      0.118228
      NaN
      NaN
      1
      0.915031
      0.084969
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      24
      -91739.02
      AL3NI2
      1000
      0.612108
      0.387892
      NaN
      NaN
      1
      0.140717
      0.859283
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      25
      -67033.34
      AL3NI2
      1000
      0.711129
      0.288871
      NaN
      NaN
      1
      0.622735
      0.377265
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      26
      -70011.83
      AL3NI2
      1000
      0.769478
      0.230522
      NaN
      NaN
      1
      0.531118
      0.468882
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      27
      -79884.3
      AL3NI2
      1000
      0.702313
      0.297687
      NaN
      NaN
      1
      0.368791
      0.631209
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      28
      -39460.32
      AL3NI2
      1000
      0.930066
      0.069934
      NaN
      NaN
      1
      0.967530
      0.032470
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      29
      -93850.99
      AL3NI2
      1000
      0.602490
      0.397510
      NaN
      NaN
      1
      0.087437
      0.912563
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      46018
      -45828.58
      LIQUID
      1000
      0.050061
      0.949939
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.050061
      0.949939

      46019
      -84954.36
      LIQUID
      1000
      0.620452
      0.379548
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.620452
      0.379548

      46020
      -87462.71
      LIQUID
      1000
      0.503316
      0.496684
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.503316
      0.496684

      46021
      -76348.12
      LIQUID
      1000
      0.290815
      0.709185
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.290815
      0.709185

      46022
      -51225.52
      LIQUID
      1000
      0.086494
      0.913506
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.086494
      0.913506

      46023
      -57128.03
      LIQUID
      1000
      0.902927
      0.097073
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.902927
      0.097073

      46024
      -72423.77
      LIQUID
      1000
      0.252032
      0.747968
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.252032
      0.747968

      46025
      -83824.5
      LIQUID
      1000
      0.642015
      0.357985
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.642015
      0.357985

      46026
      -86248.2
      LIQUID
      1000
      0.588732
      0.411268
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.588732
      0.411268

      46027
      -79458.53
      LIQUID
      1000
      0.325945
      0.674055
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.325945
      0.674055

      46028
      -71009.83
      LIQUID
      1000
      0.239095
      0.760905
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.239095
      0.760905

      46029
      -81115.29
      LIQUID
      1000
      0.683374
      0.316626
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.683374
      0.316626

      46030
      -38616.7
      LIQUID
      1000
      0.004592
      0.995408
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.004592
      0.995408

      46031
      -63184.88
      LIQUID
      1000
      0.857187
      0.142813
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.857187
      0.142813

      46032
      -72116.98
      LIQUID
      1000
      0.781606
      0.218394
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.781606
      0.218394

      46033
      -87480.46
      LIQUID
      1000
      0.507160
      0.492840
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.507160
      0.492840

      46034
      -81341.86
      LIQUID
      1000
      0.350256
      0.649744
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.350256
      0.649744

      46035
      -47413.72
      LIQUID
      1000
      0.970456
      0.029544
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.970456
      0.029544

      46036
      -60905.8
      LIQUID
      1000
      0.156344
      0.843656
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.156344
      0.843656

      46037
      -87482.29
      LIQUID
      1000
      0.522727
      0.477273
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.522727
      0.477273

      46038
      -76699.85
      LIQUID
      1000
      0.294551
      0.705449
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.294551
      0.705449

      46039
      -85232.33
      LIQUID
      1000
      0.614469
      0.385531
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.614469
      0.385531

      46040
      -86957.71
      LIQUID
      1000
      0.466996
      0.533004
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.466996
      0.533004

      46041
      -59245.8
      LIQUID
      1000
      0.887299
      0.112701
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.887299
      0.112701

      46042
      -58699.88
      LIQUID
      1000
      0.139826
      0.860174
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.139826
      0.860174

      46043
      -71762.15
      LIQUID
      1000
      0.784881
      0.215119
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.784881
      0.215119

      46044
      -51452.58
      LIQUID
      1000
      0.943187
      0.056813
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.943187
      0.056813

      46045
      -64214.46
      LIQUID
      1000
      0.181979
      0.818021
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.181979
      0.818021

      46046
      -45140.79
      LIQUID
      1000
      0.045538
      0.954462
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.045538
      0.954462

      46047
      -58334.11
      LIQUID
      1000
      0.894071
      0.105929
      NaN
      NaN
      NaN
      NaN
      NaN
      ...
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      NaN
      0.894071
      0.105929

46048 rows × 28 columns

```python
nrg_df.to_excel('alni_energies.xlsx')
```

## Al-Zn (S. Mey et al., 1993)

Next, we show values from Mey et al.'s 1993 assessment of the Al-Zn system. We have not done any fitting here; we are just showing how arbitrary values for the parameters may be specified and easily used to plot the phase diagram. Note the optional *parameters* keyword argument specified for each call to **Model**. This feature is what makes it easy to perform arbitrary variable substitutions in our thermodynamic models.

```python
from pycalphad import Model, Database, binplot

db_alzn = Database('alzn_noparams.tdb')

assessed_parameters = {
    'LIQALZN0H': 10465.3,
    'LIQALZN0S': 3.39259,
    'FCCALZN0H': 7297.5,
    'FCCALZN0S': -0.47512,
    'FCCALZN1H': 6612.9,
    'FCCALZN1S': 4.5911,
    'FCCALZN2H': -3097.2,
    'FCCALZN2S': -3.306635,
    'HCPALZN0H': 18821.0,
    'HCPALZN0S': 8.95255,
    'HCPALZN3H': -702.8,
    'HCPALZN3S': 0.0
    }

# Initialize phase models
assessed_models = {
    'LIQUID': Model(db_alzn, ['AL', 'ZN'], 'LIQUID', parameters=assessed_parameters),
    'FCC_A1': Model(db_alzn, ['AL', 'ZN', 'VA'], 'FCC_A1', parameters=assessed_parameters),
    'HCP_A3': Model(db_alzn, ['AL', 'ZN', 'VA'], 'HCP_A3', parameters=assessed_parameters)
    }

fig = plt.figure(figsize=(9,6))
ax = binplot(db_alzn, ['AL', 'ZN', 'VA'] , ['LIQUID', 'FCC_A1', 'HCP_A3'], \
                   'X(ZN)', 300, 1000, ax=plt.gca(), model=assessed_models)
ax = ax.set_title('Al-Zn', fontsize=20)
```

```python
blank_parameters = {
    'LIQALZN0H': 0.0,
    'LIQALZN0S': 0.0,
    'FCCALZN0H': 0.0,
    'FCCALZN0S': 0.0,
    'FCCALZN1H': 0.0,
    'FCCALZN1S': 0.0,
    'FCCALZN2H': 0.0,
    'FCCALZN2S': 0.0,
    'HCPALZN0H': 0.0,
    'HCPALZN0S': 0.0,
    'HCPALZN3H': 0.0,
    'HCPALZN3S': 0.0
    }

models = {
    'LIQUID': Model(db_alzn, ['AL', 'ZN'], 'LIQUID', parameters=blank_parameters),
    'FCC_A1': Model(db_alzn, ['AL', 'ZN', 'VA'], 'FCC_A1', parameters=blank_parameters),
    'HCP_A3': Model(db_alzn, ['AL', 'ZN', 'VA'], 'HCP_A3', parameters=blank_parameters)
    }

fig = plt.figure(figsize=(9,6))
ax = binplot(db_alzn, ['AL', 'ZN', 'VA'] , ['LIQUID', 'FCC_A1', 'HCP_A3'], \
             'X(ZN)', 300, 1000, ax=plt.gca(), model=models)
ax = ax.set_title('Al-Zn with endmembers only', fontsize=20)
```

# Prototyping New Models (Advanced)

What if we have a new kind of functional form for the Gibbs energy we would like to experiment with?

```python
from sympy import Piecewise, log
from tinydb import where
import pycalphad.variables as v

class MyMagneticModel(Model):
    def __init__(self, db, comps, phase, **kwargs):
        super().__init__(db, comps, phase, **kwargs)
        self.models['mag'] = self.my_magnetic_energy(db.phases[phase], db.search)

    def my_magnetic_energy(self, phase, param_search):
        """
        Return the energy from magnetic ordering in symbolic form.
        The implemented model is the Inden-Hillert-Jarl formulation.
        The approach follows from the background section of W. Xiong, 2011.

        Note that this is a demonstration model; use pycalphad's builtin
        implementation as a reference.
        """
        site_ratio_normalization = self._site_ratio_normalization(phase)
        # define basic variables
        afm_factor = phase.model_hints['ihj_magnetic_afm_factor']

        bm_param_query = (
            (where('phase_name') == phase.name) & \
            (where('parameter_type') == 'BMAGN') & \
            (where('constituent_array').test(self._array_validity))
        )
        tc_param_query = (
            (where('phase_name') == phase.name) & \
            (where('parameter_type') == 'TC') & \
            (where('constituent_array').test(self._array_validity))
        )

        mean_magnetic_moment = \
            self.redlich_kister_sum(phase, param_search, bm_param_query)
        beta = Piecewise(
            (mean_magnetic_moment, mean_magnetic_moment > 0),
            (mean_magnetic_moment/afm_factor, mean_magnetic_moment <= 0)
            )

        curie_temp = \
            self.redlich_kister_sum(phase, param_search, tc_param_query)
        tc = Piecewise(
            (curie_temp, curie_temp > 0),
            (curie_temp/afm_factor, curie_temp <= 0)
            )

        tau = Piecewise(
            (v.T / tc, tc > 0),
            (10000., tc == 0) # tau is 'infinity'
            )

        # define model parameters
        p = phase.model_hints['ihj_magnetic_structure_factor']
        A = 518/1125 + (11692/15975)*(1/p - 1)
        # factor when tau < 1
        sub_tau = 1 - (1/A) * ((79/(140*p))*(tau**(-1)) + (474/497)*(1/p - 1) \
            * ((tau**3)/6 + (tau**9)/135 + (tau**15)/600)
                              )
        # factor when tau >= 1
        super_tau = -(1/A) * ((tau**-5)/10 + (tau**-15)/315 + (tau**-25)/1500)

        expr_cond_pairs = [(sub_tau, tau < 1),
                           (super_tau, tau >= 1)
                          ]

        g_term = Piecewise(*expr_cond_pairs)

        return 1e6 * v.R * v.T * log(beta+1) * \
            g_term / site_ratio_normalization
```

```python
%matplotlib inline
import matplotlib.pyplot as plt
from pycalphad import Database, binplot
db_alfe = Database('alfe_sei.TDB')
my_phases_alfe = ['LIQUID', 'B2_BCC', 'FCC_A1', 'HCP_A3', 'AL5FE2', 'AL2FE', 'AL13FE4', 'AL5FE4']

models = [{'FCC_A1': MyMagneticModel}, Model]

fig = plt.figure(figsize=(9,6))
pdens = [{'B2_BCC': 20000}, 2000]
%time binplot(db_alfe, ['AL', 'FE', 'VA'] , my_phases_alfe, 'X(AL)', 300, 2000, steps=100, \
              pdens=pdens, ax=fig.gca(), model=models, mode='numexpr')
plt.title('Al-Fe with toy magnetic model', fontsize=20)
plt.show()
```

## Experimental Ternary support (if time)
## CALPHAD modeling (assessment) will be covered tomorrow morning

# Source code available at https://github.com/richardotis/pycalphad
# Version 0.1 available on PyPI: "`pip install pycalphad`"
# Version 0.2 coming soon, with new performance improvements and features
# Please contact me if you would like to collaborate! Thank you!
#
#
# Richard Otis
# rao140@psu.edu

```python

```


# Symbolic computation of the asymptotic expansion for a disk with constant $\varepsilon_{\mathsf{c}}$

The codes below are associated to the article:

- C. Carvalho and Z. Moitier, _Asymptotics for metamaterial cavities and their effect on scattering_ [[arXiv](https://arxiv.org/abs/2010.07583), [HAL](https://hal.archives-ouvertes.fr/hal-02965993)]

## Zoïs Moitier, Camille Carvalho (2021)

_Karlsruhe Institute of Technology, Germany_

_University of California Merced, USA_

```python
import sympy as sy
from IPython.display import display

from core import *
```

```python
order = 5
```

```python
h, η = sy.symbols("h η", real=True, positive=True)
σ = sy.symbols("σ", real=True)
```

```python
def dσ_neg(expr):
    return expr.diff(σ) + η * expr

def dσ_pos(expr):
    return expr.diff(σ) - expr / η
```

```python
iε = inv_ε_expan((η,), σ, h, order)
f_neg, 𝓛_neg = op_disk(iε, dσ_neg, σ, h, order)
f_pos, 𝓛_pos = op_disk(1, dσ_pos, σ, h, order)
```

```python
Λ_ = sy.symbols("Λ", real=True)
Λ = (1 - η ** (-2)).factor() + Λ_ * h
P = 0 * σ + 1
Q = 0 * σ + 1
```

```python
ldφ = [0 for _ in range(order)]
ldψ = [0 for _ in range(order)]
```

```python
display((1 - η ** (-2)).factor())
```

$\displaystyle \frac{\left(η - 1\right) \left(η + 1\right)}{η^{2}}$

```python
for n in range(1, order):
    # Compute solution for φ
    eq_φ = ((η ** 2) * (𝓛_neg.subs(f_neg(σ), P).doit() - Λ * P)).expand()
    Sφ = eq_φ.coeff(h, n)
    ldφ[n], sol_φ = solve_exp(η, σ, Sφ)

    # Compute solution for ψ
    eq_ψ = (-(𝓛_pos.subs(f_pos(σ), Q).doit() - Λ * Q)).expand()
    Sψ = eq_ψ.coeff(h, n)
    ldψ[n], sol_ψ = solve_exp(-1 / η, σ, Sψ)

    # Compute Λ
    cφ = (-dσ_neg(sol_φ).subs(σ, 0) / η ** 2).expand()
    cψ = dσ_pos(sol_ψ).subs(σ, 0)

    Λn = sy.solve(sy.Eq(cφ, cψ), Λ_)[0].factor()
    # print(sy.horner(Λn.expand()))
    display(Λn)

    # Subs
    Λ = Λ.subs(Λ_, Λn) + Λ_ * h ** (n + 1)
    P += sol_φ.subs(Λ_, Λn).factor() * h ** n
    Q += sol_ψ.subs(Λ_, Λn).factor() * h ** n
```

$\displaystyle - \frac{\left(η - 1\right)^{2} \left(η + 1\right)^{2}}{η^{3}}$

$\displaystyle - \frac{\left(η - 1\right) \left(η + 1\right) \left(η^{4} + 1\right) \left(η^{4} - η^{2} + 1\right)}{2 η^{6}}$

$\displaystyle - \frac{\left(η - 1\right)^{2} \left(η + 1\right)^{2} \left(10 η^{12} - 2 η^{10} + 7 η^{8} - 2 η^{6} + 7 η^{4} - 2 η^{2} + 10\right)}{8 η^{9}}$

$\displaystyle - \frac{\left(η - 1\right)^{3} \left(η + 1\right)^{3} \left(44 η^{16} + 14 η^{14} + 41 η^{12} + 18 η^{10} + 42 η^{8} + 18 η^{6} + 41 η^{4} + 14 η^{2} + 44\right)}{8 η^{12}}$


## Numerical method

Here we will solve few problems by numerical method using Lagrangian grid which deformed and moved together with system.
We will use Wilkins Method.

### Numerical implementation of boundary conditions

#### Basics of the system
Lets consider tho slabs collision problem. Suppose slabs have areas $AB$ and $CD$ on $x$ axes:

we call $A$, $B$, $C$ and $D$ *side bars*

We mark here and following formulas in varible $v^n_i$ top index $n$ as time and $i$ as index of *inner bars* in the grid.

*Bar* has always integer index $k$ and *node* has index $k\pm1/2$

For *inner bars* we will use:

for speeds:
$\begin{align}
v^{n+1/2}_{i} = v^{n-1/2}_{i} + 2 \Delta t \frac{\Sigma^n_{i+1/2} - \Sigma^n_{i-1/2}}{\phi^n_{i+1/2} + \phi^n_{i - 1/2}}
\end{align}$

for positions:
$\begin{align}
x^{n+1}_i = x^{n}_i + \Delta t v^{n+1/2}_i \\
\end{align}$

where $\Delta t$ is time step; $\phi^n_{i+1/2} = \rho^n_{i+1/2}(x^n_{i+1} - x^n_{i})$ where $\rho$ is density.

We suppose that all values of all variables are know for moment $t^n$ and before it.

#### Collision

Slabs collide at moment $t^n$, we merge $B$ and $C$ and set following: $\Sigma_B = \Sigma_C$, where $\Sigma$ is tension.

Boundary conditions for the *side bar*s we find at moment $t^n$ as follows. Let's add a *mock node* $i - 1/2$ with *bars* $i - 1$ and $i$ which we will use only for computations.
This *mock node* we place to the left corner of the right stab.

And we suposse following for the *mock node*:

$\begin{align}
\Sigma^n_{i-1/2} = 0 ; \phi^n_{i-1/2} = 0
\end{align}$

```python

```


```python
%run header.ipynb
```

```python
from sympy import pi
from sympy.physics.units import meter, foot
```

```python
# dictionary that holds all values.
# if already defined (such as by another notebook) then don't override
if "values" not in vars():
    values={"d": 12.1 * meter}

Formula.set_global_values(values)
```

```python
d = Formula("d")
area_formula = Formula(value_name="A", formula="pi*(d**2/4)", sigfigs=4)
```

The area of a circle with diameter {{ d.get_inline() }} is:

{{ area_formula.get_display(oneline=False) }}

```python

```


```python
import sympy
sympy.init_printing()
```

# Diagramas de bloques

Es conveniente analizar sistemas elaborados como una combinación de sistemas más sencillos.

A continuación los principales tipos de combinación.

## Concatenación / Serie

Cuando dos sistemas LTI $H_1$ y $H_2$ están interconectados de manera que la salida de $H_1$ es la entrada de $H_2$. Esta es la conexión en **Serie** o **Concatenación de sistemas**.

Suponga que:
- La respuesta impulsional de $H_1$ es $h_1(t)$.
- La respuesta impulsional de $H_2$ es $h_2(t)$.
- La concatenación se excita con una señal $x(t)$.

La salida de $H_1$ es entonces $x(t)*h_1(t)$. La salida de la concatenación de sistemas es:

\begin{equation}
y(t) =  x(t) * h_1(t) * h_2(t)
\end{equation}

En el dominio de Laplace, la convolución anterior se transforma en:

\begin{equation}
Y(s) = \underbrace{H_1(s) \cdot H_2(s)}_{H(s)} \cdot X(s)
\end{equation}

Así, los dos sistemas concatenados tienen una equivalencia $H(s) = H_1(s) \cdot H_2(s)$.

Si se tienen $N$ sistemas conectados en serie, el modelo equivalente está dado por la multiplicación de las respectivas funciones de transferencia.

\begin{equation}
H(s) = \prod_{n=1}^{N} H_n(s)
\end{equation}

**Ejemplo**

Al concatenar filtros de bajo orden se obtiene un filtro de orden más elevado.

Suponga las siguientes funciones de transferencia para filtros de orden 2.

\begin{equation}
G(s) = \frac{1}{LC s^2 + RC s + 1}
\end{equation}

Si se concatenan $N$ filtros con acoples de impedancia, se obtiene:

\begin{equation}
H_N(s) = \left( \frac{1}{LC s^2 + RC s + 1} \right)^N
\end{equation}

## Paralelo

Cuando dos sistemas LTI comparten la misma entrada y sus salidas se combinan, están conectados en paralelo.

Cuando dos sistemas LTI $H_1$ y $H_2$ están interconectados de manera que tienen la misma entrada y sus salidas se combinan, esta es la conexión en **Paralelo** .

Suponga que:
- La respuesta impulsional de $H_1$ es $h_1(t)$.
- La respuesta impulsional de $H_2$ es $h_2(t)$.
- La conexión paralela se excita con una señal $x(t)$.

La salida de $H_1$ es entonces $x(t)*h_1(t)$. La salida de $H_2$ es entonces $x(t)*h_2(t)$. La salida de la conexión paralela es:

\begin{equation}
y(t) = h_1(t) * x(t) + h_2(t) * x(t)
\end{equation}

En el dominio de Laplace, la combinación anterior se transforma en:

\begin{equation}
Y(s) = \underbrace{\left( H_1(s) + H_2(s)  \right)}_{H(s)} \cdot X(s)
\end{equation}

Así, los dos sistemas en paralelo tienen una equivalencia $H(s) = H_1(s) + H_2(s)$.

Si se tienen $N$ sistemas conectados en paralelo, el modelo equivalente está dado por la suma de las respectivas funciones de transferencia.

\begin{equation}
H(s) = \sum_{n=1}^{N} H_n(s)
\end{equation}

## Realimentación

En algunos casos, parte de la señal de salida de $H_1$ es devuelta a su entrada a través de otro sistema $H_2$. Esta conexión es llamada **realimentación**.

La salida $y(t)$ está dada por:

\begin{equation}
y(t) = \underbrace{(x(t) + \overbrace{y(t) * h_2(t)}^{b(t)})}_{e(t)} * h_1(t)
\end{equation}

En el dominio de Laplace, la expresión se transforma en:

\begin{equation}
y(t) = (x(t) + y(t) * h_2(t)) * h_1(t)
\end{equation}

\begin{equation}
Y(s) = (X(s) + Y(s) \cdot H_2(s)) \cdot H_1(s)
\end{equation}

\begin{equation}
Y(s) = X(s) \cdot H_1(s) + Y(s) \cdot H_2(s) \cdot H_1(s)
\end{equation}

\begin{equation}
Y(s) -  Y(s) \cdot H_2(s) \cdot H_1(s) = X(s) \cdot H_1(s)
\end{equation}

\begin{equation}
Y(s) \cdot (1 -  H_2(s) \cdot H_1(s)) = X(s) \cdot H_1(s)
\end{equation}

\begin{equation}
Y(s) = \frac{H_1(s)}{1 - H_1(s) \cdot H_2(s)} \cdot X(s)
\end{equation}

La equivalencia es

\begin{equation}
H(s) = \frac{H_1(s)}{1 - H_1(s) \cdot H_2(s)}
\end{equation}

Si la realimentación fuera negativa, la equivalencia sería:

\begin{equation}
H(s) = \frac{H_1(s)}{1 + H_1(s) \cdot H_2(s)}
\end{equation}

## Álgebra de bloques

Cuando hay varios subsistemas interactuando entre sí, se modelan por separado y se identifican las conexiones entre ellos. Para encontrar equivalencias deben reducirse el sistema a una sola expresión con las siguientes normas.

**Ejemplo**

```python

```

## Sympy reduciendo bloques

Con la ayuda de `Sympy` podrá reducir los diagramas de bloques.

**ejemplo**

$$ V2 = V1\cdot G1 \Rightarrow V2 - V1 \cdot G1 = 0$$
$$ V4 = V3\cdot G2 \Rightarrow V4 - V3 \cdot G2 = 0$$
$$ C = V5\cdot G3 \Rightarrow C - V5 \cdot G3 = 0$$
$$ V6 = V4\cdot H1 \Rightarrow V6 - V4 \cdot H1 = 0$$

```python
(R, V1, V2, V3, V4, V5, V6, V7, V8, C,
 G1, G2, G3, H1, H2, H3) = sympy.symbols('R, V1, V2, V3, V4, V5, V6, V7, V8, C,'
                                         'G1, G2, G3, H1, H2, H3')
incognitas = V1, V2, V3, V4, V5, V6, V7, V8, C
```

```python
ecs = [# bloques
       V2 - G1*V1,
       V4 - G2*V3,
       C - G3*V5,
       V6 - H1*V4,
       V7 - H2*V4,
       V8 - H3*C,
       # sumas
       V1 - (R - V6),
       V3 - (V2 - V7),
       V5 - (V4 + V3 - V8),
       ]
ecs
```

```python
solucion = sympy.solve(ecs, incognitas)
solucion
```

De todas las incognitas halladas solo interesa la salida $C$.

```python
solucion[C]
```

Esta expresión puede factorizarse.

```python
solucion[C].factor()
```

Observe que se ha encontrado una expresión para $C$ y el bloque equivalente debe ser $\frac{C}{R}$, es decir:

```python
solucion[C].factor()/R
```

## Ejercicio para entregar

1. Reducir el siguiente diagrama.

2. Realice el ejercicio en papel y verifique su resultado con `Sympy`.
3. Documente el proceso y la comparación y suba un archivo pdf a la plataforma.

```python

```


NOTE: Videos and graphs do not display immediately on all devices.  If this is the case for you, click the "Cell" menu option, and select "Run All".  If you still do not see the videos/graphs, try a different browser.  If that also does not work, please let your instructor know by email.

# Improper Integrals

In this notebook, we extend the concept of a definite integral, $\displaystyle \int_a^b f(x) \ dx$ (defined on a finite interval $[a,b]$), to the case where either
1. the interval is infinite, or
2. the integrand approaches infinity at one or more points on the interval $[a,b]$.

Such integrals are called improper integrals.

### 1. Motivational Example Leading to an Improper Integral

Suppose that a chemical is produced at a rate of $e^{-t}$ moles/second.  At time $t=0$, this chemical is produced at a rate of 1 mole/second, after which the rate of production decreases exponentially.

The amount of chemical produced up to time $t=3$ seconds is represented by the definite integral
\begin{align}
\int_0^{10} e^{-t} \ dt
& = \left. \left( - e^{-t} \right) \right|_0^{3} \\
& = -e^{-3} - \left( -e^{0} \right) \\
& = -e^{-3} + 1 = 1 - e^{-3} \approx 0.95 \ \mbox{moles}.
\end{align}

What if the process runs forever?  Will the amount of chemical produced be infinite?  Or finite?  If the amount is finite, how much?  It seems that we need to evaluate the following improper integral:
$$\int_0^{\infty} e^{-t}\ dt.$$

But how do we do that?

### 2. Handling an Improper Integral of the Form $\displaystyle \int_a^{\infty} f(x) \ dx$

In the following video, we will show how to deal with an improper integral of the form $\displaystyle \int_a^{\infty} f(x) \ dx$, and work through a couple of examples, including the example introduced above.

```python
from IPython.display import YouTubeVideo
YouTubeVideo('xOiCve1hQCE')
```

### 3. Handling Improper Integrals of the Form $\displaystyle \int_{-\infty}^{a} f(x) \ dx$ and $\displaystyle \int_{-\infty}^{\infty} f(x) \ dx$

Integrals of the form $\displaystyle \int_{-\infty}^a f(x) \ dx$ and $\displaystyle \int_{-\infty}^{\infty} f(x) \ dx$ also are improper integrals.

In the following video, we show how to deal with these improper integrals and work through a few additional examples.

```python
from IPython.display import YouTubeVideo
YouTubeVideo('FOK__rFUqA4')
```

### 4. Other Improper Integrals

The improper integrals considered above have at least one infinite limit of integration.  These improper integrals are all referred to as improper integrals of the first kind.

Improper integrals of the second kind are those where the integrand approaches infinity at least once on the interval of integration.  An example is $\displaystyle \int_{-1}^{2} \frac{1}{x^2} \ dx$.  Recall that the antiderivative of $1/x^2$ is $-1/x$.  If we were to proceed carelessly, we might write the following:
\begin{align}
\int_{-1}^2 \frac{1}{x^2} \ dx
& = \left. \left( - \frac{1}{x} \right) \right|_{-1}^2 \\
& = -\frac{1}{2} - \left( - \frac{1}{-1} \right) \\
& = -\frac{1}{2} + \frac{1}{-1} = -\frac{1}{2} - 1 = -\frac{3}{2}.
\end{align}
We obtained a negative value.  But surely this cannot be right, since the function $\displaystyle f(x) = \frac{1}{x^2}$ is a positive function (it lies entirely above the $x$-axis), as shown in the following graph:

```python
%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
x = np.linspace(-4, 4, 1000)
plt.plot(x, 1/(x*x), label=r'$f(x) = \frac{1}{x^2}$')
plt.xlim(-4,4)
plt.ylim(0,100)
plt.xlabel('x', fontsize=15)
plt.ylabel('y', fontsize=15)
plt.legend(loc='best', fontsize=15)
plt.show()
```

Since $\displaystyle f(x) = \frac{1}{x^2}$ is a positive function, any definite integral must evaluate to a positive number.

The reason for the strange result (obtaining a negative number) lies in the fact that $\displaystyle \int_{-1}^{2} \frac{1}{x^2} \ dx$ is not a definite integral, but an improper integral whose integrand is undefined at $0$.

In the following video, we show how to handle improper integrals of the second kind, and work through a couple of examples, including the example introduced above.

```python
from IPython.display import YouTubeVideo
YouTubeVideo('Hb8AIb92zxs')
```

### 5. Summary

If an improper integral has an integrand for which we can find an antiderivative (through any method at our disposal), then we can introduce limits to take care of any improprieties, and carry the calculation of the improper integral to completion.

#### Definitions of Improper Integrals of the First Kind

For an improper integral of the first kind, the impropriety is at least one infinite limit of integration.

- If $\displaystyle \int_a^b f(x) \ dx$ exists for every number $b \ge a$, then $$\int_a^{\infty} f(x) \ dx = \lim_{b \to \infty} \int_a^b f(x) \ dx$$ provided this limit exits (as a finite number).
- If $\displaystyle \int_a^b f(x) \ dx$ exists for every number $a \le b$, then $$\int_{-\infty}^{b} f(x) \ dx = \lim_{a \to -\infty} \int_a^b f(x) \ dx$$ provided this limit exits (as a finite number).
- An improper integral is called convergent if the corresponding limit exists; otherwise it is called divergent.
- If both $\displaystyle \int_{-\infty}^c f(x) \ dx$ and $\displaystyle \int_c^{\infty} f(x) \ dx$ are convergent, then $$\int_{-\infty}^{\infty} f(x) \ dx = \int_{-\infty}^c f(x) \ dx + \int_c^{\infty} f(x) \ dx$$ where $c$ is any real number.

#### Definition of Improper Integrals of the Second Kind

For an improper integral of the second kind, the impropriety is the integrand that goes to infinity somewhere in the interval of integration.

- If $f$ is continuous on $[a,c)$ and is discontinuous at $c$, then $$\int_a^c f(x) \ dx = \lim_{b \to c^-} \int_a^b f(x) \ dx$$ provided this limits exists (as a finite number).
- If $f$ is continuous on $(c,b]$ and is discontinuous at $c$, then $$\int_c^b f(x) \ dx = \lim_{a \to c^+} \int_a^b f(x) \ dx$$ provided this limits exists (as a finite number).
- If $f(x)$ has a discontinuity at $c$, where $a < c < b$, and both $\displaystyle \int_a^c f(x) \ dx$ and $\displaystyle \int_c^b f(x) \ dx$ are convergent, then $$\int_a^b f(x) \ dx = \int_a^c f(x) \ dx + \int_c^b f(x) \ dx.$$

### 6. Further Study

Please refer to Section 5.8 in the textbook for additional treatment of this topic.

### 7. Don't Forget

Don't forget to return to eClass to complete the pre-class quiz.

```python

```


```python
%load_ext autoreload
%autoreload 2

%matplotlib inline
```

## Basic setup

Create anaconda environment

```bash
conda create -n ml python=3.7.4 jupyter
```
Install fastai library

```bash
conda install -c pytorch -c fastai fastai
```

# Special types of matrices

Identity matrix $I_{n} \in \mathbb{R}^{n \times n}$ is a matrix which does not change other matrix after multiplication. These kind of matrices contain ones on main diagonal and zero everywhere else
$$\begin{align} I_{n} &= \begin{pmatrix}
           1, 0, \dots, 0 \\
           0, 1, \dots, 0 \\
           \vdots \\
           0, 0, \dots, 1 \\
         \end{pmatrix}
  \end{align}$$

or we can define it with property $\forall a \in \mathbb{R}^{1 \times n}$ holds $aI_{n} = a$ or $\forall a \in \mathbb{R}^{n \times 1}$ holds $I_{n}a =a$

```python
import numpy as np
```

```python
I = np.identity(4)
```

```python
I
```

    array([[1., 0., 0., 0.],
           [0., 1., 0., 0.],
           [0., 0., 1., 0.],
           [0., 0., 0., 1.]])

```python
A = np.random.random(size=(4, 5))
B = np.random.random(size=(6, 4))
```

```python
A
```

    array([[0.08180764, 0.13233535, 0.50190654, 0.94090784, 0.23366253],
           [0.65533627, 0.89712887, 0.06492877, 0.35594497, 0.26581298],
           [0.67741343, 0.10612077, 0.76232457, 0.07079163, 0.9766638 ],
           [0.78773022, 0.2730207 , 0.96932571, 0.89913794, 0.16359123]])

```python
B
```

    array([[0.59625685, 0.21514496, 0.22955471, 0.59314179],
           [0.75225961, 0.04911663, 0.82749338, 0.11810451],
           [0.35309953, 0.21511622, 0.36902663, 0.83973281],
           [0.34193753, 0.75561623, 0.1429398 , 0.79237438],
           [0.88373539, 0.16254689, 0.46374738, 0.66354648],
           [0.05174316, 0.53756595, 0.1425859 , 0.76509128]])

```python
I @ A
```

    array([[0.08180764, 0.13233535, 0.50190654, 0.94090784, 0.23366253],
           [0.65533627, 0.89712887, 0.06492877, 0.35594497, 0.26581298],
           [0.67741343, 0.10612077, 0.76232457, 0.07079163, 0.9766638 ],
           [0.78773022, 0.2730207 , 0.96932571, 0.89913794, 0.16359123]])

```python
B @ I
```

    array([[0.59625685, 0.21514496, 0.22955471, 0.59314179],
           [0.75225961, 0.04911663, 0.82749338, 0.11810451],
           [0.35309953, 0.21511622, 0.36902663, 0.83973281],
           [0.34193753, 0.75561623, 0.1429398 , 0.79237438],
           [0.88373539, 0.16254689, 0.46374738, 0.66354648],
           [0.05174316, 0.53756595, 0.1425859 , 0.76509128]])

Inverse matrix of $A \in \mathbb{R}^{n \times n}$, is the matrix $A^{-1} \in \mathbb{R}^{n \times n}$ for which $A^{-1}A = I$

```python
A = np.random.random(size=(8, 8))
A
```

    array([[0.81890884, 0.78825974, 0.87439842, 0.94759203, 0.5461928 ,
            0.13407198, 0.96945139, 0.9188376 ],
           [0.87818807, 0.56492261, 0.16592348, 0.42610048, 0.74483667,
            0.51242792, 0.68651341, 0.32431056],
           [0.96894838, 0.61132545, 0.16142936, 0.84775417, 0.47020687,
            0.15378486, 0.10566062, 0.65394892],
           [0.1161688 , 0.12452046, 0.11319474, 0.90558559, 0.26105095,
            0.5540081 , 0.36826309, 0.57233497],
           [0.49719397, 0.87143557, 0.82084637, 0.91544766, 0.28174064,
            0.95373883, 0.24207103, 0.71282905],
           [0.19356225, 0.79483475, 0.47494288, 0.32012374, 0.48439538,
            0.47473527, 0.70758254, 0.03913135],
           [0.84760828, 0.23620354, 0.41211923, 0.86124438, 0.88559007,
            0.2094045 , 0.40099728, 0.30124624],
           [0.94832213, 0.03883361, 0.77862773, 0.28588957, 0.65135352,
            0.779747  , 0.31322503, 0.42284284]])

```python
invA = np.linalg.inv(A)
invA
```

    array([[  3.46384409,  19.4941747 , -19.57516426,  -9.99832219,
             16.87962614, -21.72840338,  10.68560741, -12.72897264],
           [ -1.10586623,  -5.12078341,   6.11757728,   2.0388313 ,
             -4.27217314,   6.48526459,  -3.30193302,   3.0640342 ],
           [  0.68791074,   0.38077008,  -1.9615242 ,  -1.32420149,
              1.6404582 ,  -1.4065594 ,   1.33985945,  -0.55078586],
           [  2.58608377,  14.65423917, -16.01834189,  -7.15281545,
             13.78490596, -16.9852474 ,   9.6480159 , -10.94445835],
           [ -5.82030521, -30.09706056,  31.5845337 ,  15.0318286 ,
            -27.0470216 ,  34.0190968 , -16.34500627,  20.6302604 ],
           [  0.05003706,   4.15891093,  -4.04486993,  -1.3253622 ,
              3.61229059,  -4.0262299 ,   1.62408924,  -2.12303422],
           [  2.82978224,  12.70682825, -13.32494727,  -5.54530857,
             10.17134655, -13.3880142 ,   6.46532306,  -8.29540923],
           [ -3.90488946, -24.57896545,  26.45841824,  12.90733087,
            -22.33715585,  27.14770809, -15.25813491,  17.32584407]])

```python
Ia = invA @ A
print(Ia)
```

    [[ 1.00000000e+00 -3.98888673e-15 -4.35219105e-15 -4.70036671e-15
      -6.60232898e-15 -1.28837269e-15 -4.56649980e-15  3.96398639e-16]
     [ 1.42512459e-15  1.00000000e+00  2.43207058e-15  6.79481724e-16
       7.88674531e-16  1.83800676e-15  9.83902242e-16 -1.77459679e-16]
     [ 4.04279473e-16  2.17604780e-15  1.00000000e+00  8.57498938e-16
       5.11228527e-16  1.24790256e-15  5.08088010e-16  6.64935732e-16]
     [-7.07300473e-15 -2.63545716e-15 -3.60809150e-15  1.00000000e+00
      -3.53441202e-15 -1.22768382e-15 -2.29970582e-15 -2.62264867e-16]
     [ 1.30348743e-14  2.95515821e-15  2.75798559e-15  1.23000406e-14
       1.00000000e+00  4.20945500e-15 -9.87233322e-16  4.63996262e-15]
     [-2.80015751e-15 -1.62318587e-15 -9.97625070e-16 -1.92358684e-15
      -1.08689218e-15  1.00000000e+00 -5.90866743e-16 -1.25167019e-15]
     [-8.88178420e-15 -3.55271368e-15 -3.55271368e-15 -5.32907052e-15
      -2.66453526e-15 -1.77635684e-15  1.00000000e+00 -1.33226763e-15]
     [ 1.24344979e-14  0.00000000e+00  3.55271368e-15  5.32907052e-15
       5.32907052e-15  0.00000000e+00  3.55271368e-15  1.00000000e+00]]

```python
np.round(Ia)
```

    array([[ 1., -0., -0., -0., -0., -0., -0.,  0.],
           [ 0.,  1.,  0.,  0.,  0.,  0.,  0., -0.],
           [ 0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.],
           [-0., -0., -0.,  1., -0., -0., -0., -0.],
           [ 0.,  0.,  0.,  0.,  1.,  0., -0.,  0.],
           [-0., -0., -0., -0., -0.,  1., -0., -0.],
           [-0., -0., -0., -0., -0., -0.,  1., -0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.]])

## Vector space

$v, u \in \mathbb{R}^{n}$ and for every $\alpha \in \mathbb{R}^{1}$ we have $u + v \in \mathbb{R}^{n}$ and $\alpha u \in \mathbb{R}^{n}$

So we have a sum and multiplication on scalar with properties:
- for every $u, v, w \in \mathbb{R}^{n}$: $(u + v) + w = u + (u + w)$
- for every $u, v \in \mathbb{R}^{n}$: u + v = v + u
- there exists $0 \in \mathbb{R}^{n}$ such that: $0 + u = u + 0 = u$
- for every $u \in \mathbb{R}^{n}$ there exists $-u \in \mathbb{R}^{n}$ such that: $u + (-u) = (-u) + u = 0$
- for every $\alpha, \beta \in \mathbb{R}^{1}$ and every $u \in \mathbb{R}^{n}$: $\alpha(\beta u) = (\alpha \beta u)$
- for every $u \in \mathbb{R}^{n}$: $1u=u1=u$
- for every $u, v \in \mathbb{R}^{n}$ and $\alpha \in \mathbb{R}^{1}$: $\alpha (u + v) = \alpha u + \alpha v$
- for every $\alpha, \beta \in \mathbb{R}^{1}$ and every $u \in \mathbb{R}^{n}$: $(\alpha + \beta)u = \alpha u + \beta v$

So we can define $-$ and $:$ operations as well
If some structure satisfies such properties it's called vector space

```python
import random
```

```python
u = np.random.random(5)
v = np.random.random(5)
x = random.random()
y = random.random()
w = u * x + v * y
u, v, x, y, w
```

    (array([0.17466798, 0.93788138, 0.42326657, 0.50182232, 0.59076164]),
     array([0.36051146, 0.679797  , 0.18116046, 0.11820887, 0.77204126]),
     0.4786435074739014,
     0.7460982666337203,
     array([0.35258067, 0.95610619, 0.3377573 , 0.32838943, 0.85878287]))

## Linear combination

Let $v_1, v_2, \dots v_n \in \mathbb{R}^{n}$ and $\alpha_{1}, \alpha_{2} \dots \alpha_{n} \in \mathbb{R}^{1}$ then linear combination of this vectors is called the vector $w = \sum_{i=1}^{n}\alpha_{i} v_{i}$

Vectors $v_1, v_2, \dots v_n \in \mathbb{R}^{n}$ are called linearly independent if none of them can be linear combination of others, of for each $i \in (1 \dots n)$ there is no such $\alpha_{1}, \alpha_{2} \dots \alpha_{i-1}, \alpha_{i+1} \dots \alpha_{n_1} \in \mathbb{R}^{1}$ such that $u_i = \sum_{k = 1, k \neq i}^{n}\alpha_{k}u_k $

Maximum amount of linearly independent vectors in vector space is called dimension of this space

Every vector space has basis, linearly independent vectors $e_1, e_2 \dots e_n $ such that every other vector from this space can be achieved by the linear combination of this basis $u = \sum_{i=1}^n\alpha_{i}e_i$ and $(\alpha_{1}, \alpha_{2}, \dots, \alpha_{n})$ are called the coordinates of the vector $u$

For $\mathbb{R}^{n}$ basis is $e_1 = (1, 0, \dots, 0), e_2 = (0, 1, \dots, 0), \dots e_i = (0, 0, \dots, 1, \dots, 0), e_n = (0, 0, \dots, 1)$
Or for $\mathbb{R}^{2}$ basis is $e_1=(1, 0), e_2=(0, 1)$ and for $\in \mathbb{R}^{3}$: $e_1=(1, 0, 0), e_2 = (0, 1, 0), e_3=(0, 0, 1)$

## Linear maps

Map $f:X \to Y$ between vector spaces $\mathbb{U}$ and $\mathbb{V}$ is called linear (or linear transformation) if for every $u, v \in \mathbb(U)$ and every scalar $\alpha in \in \mathbb{R}^{1}$ we have:
- $f(u + v) = f(u) + f(v)$
- $f(\alpha u) = \alpha f(u)$

Let $e_1, e_2, \dots, e_n$ be a basis for linear space $\mathbb{U}$ and $l_1, l_2, \dots, l_m$ basis for $\mathbb{V}$
then $f(e_i) = a_1{li}_1 + a_{2i}l_2 + \dots + a_{mi}l_m$ for some $a_1, \dots, a_m \in \in \mathbb{R}^{1}$
for we have the following matrix:
$$\begin{align} T &= \begin{pmatrix}
           a_{11}, a_{12}, \dots, a_{1n} \\
           a_{21}, a_{22}, \dots, a_{2n} \\
           \vdots \\
           a_{m1}, a_{n2}, \dots, a_{mn} \\
         \end{pmatrix}
  \end{align}$$

which is called the transformation matrix

For each $u \in \mathbb{U}$ there exists $b_1, b_2, \dots, b_n \in \mathbb{R}^{1}$ such that: $u = b_1e_1 + b_2e_2 + \dots + b_ne_n$ and for $f(u)$ (from linear property) we have $$f(u) = b_1f(e_1) + b_2f(e_2) + \dots + b_nf(e_n)$$
then from the property - $f(e_i) = a_{1i}l_1 + a_{2i}l_2 + \dots + a_{mi}l_m$, we get:
$$f(u) = Tb$$

## Eigenvectors and Eigenvalues

If for some linear transformation $T:\mathbb{U} \to \mathbb{V}$ with transformation matrix $M$, there exists non-zero vector $u \in \mathbb{U}$ and scalar $\lambda \in \mathbb{R}^{1}$ such that $T(u) = Mu = \lambda u$, this vector is called eigenvector for the transformation $T$ and $\lambda$ is called eigenvalue

$Mu = \lambda u$ then $Mu - \lambda u = 0$ and
$$(M - I\lambda)u = 0$$

This equasion has a solution if $|M - I\lambda|=0$, thus, we can calculate eigenvector and eigenvalue for linear transformation

Consider eigenvectors $v_1, v_2, \dots, v_n$ and eigenvalues $\lambda_{1}, \lambda_{2}, \dots, \lambda_{n}$ from basis $e_1, e_2, \dots, e_n$ of transformation matrix $A$, let $Q = (v_1, v_2 \dots, v_n)$ then:
$$AQ = (\lambda_{1}v_1, \lambda_{2}v_2, \dots, \lambda_{n}v_n)$$
define:
$$\begin{align} \Lambda &= \begin{pmatrix}
           \lambda_{11}, 0, \dots, 0 \\
           0, \lambda_{22}, \dots, 0 \\
           \vdots \\
           0, 0, \dots, \lambda_{nn} \\
         \end{pmatrix}
  \end{align}$$

$$AQ = Q\Lambda$$

$$AQQ^{-1}=Q\Lambda Q^{-1}$$

$$AI=Q\Lambda Q^{-1}$$

$$A=Q\Lambda Q_{-1}$$

This is called eigendecomposition of matrix $A$

## Visualisation of Eigenvalues and Eigenvectors

If we consider transformation matrix A again, and we take bunch of vectors (as much as possible) from given vector
space where that matrix is doing transformations, we can find some vectors that never change their orientation but maybe they are scaled by some other factor.

Each such vector is called Eigenvector of matrix A which direction isn't affected by transformation, but scale is affected. Scale of that vector will be eigenvalue of that vector.

Each eigenvector has its eigenvalue and these can be multiple because of several axis of transformation.

### Eigenvalues and Eigenvectors example

### Non-Eigenvalues and Eigenvectors example

## Vector (Matrix) norm

$$||x||_{p} = \sqrt[p]{\sum_{i=1}^{n}x_i^{p}}$$
$$||x||_{p} = (\sum_{i=1}^{n}x_i^{p})^{1/p}$$

$L_2$ norm
$$||x||_{2} = (\sum_{i=1}^{n}x_i^{2})^{1/2}$$
$L_1$ norm
$$||x||_{1} = \sum_{i=1}^{n}|x_i|$$

$$||x||_{\infty} = max|x_i|$$

For matrices Frobenius norm:
$$||A||_{F} = (\sum_{i=1, j=1}^{n, m}a_{ij})^2$$

## Determinant as a scaling factor

Here is a link to the [video](https://www.youtube.com/watch?v=Ip3X9LOh2dk) about determinants

Given two vectors x,y in space and some transformation matrix A.
If we multiply these vectors by given transformation matrix, we will get transformed vectors.
Area value before and after transformation will be changed with exactly the value of determinant of A.

## SVD

For any $A \in \mathbb{R}^{n \times m} (\mathbb{C}^{n \times m})$ there exists decomposition:
$$A = U \Sigma V^{T}$$
where $U \in \mathbb{R}^{n \times n}(\mathbb{C}^{n \times n})$ is a square matrix, $\Sigma \in \mathbb{R}^{n \times m} (\mathbb{C}^{n \times m})$ is a diagonal matrix and  $V \in \mathbb{R}^{n \times n} (\mathbb{C}^{n \times n})$ is also a square matrix
#### Note: We only discuss real valued vectors, matrices and tensors in this course

short [tutorial](http://web.mit.edu/be.400/www/SVD/Singular_Value_Decomposition.htm) on SVD from MIT

very interesting medium [blog](https://medium.com/@jonathan_hui/machine-learning-singular-value-decomposition-svd-principal-component-analysis-pca-1d45e885e491) on SVD & PCA

```python
A = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]])
A, A.shape
```

    (array([[ 1,  2,  3],
            [ 4,  5,  6],
            [ 7,  8,  9],
            [10, 11, 12]]), (4, 3))

```python
U, S, V_T = np.linalg.svd(A)
U, S, V_T, U.shape, S.shape, V_T.shape
```

    (array([[-0.14087668,  0.82471435,  0.53358462,  0.12364244],
            [-0.34394629,  0.42626394, -0.8036038 ,  0.2328539 ],
            [-0.54701591,  0.02781353,  0.00645373, -0.83663514],
            [-0.75008553, -0.37063688,  0.26356544,  0.48013879]]),
     array([2.54624074e+01, 1.29066168e+00, 1.38648772e-15]),
     array([[-0.50453315, -0.5745157 , -0.64449826],
            [-0.76077568, -0.05714052,  0.64649464],
            [-0.40824829,  0.81649658, -0.40824829]]),
     (4, 4),
     (3,),
     (3, 3))

```python
Sg = np.diag(S)
Sg, Sg.shape
```

    (array([[2.54624074e+01, 0.00000000e+00, 0.00000000e+00],
            [0.00000000e+00, 1.29066168e+00, 0.00000000e+00],
            [0.00000000e+00, 0.00000000e+00, 1.38648772e-15]]), (3, 3))

```python
np.linalg.norm(A), np.round(U[:, 0] @ U[:, 1].T), np.linalg.norm(
    U[2, :]
), np.linalg.norm(V_T)
```

    (25.495097567963924, -0.0, 1.0, 1.7320508075688772)

# Matrix Decompositions

[fastai LA](https://nbviewer.jupyter.org/github/fastai/numerical-linear-algebra/blob/master/nbs/1.%20Why%20are%20we%20here.ipynb#Matrix-Decompositions)

[advanced matrix decompositions](https://sites.google.com/site/igorcarron2/matrixfactorizations)

[nfm tutorial](https://perso.telecom-paristech.fr/essid/teach/NMF_tutorial_ICME-2014.pdf)

[topic modeling](https://medium.com/@nixalo/comp-linalg-l2-topic-modeling-with-nmf-svd-78c94330d45f)

[background removal using svd](https://medium.com/@siavashmortezavi/fast-randomized-svd-singular-value-decomposition-using-pytorch-and-gpus-46b627511a6d)

## PCA

### Data Reduction
- PCA is most commonly used to condense the information contained in a large number of original variables into a smaller set of new composite dimensions, with a minimum loss of information.

[Example](https://www.projectrhea.org/rhea/index.php/PCA_Theory_Examples) of using PCA on image compression

#### Mapping of 2D points into 1D.
PCA Takes the most optimal 1d axis to save data information better, reducing memory by factor of 2.

### Interpretation
- PCA can be used to discover important features of a large data set. It often reveals relationships that were previously unsuspected, thereby allowing interpretations that would not ordinarily result.
PCA is typically used as an intermediate step in data analysis when the number of input variables is otherwise too large for useful analysis.

[example](https://towardsdatascience.com/visualising-high-dimensional-datasets-using-pca-and-t-sne-in-python-8ef87e7915b) usage of pca (and t-SNE) for data visualization

[example](https://github.com/aviolante/sas-python-work/blob/master/tSneExampleBlogPost.ipynb) notebook for comparing PCA and t-SNE for visualizing MNIST data

Let $X = (x^{1}, x^{2}, \dots x^{m})$ is our data where $x^{i} = (x_1^{i}, x_2^{i}, \dots, x_n^{i}) \in \mathbb{R}^{n}$ for each $i \in (1, 2, \dots, m)$

Normalize data with mean $x^{i} = x^{i} - \frac{1}{m}\sum_{i=1}^{m}x^{i}$

compute the covariance matrix:
$$A = \frac{1}{m} \sum_{i=1}^{m}(x^{i})(x^{i})^{T}$$

take SVD from $X$:
$$ A = U\Sigma V^{T}$$
and consider first $k \leq n$ columns of $U \in \mathbb{R}^{n\times n}$:
$$u_1, u_2, \dots, u_k \in \mathbb{R}^{n}$$
now consider the matrix $U_{r} = (u_1, u_2, \dots, u_k)$ and
$$z^{i} = U_{r}^{T}x^{i}$$

$U_{r}^{T} \in \mathbb{R}^{k \times n}$ and $x^{i} \in \mathbb{R}^{n \times 1}$ thus $z^{i} \in \mathbb{R}^{k \times 1}$

### Reconstruction

We can approximate the reconstruction of the original value of $x^{i}$ as
$$x_{a}^{i} = U_{r}z^{i}$$

$z^{i} \in \mathbb{R}^{n \times 1}$

to check our method we should compare original value to approximation:
$$\frac{
\frac{1}{m}\sum_{i=1}^{m}||x^{i} - x_{a}^{i}||^{2}
}{
\frac{1}{m}\sum_{i=1}^{m}||x^{i}||^{2}
} \leq \epsilon$$

$\epsilon$ might be any value, e.g $\epsilon = 0.01$

$$\frac{
\frac{1}{m}\sum_{i=1}^{m}||x^{i} - x_{a}^{i}||^{2}
}{
\frac{1}{m}\sum_{i=1}^{m}||x^{i}||^{2}
} \leq  = 1 -
\frac{
\sum_{i=1}^{k}S_{ii}
}{
\sum_{j=1}^{n}S_{jj}
}$$

So we can calculate
$$\frac{
\sum_{i=1}^{k}S_{ii}
}{
\sum_{j=1}^{n}S_{jj}
} \geq \epsilon$$

Only one decomposition is enough

[pca](https://www.coursera.org/learn/machine-learning/lecture/GBFTt/principal-component-analysis-problem-formulation)

# Additional Materials

[jupyter-notebook tips&tricks&shortcuts](https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/)


# Infectious disease modelling

## The effect of transient behaviour on final epidemic size

With the recent Coronavirus pandemic, a lot of effort has been put into modelling the spread of infectious diseases. The simplest model is known as SIR (Susceptable-Infectious-Recovered) introduced by <cite>Kermack and McKendrick (1927)</cite> is an ODE system of the form
$$\begin{align}
\dot S_i &= -\lambda_i(t)S_i \\
\dot I_i &= \lambda_i(t)S_i - \gamma I_i \\
\dot R_i &= \gamma I_i
\end{align}$$

Where $S_i,I_i,R_i$ are susceptable, infectious and recovered percentages of the population $N$ in compartment i such that $\sum_i S_i+I_i+R_i=N$. In its most basic form, this model has only one compartment (i.e the index $i\in \{1\}$). However, if we have $n$ compartments, the linear response from the initial state may have a non-normal matrix contact structure. We first consider the 2 compartment model $$
\begin{align}
\lambda_1(t) = \beta(C_{11}\frac{I_1}{f_1} + C_{12}\frac{I_2}{f_1})\\
\lambda_2(t) = \beta(C_{21}\frac{I_1}{f_2} + C_{22}\frac{I_2}{f_2})
\end{align}$$

where our contact matrix obeys $f_1 C_{12} = f_2 C_{21}$ where $f_i = \frac{N_i}{N}$ is the fraction of the population in each compartment. If $f_1 = f_2$ then $C$ is symmetric and hence normal. However, there is the possibility for non-normality which we investigate here.

```python
import pyross
import numpy as np
import matplotlib.pyplot as plt
import scipy.linalg as spl
```

```python
M = 2                  # the SIR model has no age structure
# Ni = 1000*np.ones(M)   # so there is only one age group
N = 100000         # and the total population is the size of this age group

Ni = np.zeros((M))      # population in each group
fi = np.zeros((M))      # fraction of population in age age group
# set the age structure
fi = np.array((0.25, 0.75))
for i in range(M):
    Ni[i] = fi[i]*N

beta  = 0.02            # infection rate
gamma = 0.007
gIa   = gamma            # recovery rate of asymptomatic infectives
gIs   = gamma            # recovery rate of symptomatic infectives
alpha = 0              # fraction of asymptomatic infectives
fsa   = 1              # Fraction by which symptomatic individuals do not self isolate

Ia0 = np.array([0,0])     # the SIR model has only one kind of infective
Is0 = np.array([1,.1])     # we take these to be symptomatic
R0  = np.array([0,0])     # and assume there are no recovered individuals initially
S0  = Ni    # so that the initial susceptibles are obtained from S + Ia + Is + R = N
### No f_i present here

# set the contact structure
C11, C22, C12 = 1,1,4
C = np.array(([C11, C12], [C12*fi[1]/fi[0], C22]))

# if Ni[0]*C[0,1]!=Ni[1]*C[1,0]:
#     raise Exception("invalid contact matrix")

# there is no contact structure
def contactMatrix(t):
    return C

# duration of simulation and data file
Tf = 160;  Nt=160;

# instantiate model
parameters = {'alpha':alpha, 'beta':beta, 'gIa':gIa, 'gIs':gIs,'fsa':fsa}
model = pyross.deterministic.SIR(parameters, M, Ni)

# simulate model
data = model.simulate(S0, Ia0, Is0, contactMatrix, Tf, Nt)
```

```python
# matrix for linearised dynamics
C=contactMatrix(0)
A=((beta*C-gamma*np.identity(len(C))).T*fi).T/fi
mcA=pyross.contactMatrix.characterise_transient(A, ord=1)
AP = A-np.max(np.linalg.eigvals(A))*np.identity(len(A))
mcAA = pyross.contactMatrix.characterise_transient(AP,ord=1)
print(mcAA)
```

    [0.0000000e+00+0.j 2.3476927e-01+0.j 3.0905325e+00+0.j 1.1283435e+03+0.j
     6.9333333e-01+0.j]

Kreiss constant of $\Gamma = A-\lambda_{Max}A$ is ~3.09

```python
# plot the data and obtain the epidemic curve
Sa  = data['X'][:,:1].flatten()
Sk  = data['X'][:,1:M].flatten()
St=Sa+Sk
# Ia = data['X'][:,1].flatten()
Isa = data['X'][:,2*M:2*M+1].flatten()
Isk = data['X'][:,2*M+1:3*M].flatten()
It = Isa + Isk
# It = np.sqrt(Isa**2 + Isk**2)
t = data['t']

fig = plt.figure(num=None, figsize=(10, 8), dpi=80, facecolor='w', edgecolor='k')
plt.rcParams.update({'font.size': 22})

plt.fill_between(t, 0, St/N, color="#348ABD", alpha=0.3)
plt.plot(t, St/N, '-', color="#348ABD", label='$S$', lw=4)

plt.fill_between(t, 0, It/N, color='#A60628', alpha=0.3)
plt.plot(t, It/N, '-', color='#A60628', label='$I$', lw=4)

Rt=N-St-It; plt.fill_between(t, 0, Rt/N, color="dimgrey", alpha=0.3)
plt.plot(t, Rt/N, '-', color="dimgrey", label='$R$', lw=4)

plt.autoscale(enable=True, axis='x', tight=True)

###Estimate from Kreiss constant
plt.plot(t,mcAA[2]*It[0]*np.exp(mcA[0]*t)/N,'-', color="green",
         label='$Estimate$', lw=4)
# plt.ylim([0,1])
plt.yscale('log')
# plt.xlim([0,60])
plt.xlabel("time")
plt.ylabel("% of population")
plt.legend(fontsize=26); plt.grid()
```

```python
# plot the data and obtain the epidemic curve
Sa  = data['X'][:,:1].flatten()
Sk  = data['X'][:,1:M].flatten()
St=Sa+Sk
Isa = data['X'][:,2*M:2*M+1].flatten()
Isk = data['X'][:,2*M+1:3*M].flatten()
It = Isa + Isk
t = data['t']

fig = plt.figure(num=None, figsize=(10, 8), dpi=80, facecolor='w', edgecolor='k')
plt.rcParams.update({'font.size': 22})

plt.fill_between(t, 0, St/N, color="#348ABD", alpha=0.3)
plt.plot(t, St/N, '-', color="#348ABD", label='$S$', lw=4)

plt.fill_between(t, 0, It/N, color='#A60628', alpha=0.3)
plt.plot(t, It/N, '-', color='#A60628', label='$I$', lw=4)

Rt=N-St-It; plt.fill_between(t, 0, Rt/N, color="dimgrey", alpha=0.3)
plt.plot(t, Rt/N, '-', color="dimgrey", label='$R$', lw=4)

###Estimate from Kreiss constant
plt.plot(t,mcAA[2]*It[0]*np.exp(mcA[0]*t)/N,'-', color="green",
         label='$Estimate$', lw=4)
plt.ylim([0,1])
# plt.yscale('log')

plt.legend(fontsize=26); plt.grid()
plt.autoscale(enable=True, axis='x', tight=True)
# plt.xlim([0,100])

```

The only pitfall is that we need to specify the order of the spectral norm. Most physics processes naturally take place in an $L2$ norm space. However, here we are interested in $I_{total} = \sum_n I_n$ which is an $L1$ norm. This option can be specified in `characterise_transient`.

```python

```