## author: Keer Mei
- github: kmei92
- email: keer.mei18@imperial.ac.uk

# Setup

```python
# importing the necessary modules
import sys
sys.path.append("..")
from fireframe.PDESystem import *
from fireframe.PDESubsystem import *
from fireframe.pdeforms import *

%load_ext autoreload
%autoreload 2
```

# Description:

This demonstration provides the spatial and temporal convergence of the radionuclide transport kinetic equations coupled to the incompressible Navier-Stokes and continuity equations.
The manufactured solution for radionuclide 1 in the dissolved phase is given as:

$$ C_{d, i} = \exp(xyt), $$
where x and y are spatial coordinates and t is time

The entire source term calculated from this manufactured solution is:

$$ f_{source} = xyC_{d, 1exact} + u\cdot(ytC_{d, 1exact}, \quad xtC_{d, 1exact}) \\
- K_d((yt)^2C_{d, 1exact} + (xt)^2C_{d, 1exact}) \\
+ k_1C_{d, 1exact} + \lambda_1 C_{d, 1exact} $$

# Variational forms:

The variational form for the radionuclide transport kinetic equations is given in:
- fireframe.pdeforms.radio_transport_coupled_mms (with the exact source term written in the expanded form above)
- fireframe.pdeforms.radio_transport_coupled_mms_ufl (with the source term obtained from the manufactured term differentiated using ufl syntax)

# Creating pde_solver, a child object from PDESystem

```python
# a PDESystem child class called PDESolver. Note, the setup_constants function and setup_bcs are declared
# and overloaded here and the PDESystem's functions are not used. This is designed for clarity
class pde_solver(PDESystem):
    def __init__(self, comp, mesh, parameters):
        PDESystem.__init__(self, comp, mesh, parameters)

    def setup_bcs(self):
        x, y = fd.SpatialCoordinate(self.mesh)
        c0 = fd.exp(x*y*self.t) # analytical solution to the specific activity of dissolved phase radionuclide
        # one

        bcu = [fd.DirichletBC(self.V['u'], fd.Constant((0,0)), (10, 12)), # top-bottom and cylinder
          fd.DirichletBC(self.V['u'], ((1.0*(y - 1)*(2 - y))/(0.5**2) ,0), 9)] # inflow
        bcp = [fd.DirichletBC(self.V['p'], fd.Constant(0), 11)]  # outflow
        bcc1 = [fd.DirichletBC(self.V['c'].sub(0), c0, 'on_boundary')] # maintain a Dirichlet boundary for
        # radionuclide one concentration

        self.bc['u'][0] = [bcu, None, None, None,'fixed'] # static boundary
        self.bc['p'] = [[bcp, None, None, None, 'fixed']] # static boundary
        self.bc['c'][0] = [bcc1, c0, 'on_boundary', 0, 'update'] # time varying Dirichlet boundary

    def setup_constants(self):
        x, y = fd.SpatialCoordinate(self.mesh)

        self.constants = {
            'deltat' : fd.Constant(self.prm['dt']),
            'Kd' : fd.Constant(0.01),
            'k1' : fd.Constant(0.005),
            'k2' : fd.Constant(0.00005),
            'lamd1' : fd.Constant(0.000005),
            'lamd2' : fd.Constant(0.0),
            'rho_s' : fd.Constant(1.),
            'L' :  fd.Constant(1.),
            'phi' : fd.Constant(0.3),
            'n' : fd.FacetNormal(self.mesh),
            'f' : fd.Constant((0.0, 0.0)),
            'nu' : fd.Constant(0.001),
            'frac' : fd.Constant(1.),
        }
```

# MMS Code Verification

## Spatial discretization

```python
import sympy as sy

# update the parameters
solver_parameters = recursive_update(solver_parameters,
{
'space': {'u': fd.VectorFunctionSpace, 'c': fd.MixedFunctionSpace, 'd' : fd.MixedFunctionSpace},
'degree': {'u': 2},
'order' : {'c': 3, 'd':3},
'ksp_type': {'u': 'gmres', 'p': 'gmres',  'c': 'gmres', 'd':'gmres'},
'precond': {'u': 'sor', 'p' : 'sor', 'c': 'sor', 'd':'gmres'},
'dt' : 0.001,
'T' : 1.5})

#load mesh
mesh = fd.Mesh("../meshes/step1.msh")

# add subsystems for navier stokes and radio_transport
solver = pde_solver([['u', 'p']], mesh, solver_parameters)
# solver.add_subsystem(['cd', 'cs', 'as'], solver_parameters)
solver.add_subsystem(['c', 'd'], solver_parameters)
#setup constants
solver.setup_constants()
# define subsystems and variable sequence
solver.define(['u', 'p', 'u'], 'up', navier_stokes)
solver.define(['c', 'd'], 'cd', radio_transport_coupled_mms)
# setup boundary conditions
solver.setup_bcs()
x, y, t = sy.symbols(('x', 'y', 't'))
expr = sy.exp(x*y*t)
meshes = [fd.Mesh("../meshes/step%d.msh" % i) for i in range(1, 9)]
solver.test_mms('c', expr, spatial=True, f_dict={"exp":fd.exp}, meshes=meshes, plot=True, index=0)
```

## Temporal discretization

```python
# update the parameters
solver_parameters = recursive_update(solver_parameters,
{
'space': {'u': fd.VectorFunctionSpace, 'c': fd.MixedFunctionSpace, 'd' : fd.MixedFunctionSpace},
'degree': {'u': 2},
'order' : {'c': 3, 'd':3},
'ksp_type': {'u': 'gmres', 'p': 'gmres',  'c': 'gmres', 'd':'gmres'},
'precond': {'u': 'sor', 'p' : 'sor', 'c': 'sor', 'd':'gmres'},
'dt' : 0.01,
'T' : 0.1})

#load mesh
mesh = fd.Mesh("../meshes/step1.msh")
deltat = [0.01 / (2**i) for i in range(8)]

# add subsystems for navier stokes and radio_transport
solver = pde_solver([['u', 'p']], mesh, solver_parameters)
# solver.add_subsystem(['cd', 'cs', 'as'], solver_parameters)
solver.add_subsystem(['c', 'd'], solver_parameters)
#setup constants
solver.setup_constants()
# define subsystems and variable sequence
solver.define(['u', 'p', 'u'], 'up', navier_stokes)
solver.define(['c', 'd'], 'cd', radio_transport_coupled_mms)
# setup boundary conditions
solver.setup_bcs()

x, y, t = sy.symbols(('x', 'y', 't'))
expr = sy.exp(x*y*t)
solver.test_mms('c', expr, temporal=True, f_dict={"exp":fd.exp}, dt_list=deltat, plot=True, index=0)
```

```python
# update the parameters
solver_parameters = recursive_update(solver_parameters,
{
'space': {'u': fd.VectorFunctionSpace, 'c': fd.MixedFunctionSpace, 'd' : fd.MixedFunctionSpace},
'degree': {'u': 2},
'order' : {'c': 3, 'd':3},
'ksp_type': {'u': 'gmres', 'p': 'gmres',  'c': 'gmres', 'd':'gmres'},
'precond': {'u': 'sor', 'p' : 'sor', 'c': 'sor', 'd':'gmres'},
'dt' : 0.01,
'T' : 0.1})

#load mesh
mesh = fd.Mesh("../meshes/step1.msh")
deltat = [0.01 / (2**i) for i in range(6)]

# add subsystems for navier stokes and radio_transport
solver = pde_solver([['u', 'p']], mesh, solver_parameters)
# solver.add_subsystem(['cd', 'cs', 'as'], solver_parameters)
solver.add_subsystem(['c', 'd'], solver_parameters)
#setup constants
solver.setup_constants()
# define subsystems and variable sequence
solver.define(['u', 'p', 'u'], 'up', navier_stokes)
solver.define(['c', 'd'], 'cd', radio_transport_coupled_mms)
# setup boundary conditions
solver.setup_bcs()

x, y, t = sy.symbols(('x', 'y', 't'))
expr = sy.exp(x*y*t)
solver.test_mms('c', expr, temporal=True, f_dict={"exp":fd.exp}, dt_list=deltat, plot=True, index=0)
```


## Project One: Maximum Sub-Array Summation

### Group 12

#### Group Members

* Kyle Guthrie
* Michael C. Stramel
* Alex Miranda

# Enumeration

## Pseudo-code
The "Enumeration" maximum sub-array algorithm is described by the following pseudo-code:
~~~~
ENUMERATION-MAX-SUBARRAY(A[1,...,N]) {
    if N == 0 {
        return 0, A
    } else {
        max_sum = -Infinity
    }

    for i from 1 to N {
        for j from i to N {
            current_sum = 0
            for k from i to j {
                current_sum = current_sum + A[k]
                if current_sum > max_sum {
                    max_sum = current_sum
                    start_index = i
                    end_index = j
                }
            }
        }
    }
    return max_sum, A[start_index, ..., end_index]
}
~~~~

## Theoretical Run-time Analysis
The outer $i$ loop runs from $1$ to $N$, the first inner $j$ loop runs from $i$ to $N$, and the second inner loop runs from $i$ to $j$. We can compute the number of iterations as:

* $\sum_{i=1}^N \sum_{j=i}^N \sum_{k=i}^j \Theta(1)$
* $\sum_{i=1}^N \sum_{j=i}^N (j - i + 1) \Theta(1)$
* $\sum_{i=1}^N (\sum_{j=i}^N (1 - i) + \sum_{j=i}^N j) \Theta(1)$
* $\sum_{i=1}^N ((i - 1)(i - N - 1) - \frac{1}{2}(i + N)(i - N - 1)) \Theta(1)$
* $\sum_{i=1}^N ((i^2 - iN - 2i + N + 1) - \frac{1}{2}(i^2 - i - N^2 - N)) \Theta(1)$
* $\sum_{i=1}^N \frac{1}{2}i^2 - iN - \frac{3}{2}i + \frac{1}{2}N^2 + \frac{3}{2}N + 1) \Theta(1)$
* $\sum_{i=1}^N \frac{1}{2}(i^2 - 2iN - 3i) + \sum_{i=1}^N \frac{1}{2}(N^2 + 3N + 2) \Theta(1)$

So we can find the sums term by term for the terms with $i$ while the sum of terms without $i$ will remain constant therefore:

* $\sum_{i=1}^N \frac{1}{2}(N^2 + 3N + 2) = (\frac{1}{2}N^2 + \frac{3}{2}N + 1)\sum_{i=1}^N 1$
* $(\frac{1}{2}N^2 + \frac{3}{2}N + 1)\sum_{i=1}^N 1 = (\frac{1}{2}N^2 + \frac{3}{2}N + 1) * N$
* $(\frac{1}{2}N^2 + \frac{3}{2}N + 1) * N = \frac{1}{2}N^3 + \frac{3}{2}N^2 + N$
* $\sum_{i=1}^N \frac{1}{2} i^2 = \frac{1}{2}(\frac{1}{6}N(N + 1)(2N + 1))$
* $\frac{1}{2}(\frac{1}{6}N(N + 1)(2N + 1)) = \frac{1}{6}N^3 + \frac{1}{4}N^2 + \frac{1}{12}N$
* $\sum_{i=1}^N \frac{1}{2} (-2iN) = -\frac{1}{2}N(N + 1) * N = -\frac{1}{2}(N^3 + N^2)$
* $\sum_{i=1}^N \frac{1}{2} (-3i) = \frac{1}{2}(-\frac{3}{2}N(N + 1)) = -\frac{3}{4}(N^2 + N)$

After collecting like terms:

* $\frac{1}{6}N^3 + \frac{1}{2}N^2 + \frac{1}{3}N$
* $\sum_{i=1}^N \sum_{j=i}^N \sum_{k=i}^j \Theta(1) = \frac{1}{6}N^3 + \frac{1}{2}N^2 + \frac{1}{3}N \cdot \Theta(1)$
* $\frac{1}{6}N^3 + \frac{1}{2}N^2 + \frac{1}{3}N \cdot \Theta(1) = \Theta(N^3)$ (Because the $N^3$ will dominate)

Thus the runtime of the whole algorithm is equivalent to $\Theta(N^{3})$.

## Experimental Analysis
For a series of array sizes $N$, $10$ random arrays were generated and run through the "Enumeration" algorithm. The CPU clock time was recorded for each of the $10$ random array inputs, and an average run time was computed. Below is the plot of average run time versus $N$ for the "Enumeration" algorithm.

The equation of the best fit curve to the runtime data where $x$ stands in for $N$:

$$y = 2.12118 * 10^{-8} * x^3 + 1.53069 * 10^{-3} * x - 1.77696 * 10^{-1}$$

The highest degree of the best fit curve is three and as shown in the plot above, fits the data points very closely which stands to corroborate the theoretical run-time of $\Theta(N^{3})$.

Based on the average run time data curve fit, we would expect the "Enumeration" to be able to process the following number of elements in the given amount of time:

|    Time    | Max Input Size |
|:----------:|:--------------:|
| 10 seconds |       798      |
| 30 seconds |      1150      |
| 60 seconds |      1445      |

# Better Enumeration

## Pseudo-Code
The "Better Enumeration" maximum sub-array algorithm is described by the following pseudo-code:
~~~~
BETTER-ENUMERATION-MAX-SUBARRAY(A[1, ..., N])
	maximum sum = -Infinity
	for i from 1 to N
		current sum = 0
		for j from i to N
			current sum = current sum + A[j]
			if current sum > maximum sum
				maximum sum = current sum
				start index = i
				end index = j

	return maximum sum, A[start index, ..., end index]
~~~~

## Theoretical Run-time Analysis
The outer $i$ loop runs from $1$ to $N$, and the inner $j$ loop runs from $i$ to $N$. Inside the inner loop are constant time operations. We can compute the number of iterations of these constant time operations as:

\begin{equation}
\begin{split}
\sum_{i=1}^N \sum_{j=i}^N \Theta(1) & =  \sum_{i=1}^N (N + 1 - i)\cdot \Theta(1) =N(N+1)\cdot \Theta(1) -\frac{1}{2}N(N+1)\cdot \Theta(1) \\
& = \frac{1}{2}N(N+1)\cdot \Theta(1) = \Theta(N^2)
\end{split}
\end{equation}

Thus, the theoritical run-time is $\Theta(N^2)$.

## Experimental Analysis
For a series of array sizes $N$, 10 random arrays were generated and run through the "Better Enumeration" algorithm. The CPU clock time was recorded for each of the 10 random array inputs, and an average run time was computed. Below is the plot of average run time versus $N$ for the "Better Enumeration" algorithm.

A curve fit was applied to the average run time data, which resulted in the following fit equation as a function of $x$ standing in for $N$:

$$y = 5.48722 * 10^{-8} * x^{2} + 1.42659 * 10^{-4} * x - 7.776 * 10^{-1}$$

The fit curve for the plotted data has the same degree as the theoretical runtime of $\Theta(N^{2})$ so the experimental appears to match the theoretical runtime.

Based on the average run time data curve fit, we would expect the "Better Enumeration" to be able to process the following number of elements in the given amount of time:

|    Time    | Max Input Size |
|:----------:|:--------------:|
| 10 seconds |      12775     |
| 30 seconds |      22419     |
| 60 seconds |      32006     |

# Divide and Conquer

## Pseudo-code
The "Divide and Conquer" maximum sub-array algorithm is described by the following pseudo-code:
~~~~
DIVIDE_AND_CONQUER(A[1,...,N]){
    if N == 0 {
        return 0, A
    } else if N == 1 {
        return A[0], A
    }

    tmp_max = 0
    mid_max = 0
    mid_start = 0
    mid_end = 0
~~~~

~~~~
    left_max = 0
    right_max = 0

    midpoint = N / 2

    mid_start = midpoint
    mid_end = midpoint

    for i from A[N,...,midpoint] {
        tmp_max = tmp_max + A[i]
        if tmp_max > left_max {
            left_max = tmp_max
            mid_start = i
        }
    }
~~~~

~~~~
    tmp_max = 0

    for i from A[midpoint,...,N] {
        tmp_max = tmp_max + A[i]
        if tmp_max > right_max {
            right_max = tmp_max
            mid_end = i + 1
        }
    }
~~~~

~~~~
    mid_max = left_max + right_max

    left_max, left_subarray = DIVIDE_AND_CONQUER(A[0,...,midpoint])
    right_max, right_subarray = DIVIDE_AND_CONQUER(A[midpoint,...,N])

    if mid_max >= left_max and mid_max >= right_max {
        return mid_max, A[mid_start,...,mid_end]
    } else if left_max >= right_max and left_max > mid_max {
        return left_max, left_subarray
    } else if right_max > left_max and right_max > mid_max {
        return right_max, right_subarray
    }
}
~~~~

## Theoretical Run-time Analysis

In order to determine the run-time of the divide and conquer algorithm we will need to derive its recurrence. We will make a simplifying assumption that the original problem size is a power of two so that all of the subproblem sizes will remain integers. We will denote $T(n)$ as the run-time of "divide and conquer" on a subarray of $n$ elements. The base case  is when $n = 0$ or $n = 1$ which will take constant time and as a result:

* $T(1) = \Theta(1)$

The recursive case occurs when $n \gt 1$. The variable initializing prior to the first for loop will also take constant time. The following for loops are used to find the maximum sub-array that crosses the array's midpoint and will add a runtime of $\Theta(n)$. After the loops there are two recursive calls to the main function where we spend $T(\frac{n}{2})$ solving each of them so the total contribution to the running time is $2T(\frac{n}{2})$. The remaining if-else block would also run in constant time $(\Theta(1))$. So the recurrence in total for $n \gt 1$ is:

* $T(n) = 2T(\frac{n}{2}) + \Theta(n) + \Theta(1)$
* $T(n) = 2T(\frac{n}{2}) + \Theta(n)$

So the complete recurrence is:

* $T(n) = \Theta(1)$ (when $n = 0$ or $n = 1$)
* $T(n) = 2T(\frac{n}{2}) + \Theta(n)$ (when $n \gt 1$)

The recurrence can be solved using the master method as shown below:

* $T(n) = 2T(\frac{n}{2}) + \Theta(n)$
* $a = 2, b = 2, f(n) = \Theta(n)$
* $n^{log_{b}(a)} = n^{log_{2}(2)} = n^{1}$
* $\Theta(n^{log_{b}(a)}) = \Theta(n)$ (Case 2 applies)
* $T(n) = \Theta(n^{log_{2}(2)} * log_{2}(n)) = \Theta(n * log_2(n))$ (By the master theorem)

So substituting in the runtime we get:

* $T(n) = \Theta(n * log_2(n)) + \Theta(n)$

The $\Theta(n)$ term drops off leaving us with:

* $T(n) = \Theta(n * log_2(n))$

The runtime above is for the divide and conquer algorithm for finding maximum sub arrays.

## Experimental Analysis
For a series of array sizes $N$, $10$ random arrays were generated and run through the "Divide and Conquer" algorithm. The CPU clock time was recorded for each of the $10$ random array inputs, and an average run time was computed. Below is the plot of average run time versus $N$ for the "Divide and Conquer" algorithm.

A linear fit and logarithmic fit were applied to the average run time data, which resulted in the following fit equations as a function of $N$:

The function of the best logarithmic fit curve where $x$ is substituted for $N$:

$$y = 2.57865 * 10^{-7} * x * log(x) + 1.000$$

The function of the best linear fit curve where $x$ is substituted for $N$:

$$y = 6.15402 * 10^{-6} * x - 9.15974 * 10^{-1}$$

This plot has both a linear fit and a logarithmic fit to show how similiar they present for the values plotted. The logarithmic curve fits the data points almost exactly therefore it shows that the experimental values are strongly aligned with the theoretically derived runtime of $\Theta(n * log_2(n))$.

Based on the average run time data curve fit, we would expect the "Divide and Conquer" algorithm to be able to process the following number of elements in the given amount of time:

|    Time    | Max Input Size |
|:----------:|:--------------:|
| 10 seconds |      $1687210$      |
| 30 seconds |      $5050390$      |
| 60 seconds |      $9848770$      |

# Linear-time

## Pseudo-Code
The "Linear-time" maximum sub-array algorithm is described by the following pseudo-code:
~~~~
LINEAR-TIME-MAX-SUBARRAY(A[1, ..., N])
	maximum sum = -Infinity
	sum ending here = -Infinity
	for i from 1 to N
		ending here high index = j
		if ending here sum > 0
			ending here sum = ending here sum + A[i]
		else
			ending here low index = j
			ending here sum = A[i]

		if ending here sum > maximum sum
			maximum sum = ending here sum
			start index = ending here low index
			end index = ending here high index

	return maximum sum, A[start index, ..., end index]
~~~~

## Theoretical Run-time Analysis
The $i$ loop runs from $1$ to $N$. Inside the loop are constant time operations. We can compute the number of iterations of these constant time operations as:

\begin{equation}
\begin{split}
\sum_{i=1}^N \Theta(1) & = N\cdot \Theta(1) \\
& = \Theta(N)
\end{split}
\end{equation}

Thus, the theoritical run-time is $\Theta(N)$.

## Experimental Analysis
For a series of array sizes $N$, 10 random arrays were generated and run through the "Linear-time" algorithm. The CPU clock time was recorded for each of the 10 random array inputs, and an average run time was computed. Below is the plot of average run time versus $N$ for the "Linear-time" algorithm.

A curve fit was applied to the average run time data, which resulted in the following fit equation as a function of $x$ standing in for $N$:

$$y = 2.04735 * 10^{-7} * x - 1.4449$$

The best fit curve fits the plotted data extremely well, showing that the runtimes reflect a linear trend. The observed linear trend in the data matches with the theoretically derived runtime of $\Theta(n)$.

Based on the average run time data curve fit, we would expect the "Linear-time" algorithm to be able to process the following number of elements in the given amount of time:

|    Time    | Max Input Size |
|:----------:|:--------------:|
| 10 seconds |     55901100   |
| 30 seconds |    153588000   |
| 60 seconds |    300119000   |

# Testing
Several sets of test data were used to ensure the accuracy of each of the algorithms. These test data were comprised of arrays of values with known maximum sub-array solutions. These test sets were run through each algorithm, and the output was compared to the known solution.

Along with the provided set of test cases, several additional test cases were generated in order to test the algorithm accuracy under very specific conditions. Some examples of these test cases include:
* The trivial case of a single array element
* Arrays with a single positive value as the first or last element (to test the handling of the boundaries)
* Arrays with a single positive value in the middle of the arrary
* Arrays where the running sum reaches 0 at some point (i.e. multiple maximum sum sub-arrays possible)

All algorithms correctly solved all of the test data sets.

# Algorithm Comparison Plots

Three plots were generated with various combinations of linear and log scales to show the performance of the various algorithms together.

Linear plot:

Linear Log plot:

Log Log plot:


# Continuous Convolution Operations

## Background
### Discriminative Correlation Filters (DCF)

DCF methods train a correlation filter for the task of predicting the target classification scores. Unlike other methods, the DCF efficiently utilize all spatial shifts of the training samples by exploiting the discrete Fourier transform.

The conventaional DCF formulation is limited to a single-resolution feature map. Therefore, all feature channels must have teh same spatial resolution.

\begin{equation}
E(f) = \sum_{j=1}^m \alpha_j || S_f\{x_j\} - y_j ||^2 + \sum_{d=1}^D||\omega f^d||^2
\end{equation}

```python

```

### Deep convolutional neural network (CNN)

CNN have shown impressing performance for many tasks, and are therefore of interest for DCF-based tracking. A CNN consists of several layers of convolution, normalization and pooling operations. Recently, activations from the last convolutional layers hace been successfully employed for image classification. Features from these deep convolutional layers are discriminative while preserving spatial and structural information. Recent

```python

```

## Learning Continuous Convolution Operators

### Formulation

The aim is to train a continuous convolution operator based on training examples $x_j$. The samples consist of feature maps extracted from image patches. Each sample $x_j$ contains $D$ feature channels, $x_j^1, ... , x_j^D$, extracted from the same image patch. Conventional DCF formulations assume the feature channels th have the same spatial resolution, i.e have teh same number of spatial sample points. Unlike previous work, we eliminate the restriction in our formulation and let $N_d$ denote the number of spatial samples in $x_j^d$. In our formulation, the feature channel $x_j^d \in \mathbb{R}^{N_d}$ is viewed as a function $x_j^d[n]$ indexed by the discrete spatial variable $n \in \{0,...N_d-1\}$. The sample space is expressed as $\chi = \mathbb{R}^{N_1} \times ... \times \mathbb{R}^{D}$.


```python
#Author-Vishal Burman
```

## Vectorization and mini-batch

```python
# H1=sigma(W1*X+b1)
# H2=sigma(W2*H1+b2)
# o=softmax(W3*H2+b3)
```

```python
%matplotlib inline
from mxnet import autograd, nd
```

```python
import matplotlib.pyplot as plt
```

## Relu Function

```python
# ReLU(z)= max(z, 0)
```

```python
# Retains only positive elements and discards negative elements
```

```python
x=nd.arange(-8.0, 8.0, 0.1)
x.attach_grad()
```

```python
with autograd.record():
    y=x.relu()
# plt.figure(figsize=(4, 2.5))
# plt.plot(x, y, 'x', 'relu(x)')
plt.plot(x.asnumpy(), y.asnumpy())
```

```python
# When input is negative the derivative of the ReLU function is zero
# When the input is positive the derivative of the ReLU function is one
# When the input is zero the ReLU function is not differentiable and we take left derivative as 0
```

```python
y.backward()
```

```python
plt.plot(x.asnumpy(), x.grad.asnumpy(), 'x', 'grad of ReLU')
```

```python
# The reason for using the ReLU is that its derivatives are particularly well behaved
# They vanish or just let the argument through
# This makes argument better behaved and and reduces the vanishing gradient problem
```

## Sigmoid Function

```python
#sigmoid(x)=1/(1+exp(-x))
# The sigmoid function takes the value from R and transforms it into range(0, 1)
```

```python
with autograd.record():
    y=x.sigmoid()
plt.plot(x.asnumpy(), y.asnumpy(), 'x', 'sigmoid(x)')
```

```python
# The derivative of the sigmoid function is given by:
```

\begin{equation}
\frac{d}{dx} \mathrm{sigmoid}(x) = \frac{\exp(-x)}{(1 + \exp(-x))^2} = \mathrm{sigmoid}(x)\left(1-\mathrm{sigmoid}(x)\right).
\end{equation}

```python
y.backward()
```

```python
plt.plot(x.asnumpy(),x.grad.asnumpy(), 'x', 'grad of sigmoid')
```

```python
# When the input is 0, the derivative of the sigmoid function reaches a max of 0.25
# As input diverges from 0 the derivative approaches 0
```

## Tanh function

```python
# Like the sigmoid function the tanh function squashes its input and transforms the element into the interval of
# (-1, 1)
```

\begin{equation}
\text{tanh}(x) = \frac{1 - \exp(-2x)}{1 + \exp(-2x)}.
\end{equation}

```python
with autograd.record():
    y=x.tanh()
plt.plot(x.asnumpy(), y.asnumpy(), 'x', 'tanh(x)')
```

```python
# When the input nears 0 the function approaches a linear transformation
# The shape is similar to sigmoid function but tanh function exhibits point symmetry
```

```python
# The derivative of the tanh function is:
```

\begin{equation}
\frac{d}{dx} \mathrm{tanh}(x) = 1 - \mathrm{tanh}^2(x).
\end{equation}

```python
# As the input approaches 0 the derivative reaches a max of 1
# As the input diverges from 0 the derivative approaches 0
```

```python

```


# Intertemporal Choice

We setup and solve a very simple generic one-period consumer choice problem over two goods.  We later specialize to the case of intertemporal trade over two periods and choice over lotteries.  The consumer is assumed to have time-consistent preofereces.  Later, in a separate notebook, we look at a a three period model where consumers have time-inconsistent (quasi-hyperbolic) preferences.

The code to generate the static and interactive figures is hidden but can be revealed below.

```python
import numpy as np
import matplotlib.pyplot as plt
from ipywidgets import interact, fixed
```

```python
def U(c1, c2, alpha):
    return (c1**alpha)*(c2**(1-alpha))

def budgetc(c1, p1, p2, I):
    return (I/p2)-(p1/p2)*c1

def indif(c1, ubar, alpha):
    return (ubar/(c1**alpha))**(1/(1-alpha))
```

```python
def find_opt(p1,p2,I,alpha):
    c1 = alpha * I/p1
    c2 = (1-alpha)*I/p2
    u = U(c1,c2,alpha)
    return c1, c2, u
```

```python
alpha = 0.5
p1, p2 = 1, 1
I = 100

pmin, pmax = 1, 4
Imin, Imax = 10, 200
cmax = (3/4)*Imax/pmin
```

```python
def consume_plot(p1=p1, p2=p2, I=I, alpha=alpha):

    c1 = np.linspace(0.1,cmax,num=100)
    c1e, c2e, uebar = find_opt(p1, p2 ,I, alpha)
    idfc = indif(c1, uebar, alpha)
    budg = budgetc(c1, p1, p2, I)

    fig, ax = plt.subplots(figsize=(8,8))
    ax.plot(c1, budg, lw=2.5)
    ax.plot(c1, idfc, lw=2.5)
    ax.vlines(c1e,0,c2e, linestyles="dashed")
    ax.hlines(c2e,0,c1e, linestyles="dashed")
    ax.plot(c1e,c2e,'ob')
    ax.set_xlim(0, cmax)
    ax.set_ylim(0, cmax)
    ax.set_xlabel(r'$c_1$', fontsize=16)
    ax.set_ylabel('$c_2$', fontsize=16)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.grid()
    plt.show()
```

## Choice over two goods

A consumer chooses a consumption bundle to maximize Cobb-Douglas utility

$$U(c_1,c_2) = c_1^\alpha c_2^{\beta}$$

subject to the budget constraint

$$p_1 c_1 + p_2 c_2 \leq I $$

To plot the budget constraint we rearrange to get:

$$c_2 = \frac{I}{p_2} - \frac{p_1}{p_2} c_1$$

Likewise, to draw the indifference curve defined by $u(c_1,c_2) = \bar u$ we solve for $c_2$ to get:

$$c_2 = \left( \frac{\bar u}{c_1^\alpha}\right)^\frac{1}{\beta}$$

For Cobb-Douglas utility the marginal rate of substitution (MRS) between $c_2$ and $c_1$ is:

$$MRS = \frac{U_1}{U_2} = \frac{\alpha}{\beta} \frac{c_2}{c_1}$$

where $U_1 =\frac{\partial U}{\partial c_1}$ and $U_2 =\frac{\partial U}{\partial c_2}$

```python
consume_plot()
```

## The consumer's optimum

$$L(c_1,c_2) = U(c_1,c_2) + \lambda (I - p_1 c_1  - p_2 c_2) $$

Differentiate with respect to $c_1$ and $c_2$ and $\lambda$ to get:

$$ U_1 = \lambda{p_1}$$

$$ U_2 = \lambda{p_2}$$

$$ I = p_1 c_1  + p_2 c_2$$

Dividing the first equation by the second we get the familiar necessary tangency condition for an interior optimum:

$$MRS = \frac{U_1}{U_2} =\frac{p_1}{p_2}$$

Using our earlier expression for the MRS of a Cobb-Douglas indifference curve, substituting this into the budget constraint and rearranging then allows us to solve for the **Marshallian demand functions**:

$$c_1(p_1,p_2,I)=\frac{\alpha}{\alpha+\beta} \frac{I}{p_1}$$

$$c_1(p_1,p_2,I)=\frac{\beta}{\alpha+\beta} \frac{I}{p_2}$$

Interactive plot with sliders (visible if if running on a notebook server):

```python
interact(consume_plot,p1=(pmin,pmax,0.1),p2=(pmin,pmax,0.1), I=(Imin,Imax,10),alpha=(0.05,0.95,0.05));
```

    interactive(children=(FloatSlider(value=1.0, description='p1', max=4.0, min=1.0), FloatSlider(value=1.0, descr…

## The expenditure function

The indirect utility function:

$$v(p_1,p_2,I) = u(c_1(p_1,p_2,I),c_2(p_1,p_2,I))$$

$$c_1(p_1,p_2,I)=\alpha \frac{I}{p_1}$$

$$c_1(p_1,p_2,I)=(1-\alpha)\frac{I}{p_2}$$

$$v(p_1,p_2,I) = I \cdot  \alpha^\alpha (1-\alpha)^{1-\alpha} \cdot
\left [ \frac{p_2}{p_1} \right ]^\alpha \frac{1}{p_2} $$

$$E(p_z,p_2,\bar u) = \frac{\bar u}{\alpha^\alpha (1-\alpha)^{1-\alpha} \cdot
\left [ \frac{p_2}{p_1} \right ]^\alpha \frac{1}{p_2}}$$

To minimize expenditure needed to achieve level of utility $\bar u$ we solve:

$$\min_{c_1,c_2} p_1 c_1 + p_2 c_2 $$

s.t.

$$u(c_1, c_2) =\bar u$$

The first order conditions are identical to those we got for the utility maximization problem, and hence we have the same tangency

$$\frac{U_1}{U_2} = \frac{\alpha}{1-\alpha} \frac{c_2}{c_1} = \frac{p_1}{p_2}$$

From this we can solve:   $$c_2 = \frac{p_1}{p_2} \frac{1-\alpha}{\alpha} c_1  $$

Now substitute this into the constraint to get:

$$ u \left ( c_1, \frac{p_1}{p_2} \frac{1-\alpha}{\alpha} c_1  \right )=\bar u$$

$$ u \left ( c_1, \frac{p_1}{p_2} \frac{1-\alpha}{\alpha} c_1  \right )=\bar u$$

## Intertemporal Consumption choices

We now look at the special case of intertemporal consumption, or consumption of the same good (say corn) over two periods.  As modeled below the consumer's income is now given by the market value of and endowment bundle $(y_1,y_2)$.

The variables $c_1$ and $c_2$ now refer to consumption of (say corn) in period 1 and period 2.

As is typical of intertemporal maximization problems we will use a time-additive utility function.  The consumer who has access to a competitive financial market (they can borrow or save at interest rate $r$) maximizes:

$$U(c_1,c_2) = u(c_1) + \delta u(c_2)$$

subject to the intertemporal budget constraint:

$$ c_1 + \frac{c_2}{1+r}  = y_1 + \frac{y_2}{1+r} $$

This is just like an ordinary utility maximization problem with prices $p_1 = 1$ and $p_2 =\frac{1}{1+r}$.  Think of it this way, the price of corn is \$1 per unit in each period, but \$1 in period 1 can be placed into savings that will grow to $(1+r)$ dollars in period 2.  That means that from the standpoint of period 1 owning one unit of corn (or one dollar worth of corn) in period 2 is the equivalent of owning $\frac{1}{1+r}$ units of corn today (because placed in savings that amount of period 1 corn would grow to $\frac{1+r}{1+r} = 1$ units of corn in period 2).

The first order necessary condition for an interior optimum is:

$$u'(c_1^*) = \delta u'(c_2^*)$$

Let's adopt the widely used Constant-Relative Risk Aversion (CRRA) felicity function of the form:

$$
\begin{equation}
u\left(c_{t}\right)=\begin{cases}
\frac{c^{1-\rho}}{1-\rho}, & \text{if } \rho>0 \text{ and } \rho \neq 1 \\
ln\left(c\right) & \text{if } \rho=1
\end{cases}
\end{equation}
$$

The first order condition then becomes simply

$${c_1^*}^{-\rho} = \delta (1+r) {c_2^*}^{-\rho}$$

or

$$c_2^* = \left [\delta (1+r) \right]^\frac{1}{\rho}c_1^* $$

From the binding budget constraint we also have

$$c_2^* = E[y](1+r)-c_1^*(1+r)$$

where $E[y] = y_1 + \frac{y_2}{1+r}$

Solving for $c_1^*$ (from the FOC and this binding budget):

$$c_1^*  = \frac{E[y]}{1+\frac{\left [\delta (1+r) \right]^\frac{1}{\rho}}{1+r}}$$

If we simplify to the simple case where $\delta =\frac{1}{1+r}$, where the consumer discounts future consumption at the same rate as the market interest rate then at an optimum the consumer will keep their consumption flat at $c_2^* = c_1^*$.  If we specialize further and assume that $r=0$ then the consumer will set  $c_2^* = c_1^* =\frac{y_1+y_2}{2}$

## Saving and borrowing visualized

Let us visualize the situation.  The consumer has CRRA preferences as described above (summarized by parameters $\delta$ and $\rho$ and starts with an income endowment $(y_1, y_2)$. The market cost of funds is $r$.

### A savings case
In the diagram below the consumer is seen to be saving, i.e.  $s_1^* = y_1 - c_1^* >0$.

```python
consume_plot2(r, delta, rho, y1, y2)
```

```python
def u(c, rho):
    return (1/rho)* c**(1-rho)

def U2(c1, c2, rho, delta):
    return u(c1, rho) + delta*u(c2, rho)

def budget2(c1, r, y1, y2):
    Ey = y1 + y2/(1+r)
    return Ey*(1+r) - c1*(1+r)

def indif2(c1, ubar, rho, delta):
    return  ( ((1-rho)/delta)*(ubar - u(c1, rho)) )**(1/(1-rho))
```

```python
def find_opt2(r, rho, delta, y1, y2):
    Ey = y1 + y2/(1+r)
    A = (delta*(1+r))**(1/rho)
    c1 = Ey/(1+A/(1+r))
    c2 = c1*A
    u = U2(c1, c2, rho, delta)
    return c1, c2, u
```

Parameters for default plot:

```python
rho = 0.5
delta = 1
r = 0
y1, y2 = 80, 20

rmin, rmax = 0, 1
cmax = 150
```

```python
def consume_plot2(r, delta, rho, y1, y2):

    c1 = np.linspace(0.1,cmax,num=100)
    c1e, c2e, uebar = find_opt2(r, rho, delta, y1, y2)
    idfc = indif2(c1, uebar, rho, delta)
    budg = budget2(c1,  r, y1, y2)

    fig, ax = plt.subplots(figsize=(8,8))
    ax.plot(c1, budg, lw=2.5)
    ax.plot(c1, idfc, lw=2.5)
    ax.vlines(c1e,0,c2e, linestyles="dashed")
    ax.hlines(c2e,0,c1e, linestyles="dashed")
    ax.plot(c1e,c2e,'ob')
    ax.vlines(y1,0,y2, linestyles="dashed")
    ax.hlines(y2,0,y1, linestyles="dashed")
    ax.plot(y1,y2,'ob')
    ax.text(y1-6,y2-6,r'$y^*$',fontsize=16)
    ax.set_xlim(0, cmax)
    ax.set_ylim(0, cmax)
    ax.set_xlabel(r'$c_1$', fontsize=16)
    ax.set_ylabel('$c_2$', fontsize=16)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.grid()
    plt.show()
```

Interactive plot with sliders (visible if if running on a notebook server):

```python
consume_plot2(r, delta, rho, y1, y2)
```

```python
c1e, c2e, uebar = find_opt2(r, rho, delta, y1, y2)
```

In this particular case the consumer consumes:

```python
c1e, c2e
```

    (50.0, 50.0)

Her endowment is

```python
(y1, y2)
```

    (80, 20)

```python
y1,y2
```

And she therefore in period 1 saves $y_1-c_1$

```python
y1-c1e
```

    30.0

### A borrowing case
In the diagram below the consumer is seen to be borrowing, i.e.  $s_1^* = y_1 - c_1^* <0$.

```python
y1,y2 = 20,80
consume_plot2(r, delta, rho, y1, y2)
```

### Interactive plot

```python
interact(consume_plot2, r=(rmin,rmax,0.1), rho=fixed(rho), delta=(0.5,1,0.1), y1=(10,100,1), y2=(10,100,1));
```

    interactive(children=(FloatSlider(value=0.0, description='r', max=1.0), FloatSlider(value=0.7, description='de…

```python

```


###### Content under Creative Commons Attribution license CC-BY 4.0, code under MIT license (c) 2014 F.J.Gonzales. Portions of the code adopted from the #numericalmooc materials, also under CC-BY.

# The French Connec... eh solution

## Solving the 1-D wave equation

Welcome to this bonus notebook that ties into the second module of ["Practical Numerical Methods with Python"](http://openedx.seas.gwu.edu/courses/GW/MAE6286/2014_fall/about).

In the first notebook of the second module, we were introduced to the numerical solutions of *partial differential equations (PDEs)*. The equation used in that notebook was the 1-D linear convection equation, AKA *the advection equation*.

However, in this notebook we will explore the 1-D wave equation shown below:

\begin{equation}\frac{\partial^2 u}{\partial t^2}= c^2 \frac{\partial^2 u}{\partial x^2}.
\end{equation}

The wave equation is second order in both spatial and temporal dimensions.

If the initial displacement $\ f(x)$ and a velocity $\ g(x)$ are specified for the wave equation, then the solution of the system is:
\begin{equation}\ u(x,t) = \frac{1}{2}[f(x-ct)+f(x+ct)]+\frac{1}{2c}\int{g(\tau){\partial \tau}}\end{equation}
This solution is known as the d'Alembert solution for the 1-D wave equation.

For this notebook we will focus on the special case where the initial velocity is zero, $g(x)=0$
and the solution is simplified to the equation below:
\begin{equation}\ u(x,t)=\frac{1}{2}[f(x-ct)+f(x+ct)]\end{equation}

## The Guitar pick example

A classical example of the 1-D wave equation is when simulating the motion of a guitar string after it is plucked.

###### Image credit: G. Everstine, 2012 (Fig. 14, page 23).

As the figure above shows, the original triangular displacement splits into two waves traveling in opposite directions.  One can also see that each of the travelling waves is half the height of the original function, which corresponds to d'Alembert's solution shown previously.

## Wave motion for a bar

Now that we have established the solution for the 1-D wave equation, we will use this to simulate the longitudinal wave motion on a finite length bar.

### Problem Description

Let's say we have a bar of finite length as shown below. The length is $10 \rm{m}$ in the $x$ direction.  As can be seen, the system ends with a dashpot, also known as a damper; this dashpot is an important part of this problem because we will see how this will create a non-reflecting boundary condition.

###### Image credit: G. Everstine, 2012 (Fig. 22 in page 30).

In addition to the problem description, the initial displacement $f(x)$ is defined below:
\begin{equation}
u(x,0)=\begin{cases}2-2(x-5)^2 & \text{where } 4\leq x \leq 6,\\
0 & \text{everywhere else in } (0, 10)
\end{cases}
\end{equation}

### Discretizing the equation

Since the 1-D wave equation is second order in time and space, we will discretize the equation using central difference for both dimensions.  Below is the discretized equation:

\begin{equation}\frac{u_i^{n+1}-2u_i^n+u_i^{n-1}}{c^2\Delta t^2} =  \frac{u_{i+1}^n - 2u_i^n+u_{i-1}^n}{\Delta x^2} \end{equation}

Solving for the unknown in the discretized equation we get the following:

\begin{equation}\ u_i^{n+1}=-u_i^{n-1}+2u_i^n+C^2(u_{i+1}^n-2u_i^{n}+u_{i-1}^n) \end{equation}

In this equation $C= c\frac{\Delta t}{\Delta x}$, which equates to the CFL number.

In addition to discretizing the wave equation, we also want to take a look at our boundary conditions.  On the left-hand-side boundary, the bar is fixed, which would correlate with a Dirichlet boundary where the displacement is $u=0$.

The right-hand-side boundary is more tricky and that is where the idea of the nonreflecting boundary condition comes into play.

If we look at a simplified version of d'Alembert's solution, $f(x-ct)+f(x+ct),$ we can see that the first term represents the wave that is moving towards the right boundary while the latter term represents the wave that bounces away from the right boundary.  In the bar we are analyzing, the dashpot at the right-hand-side means that the second return wave term would go to zero.

Differentiating the first term of D'Alembert's solution in terms of space and time would result in the following PDEs:
\begin{equation}\frac{\partial u}{\partial x}= f^\prime,\frac{\partial u}{\partial t}= -cf^\prime\end{equation}

Rearranging the above equations gives the boundary condition we have to enforce on the right-hand side:

\begin{equation}\frac{\partial u}{\partial x}=-\frac{1}{c}\frac{\partial u}{\partial t}\end{equation}

Discretizing our non-reflecting boundary:

\begin{equation}\frac{u_i^n-u_{i-1}^n}{\Delta x} = \frac{-1}{c}\frac{u_i^{n+1}-u_i^n}{\Delta t}\end{equation}

and isolating for the end boundary:

\begin{equation}u_i^n= -\frac{\Delta x}{c\Delta t}(u_i^{n+1}-u_i^n)+u_{i-1}^n\end{equation}

### Lets rev up some code!

The first thing we always do is to import the necessary libraries for this problem.  In this case we will use bothy Numpy and Sympy. We will also use Matplotlib for plotting our displacement.

```
import numpy as np
import sympy as sp
from sympy.utilities.lambdify import lambdify
import matplotlib.pyplot as plt
%matplotlib inline
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

We then create the initial condition for the bar.

```
# initial conditions and parameters
nx= 50
x= sp.symbols('x')
func= 2-2*(x-5)**2
init_func= lambdify((x), func)
x_array= np.linspace(0,10,nx+1)
u_initial= np.asarray([init_func(x0) for x0 in x_array])
```

Lets plot the initial function so we can start off on the right foot.

```
plt.plot(x_array, u_initial, color='#003366', ls='--', lw=3)
plt.xlim(0,10)
plt.ylim(0,3)
plt.xlabel('x')
plt.ylabel('Displacement')
```

Now we can define a function to solve for the displacement.

```
def barsolver(u_init, T, C, nx):
    ''' Returns the displacement of the wave equation for a bar.

    Parameters
    ----------
    u_init: array of float
        initial displacement of bar
    T: integer
        final time for calculation
    C: integer
        CFL number for stability
    nx: integer
        number of gridsteps

    Returns
    -------
    u: array of float
        final displacement of bar wave motion
    '''
    #initial parameters
    c= 2
    C2= C**2
    dx= 10./nx
    dt = C*dx/c
    nt= int(round(T/dt))

    #create arrays
    u= np.zeros(nx+1)    #array holding u for u[n+1]
    u_1= np.zeros(nx+1)  #array holding u for u[n]
    u_2= np.zeros(nx+1)  #array holding u for u[n-1]

    u_1[4/dx:6/dx+1]= u_init[4/dx:6/dx+1]

    # Loop for first time step
    n = 0
    for i in range(1, nx):
        u[i] = u_1[i] + 0.5*C2*(u_1[i-1] - 2*u_1[i] + u_1[i+1])
    # Enforce boundary conditions
    u[0] = 0;  u[-1] = -dx/(c*dt)*(u[i]-u_1[i])+u[-2]

    # Switch variables before next step
    u_2[:], u_1[:] = u_1, u

    # Loops for subsequent time steps
    for n in range(1, nt):
        for i in range(1, nx):
            u[i] = - u_2[i] + 2*u_1[i] + C2*(u_1[i-1] - 2*u_1[i] + u_1[i+1])

        # Enforce boundary conditions
        u[0] = 0;  u[-1] = -dx/(c*dt)*(u[i]-u_1[i])+u[-2]

        # Switch variables before next step
        u_2[:], u_1[:] = u_1, u

    return u
```

Lets run the solver for $t=0$ and plot the displacement.

```
u_ans= barsolver(u_initial,0,1,50)
```

```
plt.plot(x_array,u_ans)
plt.xlim(0,10)
plt.ylim(-3,3)
```

As we expected, the displacement is equal to the initial parabola.

```
u_ans= barsolver(u_initial,2,1,50)  #solving at t= 2
```

```
plt.plot(x_array,u_ans)
plt.xlim(0,10)
plt.ylim(-2,2)
```

```
u_ans= barsolver(u_initial,4,1,50)  #solving at t= 4
```

```
plt.plot(x_array,u_ans)
plt.xlim(0,10)
plt.ylim(-2,2)
```

As we can see from the plots, the wave moving in the right direction has been absorbed while the left moving wave has flipped over and is now moving in the right direction.

Our code has expressed correctly the non-reflecting boundary!

##### Now your turn

What do you think will happen as more time passes?

You will need to plot the displacement for t= 6, 8, 10 and check if what you expected was correct.

## Reference

* Barba, Lorena A., et al. "MAE 6286 Practical Numerical Methods with Python," GW Open edX, The George Washingtion University, 2014. http://openedx.seas.gwu.edu/courses/GW/MAE6286/2014_fall/about .

* Chapra, Steven C and Canale, Raymond P., "Numerical Methods for Engineers," McGraw-Hill Education, Inc., New York, 2010.

* Everstine, Gordon C. "Analytical Solution of Partial Differential Equations," George Washington University, 2012. [PDF](http://gwu.geverstine.com/pdenum.pdf)

* Fitzpatrick, Richard. "Introduction: Wave Equation," University of Texas, 2006. [Wave Equation](http://farside.ph.utexas.edu/teaching/329/lectures/node89.html).

* Strauss, Walter A., "Partial Differential Equations: An Introduction," Wiley, Inc., New York, 2008.

---

###### The cell bellow loads the style of the notebook

```
from IPython.core.display import HTML
css_file = '../../styles/numericalmoocstyle.css'
HTML(open(css_file, "r").read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 750px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
margin-top:0.8em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Alegreya Sans' sans-serif;
    line-height: 140%;
    font-size: 125%;
    font-weight: 400;
    width:600px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Nixie One', serif;
    font-style:regular;
    font-weight: 400;
    font-size: 45pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}
.text_cell_render h2 {
    font-family: 'Nixie One', serif;
    font-weight: 400;
    font-size: 30pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.1em;
    margin-top: 0.3em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Nixie One', serif;
    margin-top:16px;
	font-size: 22pt;
    font-weight: 600;
    margin-bottom: 3px;
    font-style: regular;
    color: rgb(102,102,0);
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Nixie One', serif;
    font-size: 14pt;
    text-align: center;
    margin-top: 0em;
    margin-bottom: 2em;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Nixie One', sans-serif;
    font-weight: 400;
    font-size: 16pt;
    color: rgb(163,0,0);
    font-style: italic;
    margin-bottom: .1em;
    margin-top: 0.8em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'PT Mono', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

.CodeMirror{
        font-family: "PT Mono";
        font-size: 90%;
}

</style>


# Plates geometry

## Init symbols for *sympy*

```python
from sympy import *
from geom_util import *
from sympy.vector import CoordSys3D
import matplotlib.pyplot as plt
import sys
sys.path.append("../")

%matplotlib inline

%reload_ext autoreload
%autoreload 2
%aimport geom_util
```

```python
# Any tweaks that normally go in .matplotlibrc, etc., should explicitly go here
%config InlineBackend.figure_format='retina'
plt.rcParams['figure.figsize'] = (12, 12)

plt.rc('text', usetex=True)

plt.rc('font', family='serif')

init_printing()
```

```python
N = CoordSys3D('N')
alpha1, alpha2, alpha3 = symbols("alpha_1 alpha_2 alpha_3", real = True, positive=True)
```

## Cartesian coordinates

```python
x = alpha1
y = alpha2
z = alpha3

r = x*N.i + y*N.j + z*N.k
```

```python
R1=r.diff(alpha1)
R2=r.diff(alpha2)
R3=r.diff(alpha3)
```

```python
R1
```

```python
R2
```

```python
R3
```

## Draw

```python
import plot
%aimport plot

alpha1_x = lambdify([alpha1], x, "numpy")
alpha3_z = lambdify([alpha3], z, "numpy")

L_num = 2
h_num = 0.1

x1_start = 0
x1_end = L_num
x3_start = -h_num/2
x3_end = h_num/2

def alpha_to_x(a1, a2, a3):
    x=alpha1_x(a1)
    z=alpha3_z(a3)
    return x, 0, z

plot.plot_init_geometry_2(x1_start, x1_end, x3_start, x3_end, alpha_to_x)
```

```python
%aimport plot

R3_1=R3.dot(N.i)
R3_3=R3.dot(N.k)

R3_1_x = lambdify([alpha1], R3_1, "numpy")
R3_3_z = lambdify([alpha3], R3_3, "numpy")

def R3_to_x(a1, a2, a3):
    x=R3_1_x(a1)
    z=R3_3_z(a3)
    return x, 0, z

plot.plot_vectors(x1_start, x1_end, 0, alpha_to_x, R3_to_x)
```

```python
%aimport plot

R1_1=R1.dot(N.i)
R1_3=R1.dot(N.k)

R1_1_x = lambdify([alpha1], R1_1, "numpy")
R1_3_z = lambdify([alpha1], R1_3, "numpy")

def R1_to_x(a1, a2, a3):
    x=R1_1_x(a1)
    z=R1_3_z(a1)
    return x, 0, z

plot.plot_vectors(x1_start, x1_end, 0, alpha_to_x, R1_to_x)
```

### Lame params

```python
H1=S(1)
H2=S(1)
H3=S(1)

H=[H1, H2, H3]
DIM=3
dH = zeros(DIM,DIM)
for i in range(DIM):
    dH[i,0]=H[i].diff(alpha1)
    dH[i,1]=H[i].diff(alpha2)
    dH[i,2]=H[i].diff(alpha3)

dH

```


# Series de Fourier en Forma Compleja

[](https://colab.research.google.com/github/ivan-jgr/computacion-cientifica/blob/main/Laboratorios/Laboratorio-5.ipynb)

Definimos la serie de Fourier de una función periodica $f(t)$ de la siguiente forma:

$$
\hat{f}(t) = \sum_{n = -\infty}^\infty c_n e^{i \omega_n t},
$$

donde $c_n = \displaystyle \dfrac{1}{2L} \int_{-L}^L f(t) e^{-i \omega_n t} dt$.

En este laboratorio utilizaremos la librería `cmath` para manipular numeros complejos, calcular la serie de Fourier en forma compleja y utilizarla para resolver ecuaciones diferenciales ordinarias.

```python
import cmath # Libreria para manipular complejos
```

Para crear un número complejo $z = a + jb$ (note que se usa $j$ para representar la unidad imaginaria) hacemos `z = complex(a, b)`

### Aritmética de Complejos

```python
z = complex(3, 2) # creamos el complejo 3 + 2j
w = complex(1, -2) # creamos el complejo 1 - 2j

# puede obtener a la parte real de un complejo z haciendo z.real
print(f"La parte real de z es: {z.real}")
# puede obtener a la parte imaginaria de un complejo z haciendo z.imag
print(f"La parte imaginaria de z es: {z.imag}")
# Puede calcular el módulo de un complejo w haciendo abs(w)
print(f"El módulo de w es: {abs(w)}")
# Puede calcular el argumento (en radianes) de un complejo z haciendo phase(z)
print(f"El argumento de z es: {cmath.phase(z)}")
# puede sumar, restar, multiplicar  dividir complejo
print(f"El resultado de z + w es: {z + w}")
print(f"El resultado de zw es: {z*w}")
```

    La parte real de z es: 3.0
    La parte imaginaria de z es: 2.0
    El módulo de w es: 2.23606797749979
    El argumento de z es: 0.5880026035475675
    El resultado de z + w es: (4+0j)
    El resultado de zw es: (7-4j)

```python
cn = complex(3, 2) # Creamos el complejo 3 + 2j
cn_conjugado = complex(cn.real, -cn.imag) # su conjugado
print('El conjugado de  {} es {}'.format(cn, cn_conjugado))

# También podemos crear complejos de la siguiente forma
j = complex(0, 1)
dn = 3 + 2*j # cuidado con redefinir j

print('Complejo dn = {}'.format(dn))
```

    El conjugado de  (3+2j) es (3-2j)
    Complejo dn = (3+2j)

### Serie de Fourier en Forma Compleja

#### <ins> Problema 1 </ins>

1. Implemente la función `cn(f, L, n)` que calcula en coeficiente de la serie Compleja de Fourier:
\begin{equation}c_n = \dfrac{1}{2L} \int_{-L}^L f(t) e^{-i \omega_n t} dt. \label{cn} \tag{1} \end{equation}

 Recuerde que si $f(t)$ es compleja, i.e. es de la forma $f(t) = u(t) + i v(t)$, entonces: $$\int f(t) dt = \int \operatorname{Re}(f(t)) dt + i \int \operatorname{Im}(f(t)) dt$$

 Es decir que puede integrar la parte real del integrado, luego la parte imaginaria y luego formar el complejo resulte. El método `quad` que hemos usado para integrar **no** puede manejar terminos complejos, por lo que deberá integrar por separado las partes real e imaginaria de \eqref{cn}.

2. Implemente la función `Sn(f, L, N)` que calcula la $N$-ésima suma parcial de Fourier:
$$S_N (t) = \sum_{n = -N}^N c_n e^{i \omega_n t}.$$

```python
# la función exponencial la puede encontrar como np.exp y sí acepta valores complejos
import numpy as np
from scipy.integrate import quad

def cn(f, L, n):
    """
    Calcula el coeficiente cn para la serie de Fourier compleja.

    :param f: es la una función
    :param L: es el semiperíodo de f
    :param n: es un entero (positivo o negativo)

    returns: el coeficiente cn de Fourier descrito por la formula anterior.
    """
    pass
```

```python
def Sn(f, L, N):
    """
    Devuelve la suma parcial de Fourier.
    """
    pass
```

```python
# Verifiquemos que Sn nos devuelva complejos con parte imaginaria igual a cero
f_test = lambda x: abs(x)

f_test_serie = Sn(f_test, 1, 10)

t = np.linspace(-1, 1)

f_test_serie(t).imag
```

    array([-8.20588570e-18, -1.65326325e-03, -5.27721283e-04,  1.48481473e-03,
            1.00167383e-03, -1.16508065e-03, -1.37356733e-03,  7.26638248e-04,
            1.60551011e-03, -2.14159758e-04, -1.67386986e-03, -3.20139149e-04,
            1.57168151e-03,  8.21819557e-04, -1.30935688e-03, -1.23976600e-03,
            9.13623833e-04,  1.53139453e-03, -4.24803019e-04, -1.66699157e-03,
           -1.07300339e-04,  1.63274132e-03,  6.28471027e-04, -1.43213350e-03,
           -1.08560774e-03,  1.08560774e-03,  1.43213350e-03, -6.28471027e-04,
           -1.63274132e-03,  1.07300339e-04,  1.66699157e-03,  4.24803019e-04,
           -1.53139453e-03, -9.13623833e-04,  1.23976600e-03,  1.30935688e-03,
           -8.21819557e-04, -1.57168151e-03,  3.20139149e-04,  1.67386986e-03,
            2.14159758e-04, -1.60551011e-03, -7.26638248e-04,  1.37356733e-03,
            1.16508065e-03, -1.00167383e-03, -1.48481473e-03,  5.27721283e-04,
            1.65326325e-03,  8.20588570e-18])

Note que la parte imaginaria de todos los valores valuados en la serie es cero (o muy cercana a cero por la precisión del metodo `quad`) ¿Por qué?

```python
import matplotlib.pyplot as plt
from matplotlib import rcParams

rcParams['figure.figsize'] = 10, 6
plt.style.use('seaborn-whitegrid')

t = np.linspace(-2, 2, 500)
plt.plot(t, f_test_serie(t).real) # Note que graficamos valores reales
plt.show()
```

    [<matplotlib.lines.Line2D at 0x7fc67ececbd0>]

## Serie Compleja de Fourier para resolver EDOs

Nuestro objetivo es resolver la ecuación diferencial (encontrar la solución al estado estable para la carga $q(t)$) para un circuito RLC con una fuente de voltaje descrito por una función periódica $v(t)$:

La carga en el capacitor está modelada por la ecuación:
\begin{equation} L \frac{d^2q}{dt^2} + R\frac{dq}{dt} + \frac{1}{C} q = v(t). \label{eq:edo} \tag{1}\end{equation}

Podemos expandir $v(t)$ en su serie compleja de Fourier: $v(t) = \displaystyle\sum_{n = -\infty}^\infty c_n e^{i \omega_n t}$ y al usar el principio de superposición resolver
$$ L \frac{d^2q}{dt^2} + R\frac{dq}{dt} + \frac{1}{C} q = e^{i \omega_n t},$$

que podemos resolver usando coeficientes indeterminados:

- Proponemos $q_n(t) = A_n e^{i \omega_n t}$ y obtendremos que $A_n = \dfrac{C}{(1-LC\omega_n^2) + RC\omega_n i}$

Por simplicidad asumamos que $f(t)$ tiene período $T = 2 \pi$ y entonces:

$$A_n = \dfrac{C}{(1-LCn^2) + RCn i}.$$

La solución a la ecuación $\eqref{eq:edo}$ es entonces:

$$q(t) = \sum_{n = -\infty}^\infty c_n A_n e^{i \omega_n t} = \sum_{n = - \infty}^\infty c_n \dfrac{C}{(1-LCn^2) + RCn i} e^{i n t}.$$

```python
# valores de L, R, C
L, R, C = 1, 1, 1

def An(n):
    # denominador de An
    den = complex(1-L*C*n**2, R*C*n)
    return C / den

print(f"El coeficiente A3 = {An(3)}")
```

    El coeficiente A3 = (-0.1095890410958904-0.0410958904109589j)

```python
def f(t):
    if t > 0:
        return 1
    else:
        return -1

# Terminos de la serie
N = 30

q_sol = lambda x: sum([cn(f, np.pi, k)*An(k)*np.exp(-j*k*x) for k in range(-N-1, N+1)])
```

```python
rcParams['figure.figsize'] = 10, 6

v = Sn(f, np.pi, 30)

t = np.linspace(0, 6*np.pi, 200)
plt.plot(t, v(t).real, label='$v(t)$', color='green')

plt.plot(t, q_sol(t).real, label='$q(t)$', color='red')

plt.xlabel('Tiempo en segundos')
plt.legend()
plt.show()
```

```python
# Veamos como evoluciona la solución al variar el tiempo
from ipywidgets import interact, FloatSlider
rcParams['figure.figsize'] = 10, 6

def plot_sol(tmax):
    t = np.linspace(0, tmax, 200)
    plt.plot(t, q_sol(t).real, label='$q(t)$', color='red')
    plt.plot(t, v(t).real, label='$v(t)$', color='green')
    plt.scatter(tmax, q_sol(tmax).real, color='red')

    plt.ylim(-2, 2)
    plt.xlabel('Tiempo en segundos')
    plt.legend(loc='upper left')
    plt.show()

interact(plot_sol, tmax= FloatSlider(min = 0.01, max = 50, step=0.5));
```

    interactive(children=(FloatSlider(value=0.01, description='tmax', max=50.0, min=0.01, step=0.5), Output()), _d…

```python
from matplotlib import rcParams
rcParams['figure.figsize'] = 12, 8

def plot_sol(tmax):
    t = np.linspace(0, tmax, 200)

    ###########################################
    # fila 1 columna 1: Gráfica de la entrada
    plt.subplot(4, 2, 1)
    plt.plot(t, v(t).real, label='$v(t)$', color='green')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    # fila 1 columna 2: Gráfica de la salida
    plt.subplot(4, 2, 2)
    plt.plot(t, q_sol(t).real, label='$q(t)$', color='red')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    ###########################################
    # fila 2 columna 1: Primer armónico
    plt.subplot(4, 2, 3)

    v1 = cn(f, np.pi, 1)*np.exp(-j*1*t) + cn(f, np.pi, -1)*np.exp(-j*(-1)*t)
    plt.plot(t, v1.real, label='primer armónico', color='blue')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

     # fila 2 columna 2: Solucion al primer armónico
    plt.subplot(4, 2, 4)

    q1 = cn(f, np.pi, 1)*An(1)*np.exp(-j*1*t) + cn(f, np.pi, -1)*An(-1)*np.exp(-j*(-1)*t)
    plt.plot(t, q1.real, label='$q_1(t)$', color='orange')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    ###########################################
     # fila 3 columna 2: Tercer armónico
    plt.subplot(4, 2, 5)

    v3 = cn(f, np.pi, 3)*np.exp(-j*3*t) + cn(f, np.pi, -3)*np.exp(-j*(-3)*t)
    plt.plot(t, v3.real, label='tercer armónico', color='black')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    # fila 3 columna 2: Solucion al tercer armónico
    plt.subplot(4, 2, 6)

    q3 = cn(f, np.pi, 3)*An(3)*np.exp(-j*3*t) + cn(f, np.pi, -3)*An(-3)*np.exp(-j*(-3)*t)
    plt.plot(t, q3.real, label='$q_5(t)$', color='gray')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    ###########################################
    # fila 4 columna 1: Quinto armonico
    plt.subplot(4, 2, 7)

    v5 = cn(f, np.pi, 5)*np.exp(-j*5*t) + cn(f, np.pi, -5)*np.exp(-j*(-5)*t)
    plt.plot(t, v5.real, label='quinto armónico', color='y')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

    # fila 4 columna 2: Solucion al quinto armonico
    plt.subplot(4, 2, 8)

    q5 = cn(f, np.pi, 5)*An(5)*np.exp(-j*5*t) + cn(f, np.pi, -5)*An(-5)*np.exp(-j*(-5)*t)
    plt.plot(t, q5.real, label='$q_5(t)$', color='c')

    plt.ylim(-2, 2)
    plt.legend(loc='upper left')

interact(plot_sol, tmax= FloatSlider(min = 0.01, max = 50, step=0.5, value=22));
```

    interactive(children=(FloatSlider(value=22.0, description='tmax', max=50.0, min=0.01, step=0.5), Output()), _d…

Las gráficas anteriores muestran la respuesta a $v(t)$ en la primera fila y luego las respuestas al primer, tercer y quinto armónico. La serie de Fourier nos permite ver como responde el circuito a los componentes de frecuencia específicos en la función de entrada como si fuera un prima que nos permite descomponer la solución $q(t)$ en las respuestas a cada componente de frecuencia presente en $v(t)$.

Además, note la diferencia en amplitud para $q_1(t)$ con respecto a los otros armónicos. ¿Puede explicar por qué la amplitud de este armónico es mayor que el resto?

#### <ins> Problema 2 </ins>

Considere el siguiente sistema masa-resorte:

se aplica una fuerza externa periódica dada por $f(t) = \left| \sin t \right|$ con período $T = 2 \pi$.
1. Encuentre la respuesta (solución al estado estable) para la posición de la masa $x(t)$ usando superposición con $\operatorname{N} = 30$ terminos para la serie.
 - $m$ es la mada del bloque
 - $b$ es la constante de amortigüamiento viscoso
 - $k$ es la constante del resorte

2. Encuentre una expresión en forma de serie para $v(t) = \dfrac{dx}{dt} = \dfrac{d}{dt} \displaystyle \left( \sum_{n = -\infty}^\infty c_n A_n e^{i \omega_n t} \right)$, i.e. derívela término por término.

3. Genere gráficas para $x(t), v(t)$ y $f(t)$.

4. Grafique la solución para el segundo y cuarto armónico.

```python
# Defina la función den entrada f(t)
def f(t):
    pass

# Encuentre la expansion en serie de f(t) y guardela en la variable f_hat
f_hat = None

# Utilíce coeficientes indeterminados y encuentre la solucion al estado estable para la posición de la masa
# Encuentre el coeficiente An como en el ejemplo mostrado
def An(n):
    pass

N = 30
# Escriba la solución como usa serie compleja (desde -N hasta N)
x_sol = None

# Escriba v(t) en forma de serie
v_sol = None
```

```python
# Grafique f(t) junto con x_sol(t) y v_sol(t) en el intervalo [0, 50]

```

```python
# Grafique la respuesta para el segundo y cuarto armónico
```

<hr>

# **Instrucciones Generales**
1. Este laboratorio será realizado de manera *individual*.
2. **Fecha de entrega**: Viernes 17 de Septiembre de 2021. Su solución debe subirla en un archivo ZIP enviado por GES y debe contener el archivo .ipynb con sus respuesta a cada inciso solicitado en cada uno de los *Problemas* indicados en la parte superior.


## Part III: Separable convolutions  [Mobilenet](https://arxiv.org/pdf/1704.04861.pdf)
- Modify the baseline model in part I to use *separable* convolutions, similar to Mobilenet.
- Check number of parameters and compare with the baseline.
- Train classifier.

### Separable convolutions

For example, a 2D conv can be written as a sequence of 2 1D conv, i.e. \begin{equation} y[m,n]=x[m,n]*h[m,n] = h_1[m]*(h_2[n]*x[m,n])\end{equation}

assuming $x$ is a 2D input signal, $h$ is a 2D filter that can be separated into 2 1D filters $h_1$ and $h_2$, and $y$ is the output of convolving $x$ with $h$.

Similarly for 3D case, we apply the separability between feature channel and spatial dimensions (as shown in the figure below on the left), i.e. \begin{equation} y[m,n,p]=x[m,n, p]*h[m,n,p] = h_1[p]*(h_2[m,n]*x[m,n,p])\end{equation}

### Architecture
- baseline from Part I, adapted for separable convolutions; note that this is not exactly the model in Mobilenet paper.
- first conv layer stays the same, kernel = (3,3), stride=1, num_channels = 64
- depthwise conv layers:

>* channel_multiplier = 1
>* strides = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1]

- 1x1 conv layers:

>* num_channels = [128, 128, 128, 256, 256, 256, 512, 512, 512, 512]
>* strides = 1

- the depthwise and 1x1 conv layers are added after the first conv layer, alternating between the two types
- padding: SAME (snt.SAME) for all layers
- num_output_classes = 10
- BatchNorm and ReLU are added after each conv layer as illustrated in the figure above

### Imports

```
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
import time

import tensorflow as tf

# Don't forget to select GPU runtime environment in Runtime -> Change runtime type
device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
  raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))

# we will use Sonnet on top of TF
!pip install -q dm-sonnet
import sonnet as snt

import numpy as np

# Plotting library.
from matplotlib import pyplot as plt
import pylab as pl
from IPython import display
```

```
# Reset graph
tf.reset_default_graph()
```

### Download dataset to be used for training and testing
- Cifar-10 equivalent of MNIST for natural RGB images
- 60000 32x32 colour images in 10 classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck
- train: 50000; test: 10000

```
cifar10 = tf.keras.datasets.cifar10
# (down)load dataset
(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()
```

```
#@title Prepare the data for training and testing
# define dimension of the batches to sample from the datasets
BATCH_SIZE_TRAIN = 64 #@param
BATCH_SIZE_TEST = 100 #@param

# create Dataset objects using the data previously downloaded
dataset_train = tf.data.Dataset.from_tensor_slices((train_images, train_labels))
# we shuffle the data and sample repeatedly batches for training
batched_dataset_train = dataset_train.shuffle(100000).repeat().batch(BATCH_SIZE_TRAIN)
# create iterator to retrieve batches
iterator_train = batched_dataset_train.make_one_shot_iterator()
# get a training batch of images and labels
(batch_train_images, batch_train_labels) = iterator_train.get_next()

# we do the same for test dataset
dataset_test = tf.data.Dataset.from_tensor_slices((test_images, test_labels))
batched_dataset_test = dataset_test.repeat().batch(BATCH_SIZE_TEST)
iterator_test = batched_dataset_test.make_one_shot_iterator()
(batch_test_images, batch_test_labels) = iterator_test.get_next()

# Squeeze labels and convert from uint8 to int32 - required below by the loss op
batch_test_labels = tf.cast(tf.squeeze(batch_test_labels), tf.int32)
batch_train_labels = tf.cast(tf.squeeze(batch_train_labels), tf.int32)
```

```
#@title Preprocessing of data
# Data augmentation used for train preprocessing
# - scale image to [-1 , 1]
# - get a random crop
# - apply horizontal flip randomly

def train_image_preprocess(h, w, random_flip=True):
  """Image processing required for training the model."""

  def random_flip_left_right(image, flip_index, seed=None):
    shape = image.get_shape()
    if shape.ndims == 3 or shape.ndims is None:
      uniform_random = tf.random_uniform([], 0, 1.0, seed=seed)
      mirror_cond = tf.less(uniform_random, .5)
      result = tf.cond(
          mirror_cond,
          lambda: tf.reverse(image, [flip_index]),
          lambda: image
      )
      return result
    elif shape.ndims == 4:
      uniform_random = tf.random_uniform(
          [tf.shape(image)[0]], 0, 1.0, seed=seed
      )
      mirror_cond = tf.less(uniform_random, .5)
      return tf.where(
          mirror_cond,
          image,
          tf.map_fn(lambda x: tf.reverse(x, [flip_index]), image, dtype=image.dtype)
      )
    else:
      raise ValueError("\'image\' must have either 3 or 4 dimensions.")

  def fn(image):
    # Ensure the data is in range [-1, 1].
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    image = image * 2.0 - 1.0
    # Randomly choose a (h, w, 3) patch.
    image = tf.random_crop(image, size=(BATCH_SIZE_TRAIN, h, w, 3))
    # Randomly flip the image.
    image = random_flip_left_right(image, 2)
    return image

  return fn

# Test preprocessing: only scale to [-1,1].
def test_image_preprocess():
  def fn(image):
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    image = image * 2.0 - 1.0
    return image
  return fn
```

### Define classifier using separable convolutions

```
class Mobilenet(snt.AbstractModule):

  def __init__(self, num_classes, name="mobilenet"):
    super(Mobilenet, self).__init__(name=name)
    #############
    #           #
    # YOUR CODE #
    #           #
    #############

  def _build(self, inputs, is_training=None, test_local_stats=False):
    #############
    #           #
    # YOUR CODE #
    #           #
    #############

    return logits
```

```
#@title Get number of parameters in a scope by iterating through the trainable variables
def get_num_params(scope):
  total_parameters = 0
  for variable in tf.trainable_variables(scope):
    # shape is an array of tf.Dimension
    shape = variable.get_shape()
    variable_parameters = 1
    for dim in shape:
      variable_parameters *= dim.value
    total_parameters += variable_parameters
  return total_parameters
```

### Instantiate the model and connect to data

```
# First define the preprocessing ops for the train/test data
crop_height = 24 #@param
cropt_width = 24 #@param
preprocess_fn_train = train_image_preprocess(crop_height, cropt_width)
preprocess_fn_test = test_image_preprocess()

num_classes = 10 #@param
```

```
# Instantiate the model
with tf.variable_scope("mobilenet"):
  mobilenet_model = Mobilenet(num_classes=10)
```

```
# Get predictions from the model; use the corresponding preprocess ops and is_training flag
predictions_mobilenet = mobilenet_model(preprocess_fn_train(batch_train_images), is_training=True)
print (predictions_mobilenet)

test_predictions_mobilenet = mobilenet_model(preprocess_fn_test(batch_test_images), is_training=False)
print (test_predictions_mobilenet)
```

### Get number of parameters and compare with baseline

```
# Can you obtain this number by hand?
print ("Total number of parameters of Mobilenet model")
print (get_num_params("mobilenet"))
```

```
# @title Setup training (same as for baseline)
def get_loss(logits=None, labels=None):
  # We reduce over batch dimension, to ensure the loss is a scalar.
  return tf.reduce_mean(
      tf.nn.sparse_softmax_cross_entropy_with_logits(
          labels=labels, logits=logits))

# Define train and test loss functions
train_loss = get_loss(labels=batch_train_labels, logits=predictions_mobilenet)
test_loss = get_loss(labels=batch_test_labels, logits=test_predictions_mobilenet)

# For evaluation, we look at top_k_accuracy since it's easier to interpret; normally k=1 or k=5
def top_k_accuracy(k, labels, logits):
  in_top_k = tf.nn.in_top_k(predictions=tf.squeeze(logits), targets=labels, k=k)
  return tf.reduce_mean(tf.cast(in_top_k, tf.float32))

def get_optimizer(step):
  """Get the optimizer used for training."""
  lr_init = 0.1 # initial value for the learning rate
  lr_schedule = (40e3, 60e3, 80e3) # after how many iterations to reduce the learning rate
  lr_schedule = tf.to_int64(lr_schedule)
  lr_factor = 0.1 # reduce learning rate by this factor

  num_epochs = tf.reduce_sum(tf.to_float(step >= lr_schedule))
  lr = lr_init * lr_factor**num_epochs

  return tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9)

# Create a global step that is incremented during training; useful for e.g. learning rate annealing
global_step = tf.train.get_or_create_global_step()

# instantiate the optimizer
optimizer = get_optimizer(global_step)

# Get training ops
training_mobilenet_op = optimizer.minimize(train_loss, global_step)

# Retrieve the update ops, which contain the moving average ops
update_ops = tf.group(*tf.get_collection(tf.GraphKeys.UPDATE_OPS))

# Manually add the update ops to the dependency path executed at each training iteration
training_mobilenet_op = tf.group(training_mobilenet_op, update_ops)

# Get test ops
test_acc_mobilenet_op = top_k_accuracy(1, batch_test_labels, test_predictions_mobilenet)

# Function that takes a list of losses and plots them.
def plot_losses(loss_list, steps):
  display.clear_output(wait=True)
  display.display(pl.gcf())
  pl.plot(steps, loss_list, c='b')
  time.sleep(1.0)
```

### Training params

```
# Define number of training iterations and reporting intervals
TRAIN_ITERS = 90e3 #@param
REPORT_TRAIN_EVERY = 10 #@param
PLOT_EVERY = 500 #@param
REPORT_TEST_EVERY = 1000 #@param
TEST_ITERS = 50 #@param
```

### Train the model. Full training gives ~87% accuracy.

```
# Create the session and initialize variables
sess = tf.Session()
sess.run(tf.global_variables_initializer())

train_iter = 0
losses = []
steps = []
for train_iter in range(int(TRAIN_ITERS)):
  _, train_loss_np = sess.run([training_mobilenet_op, train_loss])

  if (train_iter % REPORT_TRAIN_EVERY) == 0:
    losses.append(train_loss_np)
    steps.append(train_iter)
  if (train_iter % PLOT_EVERY) == 0:
    plot_losses(losses, steps)

  if (train_iter % REPORT_TEST_EVERY) == 0:
    avg_acc = 0.0
    for test_iter in range(TEST_ITERS):
      acc = sess.run(test_acc_mobilenet_op)
      avg_acc += acc

    avg_acc /= (TEST_ITERS)
    print ('Test acc at iter {0:5d} out of {1:5d} is {2:.2f}%'.format(int(train_iter), int(TRAIN_ITERS), avg_acc*100.0))
```


$\newcommand{\pr}{\textrm{Pr}}$
$\newcommand{\l}{\left}$
$\newcommand{\r}{\right}$
$\newcommand\given[1][]{\:#1\vert\:}$
$\newcommand{\var}{\textrm{Var}}$
$\newcommand{\mc}{\mathcal}$
$\newcommand{\lp}{\left(}$
$\newcommand{\rp}{\right)}$
$\newcommand{\lb}{\left\{}$
$\newcommand{\rb}{\right\}}$
$\newcommand{\iid}{\textrm{i.i.d. }}$

# 2.1 Belief functions and probabilities

Probabilities are a way to numerically express rational beliefs.
This first section shows that probabilities satisfy some general
features that we would expect a measure of "belief" to have, so
it seems reasonable to use probabilities to represent our belief
in something.

Probability axioms:

**P1**: $0 = \pr\l(\textrm{not } H \given H\r) \leq \pr\l(F \given H\r) \leq \pr\l(H \given H\r) = 1$

**P2**: $\pr\l(F \cup G \given H\r) = \pr\l(F \given H\r) + \pr\l(G \given H\r)$ if $F \cap G = \varnothing$

**P3**: $\pr\l(F \cap G \given H\r) = \pr\l(G \given H\r)\pr\l(F \given G \cap H\r)$

# 2.2 Events, partitions, and Bayes' rule

**Definition 1 (Partition)**: A collection of sets $\left\{H_1, \cdots, H_k\right\}$ is a partition
of another set $\mathcal{H}$ if

1. the events are disjoint: $H_i \cap H_j = \varnothing$ for $i \not= j$
2. the union of the sets is $\mathcal{H}$: $\bigcup_{k=1}^K H_k = \mathcal{H}$

If $\mathcal{H}$ is the set of all possible truths and $\left\{H_1, \cdots, H_k\right\}$ is a partition of
$\mathcal{H}$, then exactly one of $\left\{H_1, \cdots, H_k\right\}$ contains the truth.

Suppose $\left\{H_1, \cdots, H_k\right\}$ is a partition of $\mathcal{H}$, $\textrm{Pr}\left(\mathcal{H}\right) = 1$,
and $E$ is some specific event. The axioms of probability imply:

**Rule of total probability**: $$\sum_{k=1}^{K} \pr\left(H_k\right) = 1$$

**Rule of marginal probability**:
\begin{align}
\pr\left(E\right) &= \sum_{k=1}^K \pr\left(E \cap H_k\right) \\
&= \sum_{k=1}^K \pr\left(E \given H_k\right)\pr\left(H_k\right)
\end{align}

**Bayes' rule**:
\begin{align}
\pr\left(H_j \given E\right) &= \frac{\pr\l(E \given H_j\r)\pr\l(H_j\r)}{\pr\l(E\r)} \\
&= \frac{\pr\l(E \given H_k\r)\pr\l(H_j\r)}{\sum_{k=1}^K \pr\l(E \given H_k\r)\pr\l(H_k\r)}
\end{align}

We would say that $H_k$ has been marginalized out in the denominator.

I think it's worth looking at the formula for Bayes' rule here. The left hand side,
$\pr\left(H_j \given E\right)$, represents that probability that $H_j$ is true given that
event $E$ happened. For us, $E$ is generally data and $H_k$ will be a parameter we are
interested in estimating. On the right side, we have $\pr\l(E \given H_k\r)$
which is the probability of observing the event/data under the parameterization $H_k$.
This is multiplied by $\pr\l(H_j\r)$ which is our prior distribution on $H_j$.
The denominator is a little more tricky. $\pr\l(E\r)$ doesn't mean much on its own. However,
we can rewrite the denominator using the rule of marginal probability as

$$\pr\l(E\r) = \sum_{k=1}^K\pr\l(E \cap H_k\r) = \sum_{k=1}^K \pr\l(E \given H_k\r)\pr\l(H_k\r)$$

Since we think $E$ depends on the $H_i$, we can treat $\pr\l(E\r)$ as a
**marginal density** and rewrite it as a joint distribution (see 2.5 below for more information).
We know $\pr\l(E \given H_k\r)$ and $\pr\l(H_k\r)$ so we can evaluate this expression.

I kind of think of
the numerator as the strength of my belief and the denominator acts to normalize that value
based on how strongly I believe all of the $H_k$.

**Bayes factors**: $\l\{H_1, \cdots, H_k\r\}$ often refer to disjoint hypotheses and $E$ refers
to data. To compare hypotheses post-experimentally, we often calculate the ratio:
\begin{align}
\frac{\pr\l(H_i \given E\r)}{\pr\l(H_j \given E\r)} &=
\frac{\pr\l(E \given H_i\r)}{\pr\l(E \given H_j\r)} \times \frac{\pr\l(H_i\r)}{\pr\l(H_j\r)} \\
&= \textrm{"Bayes factor"}\times \textrm{"prior beliefs"}
\end{align}

This quantity reminds us that Bayes' rule does not determine what our beliefs should be after we
see data, it only tells us how they should change.

# 2.3 Independence

**Definition 2 (Independence)**: Two events $F$ and $G$ are conditionally independent given
$H$ if $\pr\l(F \cap G \given H\r) = \pr\l(F \given H\r)\pr\l(G \given H\r)$.

If $F$ and $G$ are conditionally independent, then knowing $G$ will tell us nothing about $F$.

# 2.4 Random variables

A random variable is an unknown numerical quantity about which we make probability statements.

## 2.4.1 Discrete random variables

Let $Y$ be a random variable and let $\mathcal{Y}$ be the set of all possible values of $Y$.
We say $Y$ is discrete if $\mathcal{Y}$ is countable: $\mathcal{Y} = \l\{y_1, y_2, \cdots\r\}$.

For short, we will write $\pr\l(Y=y\r) = p\l(y\r)$ where $p$ is the probability density function (pdf).
The pdf has the following properties:

1. $0 \leq p\l(y\r) \leq 1$ for all $y \in \mathcal{Y}$
2. $\sum_{y \in \mathcal{Y}} p\left(y\right) = 1$

## 2.4.2 Continuous random variables

If the sample space $\mathcal{Y}$ is roughly equal to $\mathbb{R}$, then we often define
probability distributions for random variables in terms of a cumulative distribution function
(cdf): $F\l(y\r) = \pr\l(Y \leq y\r)$. Note that

* $F\l(\infty\r) = 1$
* $F\l(-\infty\r) = 0$
* $F\l(b\r) \leq F\l(a\r)$ if $b<a$
* $\pr\l(Y > a\r) = 1-F\l(a\r)$
* $\pr\l(a < Y < b\r) = F\l(b\r) - F\l(a\r)$

If $Y$ is a continuous random variable, then there exists a pdf $p$ such that
$$F\l(a\r) = \int_{-\infty}^a p\l(y\r)dy$$

The continuous pdf has analogous characteristics to the discrete pdf:

1. $0 \leq p\l(y\r) \leq 1$ for all $y \in \mathcal{Y}$
2. $\int_{y \in \mathbb{R}} p\left(y\right)dy = 1$

## 2.4.3 Descriptions of distributions

### Mean, mode, median

The **mean** or **expectation** of an unknown quantity $Y$ is given by

* $E\l[Y\r] = \sum_{y\in Y}yp\l(y\r)$ if $Y$ is discrete
* $E\l[Y\r] = \int_{y\in \mathbb{R}}yp\l(y\r)dy$ if $Y$ is continuous

The mean is the center of mass of the distribution. It is generally not equal to

* the **mode**: the probable value of $Y$
* the **median**: the value of $Y$ in the middle of the distribution

The mean is a good quantity to look at because

1. The mean is a scaled value of the total of $\l\{Y_1, \cdots, Y_n\r\}$, and the total
is often a quantity of interest.
2. If you were forced to guess the value of $Y$, guessing the mean would minimize your error
if it was measured as $\l(Y - y_{guess}\r)^2$.
3. In some simple models, the mean contains all of the information about the population that
can be obtained from the data.

### Variance

The variance is a measure of the spread:

\begin{align}
\var\l[Y\r] &= E\l[\l(Y - E\l[Y\r]\r)^2\r] \\
&= E\l[Y^2\r] - E\l[Y\r]^2
\end{align}

The variance is the average squared distance that a sample value $Y$ will be from
the population mean $E\l[Y\r]$. The standard deviation is the square root of the variance
and is on the same scale as the mean.

For a continuous, increasing cdf $F$, the $\alpha$ **quantile** is the $y_\alpha$ such that
$F\left(y_\alpha\right) \equiv \pr\l(Y \leq y_\alpha\r) = \alpha$. The **interquartile** range
is the interval $\l(y_{0.25}, y_{0.75}\r)$ which contains 50% of the mass of the distribution.

## 2.5 Joint distributions

### Discrete distributions

Let

* $\mathcal{Y}_1$, $\mathcal{Y}_2$ be two countable samples spaces
* $Y_1$, $Y_2$, be two random variables, taking values in $\mathcal{Y}_1$, $\mathcal{Y}_2$ respectively

The **joint pdf** or **joint density** of $Y_1$ and $Y_2$ is defined as
$$p_{Y_1 Y_2}\l(y_1, y_2\r) = \pr\l(\l\{Y_1 = y_1\r\} \cap \l\{Y_2 = y_2\r\} \r)$$
for $y_1 \in \mc{Y}_1$, $y_2 \in \mc{Y}_2$.

The **marginal density** can be computed from the joint density:
\begin{align}
p_{Y_1}\l(y_1\r) &\equiv \pr\l(Y_1 = y_1\r) \\
&= \sum_{y_2 \in \mc{Y}_2} \pr\l(\l\{Y_1 = y_1\r\} \cap \l\{Y_2 = y_2\r\} \r) \\
&\equiv \sum_{y_2 \in \mc{Y}_2}p_{Y_1Y_2} \l(y_1, y_2\r)
\end{align}

The **conditional density** of $Y_2$ given $\l\{Y_1 = y_1\r\}$ can be computed from the
joint density and the marginal density of $Y_1$:
\begin{align}
p_{Y_2 \given Y_1}\l(y_2 \given y_1\r) &= \frac{\pr\l(\l\{Y_1 = y_1\r\} \cap \l\{Y_2 = y_2\r\}\r)}{\pr\l(Y_1 = y_1\r)} \\
&= \frac{p_{Y_1Y_2}\l(y_1, y_2\r)}{p_{Y_1}\l(y_1\r)} \\
&= \frac{p_{Y_1Y_2}\l(y_1, y_2\r)}{\sum_{y_2 \in \mc{Y}_2}p_{Y_1Y_2} \l(y_1, y_2\r)}
\end{align}

We often drop the subscripts on the pdf's such that $p_{Y_1}\l(y_1\r)$ becomes $p\l(y_1\r)$ etc.

### Continuous joint distributions

If $Y_1$ and $Y_2$ are continuous, we have a joint cdf $F_{Y_1Y_2}\l(a, b\r) \equiv
\pr\l(\l\{Y_1 \leq a\r\} \cap \l\{Y_2 \leq b\r\}\r)$, there is a function $p_{Y_1Y_2}$ such that
$$F_{Y_1Y_2}\l(a,b\r) = \int_{-\infty}^a \int_{-\infty}^b p_{Y_1Y_2}\l(y_1, y_2\r)dy_2dy_1$$

The function $p_{Y_1Y_2}$ is the joint density of $Y_1$ and $Y_2$. As in the discrete case,
we have
* $p_{Y_1}\l(y_1\r) = \int_{-\infty}^\infty p_{Y_1Y_2}\l(y_1,y_2\r)dy_2$
* $p_{Y_2 \given Y_1}\l(y_2 \given y_1\r) = p_{Y_1Y_2}\l(y_1, y_2\r) / p_{Y_1}\l(y_1\r)$

### Mixed continuous and discrete variables

Let $Y_1$ be discrete and $Y_2$ be continuous. Suppose we have

* a marginal density $p_{Y_1}$ from our beliefs $\pr\l(Y_1=y_1\r)$
* a conditional density $p_{Y_2 \given Y_1}\left(y_2\given y_1\r)$ from
$\pr\l(Y_2 \leq y_2 \given Y_1 = y_1\r) \equiv F_{Y_2 \given Y_1}\l(y_2 \given y_1\r)$

The joint density of $Y_1$ and $Y_2$ is then
$$p_{Y_1Y_2}\l(y_1, y_2\r) = p_{Y_1}\l(y_1\r) \times p_{Y_2 \given Y_1}\l(y_2 \given y_1\r)$$

and has the property that
$$\pr\l(Y_1 \in A, Y_2 \in B\r) = \int_{y_2 \in B} \left\{\sum_{y_1 \in A} p_{Y_1Y_2}\l(y_1, y_2\r)\r\}dy_2$$

In other words, we can use summation and integration to calculate the joint density.

### Bayes rule and parameter estimation

Let $\theta$ be a continuous parameter we want to estimate and let $Y$ be a discrete
data measurement. Bayesian estimation of $\theta$ derives from the calculation of
$p\l(\theta \given y\r)$, where $y$ is the observed value of $Y$. This calculation
first requires the joint density of $\theta$ and $Y$. We can construct the joint density from

* $p\l(\theta\r)$, beliefs about $\theta$
* $p\l(y \given \theta\r)$, beliefs about $Y$ for each value of $\theta$

Having observed $\l\{Y = y\r\}$, we need to compute our updated beliefs about $\theta$:
$$p\l(\theta | y\r) = p\l(\theta, y\r) / p\lp y\rp = p\lp \theta \rp p\lp y \given \theta \rp / p\lp y \rp$$

**This conditional density is called the posterior density of $\theta$.** If $\theta_a$ and $\theta_b$
are two estimates of $\theta$, the posterior probability (density) of $\theta_a$ relative to $\theta_b$,
conditional on $Y=y$, is
\begin{align}
\frac{p\lp \theta_a \given y \rp}{p \lp \theta_b \given y \rp} &=
\frac{p\lp \theta_a \rp p\lp y | \theta_a \rp / p\lp y \rp}{p\lp \theta_b \rp p\lp y | \theta_b \rp / p\lp y \rp} \\
&= \frac{p\lp \theta_a \rp p\lp y | \theta_a \rp}{p\lp \theta_b \rp p\lp y | \theta_b \rp}
\end{align}

To evaluate the **relative** posterior odds of $\theta_a$ and $\theta_b$, we do not need to evaluate
the marginal density $p\lp y \rp$. We see that
$$p\lp \theta | y \rp \propto p\lp \theta \rp p\lp y \given \theta \rp$$

The constant of proportionality is $1 / p\lp y \rp$ which we *could* calculate using
$$p\lp y \rp = \int_\Theta p\lp y, \theta \rp d\theta = \int_\Theta p\lp y \given \theta \rp p\lp \theta \rp d\theta$$

Later we will see that the numerator is more important than this denominator.

# 2.6 Independent random variables

Suppose $Y_1, \cdots, Y_n$ are random variables and that $\theta$ is a parameter describing
the conditions under which the random variables are generated. We say that $Y_1, \cdots, Y_n$
are **conditionally independent** given $\theta$ if for every collection of $n$ sets $\lb A_1, \cdots, A_n \rb$
we have
$$\pr\lp Y_1 \in A_1, \cdots, Y_n \in A_n \given \theta \rp =
\pr\lp Y_1 \in A_1 \given \theta\rp \times \cdots \times \pr \lp Y_n \in A_n \given \theta \rp$$

This tells us that knowing $Y_j$ gives us no further information about $Y_i$ beyond what $\theta$
gives us. Under independence, the joint density of the $Y_i$ condtioned on $\theta$ is
$$p\lp y_1, y_2, \cdots, y_n \given \theta \rp = \prod_{i=1}^n p_{Y_i} \lp y_i \given \theta \rp$$

In this case, we say that $Y_1, \cdots, Y_n$ are **conditionally independent and identically
distributed (i.i.d.)**: $Y_1, \cdots, Y_n \sim \iid p\lp y \given \theta \rp$.

# 2.7 Exchangeability

**Definition 3 (Exchangeable)**: Let $p\lp y_1, \cdots, y_n\rp$ be the joint density of $Y_1, \cdots, Y_n$.
If $p\lp y_1, \cdots, y_n\rp = p\lp y_{\pi_1}, \cdots, y_{\pi_n}\rp$ for all permutations $\pi$ of
$\lb 1, \cdots, n \rb$, then $Y_1, \cdots, Y_N$ are exchangeable. This basically means that
$Y_1, \cdots, Y_N$ are exchangeable if the subscript labels convey no information about the outcomes.

Therom: If $\theta \sim p\lp \theta \rp$ and $Y_1, \cdots, Y_N$ are conditionally i.i.d. given $\theta$,
then marginally (unconditionally on $\theta$), $Y_1, \cdots, Y_N$ are exchangeable.

# 2.8 de Finetti's thereom

We know that if $Y_1, \cdots, Y_n \given \theta$ are i.i.d. and $\theta \sim p\lp \theta \rp$, then
$Y_1, \cdots, Y_N$ are exchangeable. Can we say something about the other direction?

**Theorem 1 (de Finetti)**: Let $\lb Y_1, \cdots, Y_n \rb$ be a potentially
infinite sequence of random variables all having a common sample space $\mc{Y}$.
Let $Y_i \in \mc{Y}$ for all $i \in \lb Y_1, Y_2, \cdots \rb$. Suppose that for
any $n$ our belief model for $Y_1, \cdots, Y_n$ is exchangeable
$$p \lp y_1, \cdots, y_n \rp = p\lp y_{\pi_1}, \cdots, y_{\pi_n} \rp$$
for all permutations of $\lb 1, \cdots, n \rb$. The our model can be written as
$$p\lp y_1, \cdots, y_n \rp = \int \lb \prod_1^np \lp y_i \given \theta \rp \rb p \lp \theta \rp d \theta$$
for some parameter $\theta$, some prior distribution on $\theta$ ($p \lp \theta \rp$), and some sampling
model $p\lp y \given \theta \rp$. The prior and sampling model depend on the form of the belief
model $p\lp y_1, \cdots, y_n \rp$.

The probability distribution $p\lp \theta \rp$ represents our beliefs about the outcomes
of $\lb Y_1, \cdots, Y_n \rb$, induced by our belief model $p\lp y_1, y_2, \cdots \rp$.
More precisely,

* $p\lp \theta \rp$ represents our beliefs about $\lim_{n\rightarrow \infty} Y_i / n$ in the binary case
* $p\lp \theta \rp$ represents our beliefs about $\lim_{n\rightarrow \infty} \lp Y_i \leq c \rp / n$
for each $c$ in the general case

We can summarize as
$$Y_1, \cdots, Y_n \given \theta \textrm{ are i.i.d. and }\theta \sim p\lp \theta \rp \iff
Y_1, \cdots, Y_N \textrm{ are exchangeable for all } n$$

The question is when are $Y_1, \cdots, Y_n$ exchangeable for all $n$? For this to be true,
we need both exchangeability and repeatability. Exchangeability is true when the labels
have no meaning. Repeatability is true when

* $Y_1, \cdots, Y_n$ are outcomes of a repeatable experiment
* $Y_1, \cdots, Y_n$ are sampled from a finite population *with* replacement
* $Y_1, \cdots, Y_n$ are sampled from an infinite population *without* replacement

If $Y_1, \cdots, Y_n$ are exchangeable and sampled from a finite population *without* replacement
of size $N >> n$, then they can be modeled as approximately conditionally i.i.d.

```python

```

readings/A First Course in Bayesian Statistical Methods/2. Belief, probability, and exchangeability (long).ipynb


# Logistic map explorations (Taylor and beyond)

Adapted by Dick Furnstahl from the Ipython Cookbook by Cyrille Rossant.
Lyapunov plot modified on 29-Jan-2019 after discussion with Michael Heinz.

Here we consider the *logistic map*, which illustrates how chaos can arise from a simple nonlinear equation.  The logistic map models the evolution of a population including reproduction and mortality (see Taylor section 12.9).

The defining function for the logistic map is

$ f_r(x) = r x (1-x) \;, $

where $r$ is the control parameter analogous to $\gamma$ for the damped, driven pendulum.  To study the pendulum, we looked at $\phi(t)$ at $t_0$, $t_0 + \tau$, $t_0 + 2\tau$, and so on, where $\tau$ is the driving frequency and we found $\phi(t_0 + \tau)$ from $\phi(t_0)$ by solving the differential equation for the pendulum.  After transients had died off, we characterized the value of $\gamma$ being considered by the periodicity of the $\phi(t_n)$ values: period one (all the same), period two (alternating), chaotic (no repeats) and so on.

Here instead of a differential equation telling us how to generate a trajectory, we use $f_r(x)$:

$\begin{align}
  x_1 = f_r(x_0) \ \rightarrow\ x_2 = f_r(x_1) \ \rightarrow\  x_3 = f_r(x_2) \ \rightarrow\  \ldots
\end{align}$

There will be transients at the beginning, but this sequence of $x_i$ values may reach a point $x = x^*$ such that $f(x^*) = x^*$. We call this a *fixed point*.  If instead it ends up bouncing between two values of $x$, it is period two and we call it a two-cycle.  We can have a cascade of period doubling, as found for the damped, driven pendulum, leading to chaos, which is characterized by the mapping never repeating.  We can make a corresponding bifurcation diagram and identify Lyapunov exponents for each initial condition and $r$.

To adapt this notebook to a different map (such as the sine map), you will need to:
* Change the map function and its derivative.
* Change the range of the $r$ parameter to an appropriate range for the new map.
* Possibly change the initial $x$ value ($x_0$).
* Modify the limits of the plots appropriately.

```python
%matplotlib inline

# standard numpy and matplotlib imports
import numpy as np
import matplotlib.pyplot as plt

```

```python
def logistic(r, x):
    """Logistic map function: f(x) = r x(1-x)
    """
    return r * x * (1.-x)

def logistic_deriv(r, x):
    """Logistic map derivative: f'(x) = r(1-2x)
    """
    return r * (1. - 2.*x)
```

## Explore the logistic map and its fixed points

Start with a simple plot of the logistic function.  **What does changing $r$ do?**

```python
x, step = np.linspace(0,1, num=101, endpoint=True, retstep=True)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)

r_param = 2
ax.plot(x, logistic(r_param, x), 'k-')
ax.set_xlabel('x')
ax.set_ylabel('f(x)')

fig.tight_layout()
```

Make plots showing the approach to fixed points.

**Play with the number of steps taken and the values of $r$.**
1. **Increase the number of steps in the plots to more cleanly identify the long-time trend for that value of $r$.**
2. **Try smaller values of $r$.  Are there always non-zero fixed points for small $r$?**

```python
def plot_system(r, x0, n, ax=None):
    """Plot the function and the y=x diagonal line."""
    t = np.linspace(0,1, num=101)
    ax.plot(t, logistic(r,t), 'k', lw=2)  # black, linewidth 2
    ax.plot([0,1], [0,1], 'k', lw=2)  # x is an array of 0 and 1,
                                      # y is the same array, so this plots
                                      #  a straight line from (0,0) to (1,1).

    # Recursively apply y=f(x) and plot two additional straight lines:
    #   line from (x, x) to (x, y)
    #   line from (x, y) to (y, y)
    x = x0
    for i in range(n):   # do n iterations, i = 0, 1, ..., n-1
        y = logistic(r, x)
        # Plot the two lines
        ax.plot([x,x], [x,y], color='blue', lw=1)
        ax.plot([x,y], [y,y], color='blue', lw=1)
        # Plot the positions with increasing opacity of the circles
        ax.plot([x], [y], 'or', ms=10, alpha=(i+1)/n)
        x = y  # recursive: reset x to y for the next iteration
    ax.set_xlim(0,1)
    ax.set_ylim(0,1)
    ax.set_title(f'$r={r:.1f}$, $x_0={x0:.1f}$')

fig = plt.figure(figsize=(12,12))

ax1 = fig.add_subplot(2,2,1)
# start at 0.1, r parameter is 2.5, take n steps
plot_system(r=0.7, x0=0.1, n=10, ax=ax1)

ax2 = fig.add_subplot(2,2,2)
# start at 0.1, r parameter is 3.2, take n steps
plot_system(r=3.2, x0=0.1, n=10, ax=ax2)

ax3 = fig.add_subplot(2,2,3)
# start at 0.1, r parameter is 2.6, take n steps
plot_system(r=3.5, x0=0.1, n=10, ax=ax3)

ax4 = fig.add_subplot(2,2,4)
# start at 0.1, r parameter is 3.7, take n steps
plot_system(r=3.7, x0=0.1, n=10, ax=ax4)
```

**What periods do these exhibit?  Are any chaotic?**

## Find the unstable point numerically

To find the value of $r$ at which the second fixed point becomes unstable, we must solve simultaneously the equations (see Taylor 12.9 in the "A Test for Stability" subsection):

$\begin{align}
  &f_r(x^*) = x^* \\
  &f_r'(x^*) = -1
\end{align}$

where $f'$ means the derivative with respect to $x$, or

$\begin{align}
  &f_r(x^*) - x^* = 0 \\
  &f_r'(x^*) + 1 = 0
\end{align}$

It is the latter form of the equations that we will pass to the
scipy function `fsolve` to find the values of $r$ and $x*$.

```python
from scipy.optimize import fsolve  # Google this to learn more!

def equations(passed_args):
    """Return the two expressions that must equal to zero."""
    r, x = passed_args
    return (logistic(r, x) - x, logistic_deriv(r, x) + 1)

# Call fsolve with initial values for r and x
r, x = fsolve(equations, (0.5, 0.5))

print_string = f'x* = {x:.10f},  r = {r:.10f}'
print(print_string)
```

**Verify analytically that these are the correct values for the logistic map.**  We use this same procedure for other maps where only numerical solutions are known.

## Now make the bifurcation diagram and calculate Lyapunov exponents

To make a bifurcation diagram, we:
* define an array of closely spaced $r$ values;
* for each $r$, iterate the map $x_{i+1} = f_r(x_i)$ a large number of times (e.g., 1000) to ensure convergence;
* plot the last 100 or so iterations as points at each $r$.

To think about how to calculate a Lyapunov exponent, consider two iterations of the map starting from two values of $x$ that are close together.  Call these initial values $x_0$ and $x_0 + \delta x_0$.  These are mapped by $f_r$ to $x_1$ and  $x_1 + \delta x_1$, then $x_2$ and $x_2 + \delta x_2$, up to $x_n + \delta x_n$.

How are the $\delta x_i$ related?  Each $\delta x_i$ is small, so use a Taylor expansion!
Expanding $f(x)$ about $x_n$:

$\begin{align}
  f(x_i &+ \delta x_i) = &f(x_i) + \delta x_i f'(x_i) + \frac{1}{2}(\delta x_i)^2 f''(x_i) + \cdots \\
\end{align}$

so, neglecting terms of order $(\delta x_i)^2$ and higher,

$\begin{align}
    x_{i+1} + \delta x_{i+1} \approx x_{i+1} + \delta x_i f'(x_i)
    \quad\mbox{or}\quad  \delta x_{i+1} \approx f'(x_i)\, \delta x_i \;.
\end{align}$

Iterating this result we get

$\begin{align}
  \delta x_1 &= f'(x_0)\, \delta x_0  \\
  \delta x_2 &= f'(x_1)\, \delta x_1 = f'(x_1)\times f'(x_0)\, \delta x_0  \\
  & \qquad\vdots  \\
  |\delta x_n| &= \left(\prod_{i=0}^{n-1} |f'(x_i)|\right) |\delta x_0| \;,
\end{align}$

where the last equation gives us the separation of two trajectories after $n$ steps $\delta x_n$ in terms of their
initial separation, $\delta x_0$.  (Why can we just take the absolute value of all the terms?)

We expect that this will vary exponentially at large $n$ like

$\begin{align}
  \left| \frac{\delta x_n}{\delta x_0} \right| = e^{\lambda n}
\end{align}$

and so we define the Lypunov exponent $\lambda$ by

$\begin{align}
  \lambda = \lim_{n\rightarrow\infty} \frac{1}{n} \sum_{i=1}^{n} \ln |f'(x_i)| \;,
\end{align}$

which in practice is well approximated by the sum for large $n$.

If $\lambda > 0$, then nearby trajectories diverge from each other exponentially at large $n$, which corresponds to chaos. However, if the
trajectories converge to a fixed point or a limit cycle, they will get closer together with increasing $n$, which corresponds to $\lambda<0$.

Ok, let's do it!

```python
n = 10000
# Here we'll use n values of r linearly spaced between 2.8 and 4.0
###  You may want to change the range of r for other maps
r = np.linspace(2.8, 4.0, n)

iterations = 1000   # iterations of logistic map; keep last iterations
last = 100          #  where results should have stabilized

x = 0.1 * np.ones(n)  # x_0 initial condition
                      ### (you may want to change for other maps)

lyapunov = np.zeros(n) # initialize vector used for the Lyapunov sums
```

```python
fig = plt.figure(figsize=(8,9))
ax1 = fig.add_subplot(2,1,1)   # bifurcation diagram
ax2 = fig.add_subplot(2,1,2)   # Lyapunov exponent

# Display the bifurcation diagram with one pixel per point x_n^(r) for last iterations
for i in range(iterations):
    x = logistic(r,x)  # just iterate: x_{i+1} = f_r(x_i)

    # Compute the partial sum of the Lyapunov exponent, which is the sum
    #  of derivatives of the logistic function (absolute value)
    lyapunov += np.log(abs(logistic_deriv(r,x)))
    # Display the bifurcation diagram.
    if i >= (iterations-last):  # only plot the last iterations
        ax1.plot(r, x, ',k', alpha=0.25)
ax1.set_xlim(2.8, 4)
ax1.set_xlabel(r'$r$')
ax1.set_title("Bifurcation")

# Display the Lyapunov exponent

# Negative Lyapunov exponent
ax2.plot(r[lyapunov < 0],
         lyapunov[lyapunov < 0] / iterations, 'o',
         color='black', alpha=0.5, ms=0.5)
# Positive Lyapunov exponent
ax2.plot(r[lyapunov >= 0],
         lyapunov[lyapunov >= 0] / iterations, 'o',
         color='red', alpha=0.5, ms=0.5)

# Add a zero line (lightened with alpha=0.5)
ax2.axhline(0, color='k', lw=0.5, alpha=0.5)

ax2.set_xlim(2.8, 4)
ax2.set_ylim(-2, 1)
ax2.set_xlabel(r'$r$')
ax2.set_title("Lyapunov exponent")

plt.tight_layout()
```

We see there is a fixed point for $r < 3$, then two and four periods and a chaotic behavior when $r$ belongs to certain areas of the parameter space.  **Do the values of $r$ where these behaviors occur agree with what you found in the plots with particular $r$ values?**

As for the pendulum, the Lyapunov exponent is positive when the system is chaotic (in red).  **Do these regions look consistent with the characteristics of chaos?**

```python

```