```python
import numpy as np
import matplotlib.pyplot as plt
import sympy
from sympy import init_printing
from sympy.utilities.lambdify import lambdify
```

```python
#Constructing initial value condition
init_printing(use_latex=True)
x, nu, t = sympy.symbols('x nu t')
phi = sympy.exp(-(x-4*t)**2/(4*nu*(t+1)))+sympy.exp(-(x-4*t-2*sympy.pi)**2/(4*nu*(t+1)))

phiprime=phi.diff(x)
phiprime

u=-2*nu*phiprime/phi+4

ufunc=lambdify((t,x,nu),u)
print(ufunc(1,4,3))
```

    3.49170664206445

```python
#Burger's equation
nx=101
dx=2*np.pi/(nx-1)
nt=100
nu=0.07                             #diffusion parameter (viscosity)
#sigma=0.2
dt=dx*nu

x=np.linspace(0, 2*np.pi, nx)
t=0

#Initial Condition
u=ufunc(t,x,nu)
print(u)
```

    [4.         4.06283185 4.12566371 4.18849556 4.25132741 4.31415927
     4.37699112 4.43982297 4.50265482 4.56548668 4.62831853 4.69115038
     4.75398224 4.81681409 4.87964594 4.9424778  5.00530965 5.0681415
     5.13097336 5.19380521 5.25663706 5.31946891 5.38230077 5.44513262
     5.50796447 5.57079633 5.63362818 5.69646003 5.75929189 5.82212374
     5.88495559 5.94778745 6.0106193  6.07345115 6.136283   6.19911486
     6.26194671 6.32477856 6.38761042 6.45044227 6.51327412 6.57610598
     6.63893783 6.70176967 6.76460125 6.82742866 6.89018589 6.95176632
     6.99367964 6.72527549 4.         1.27472451 1.00632036 1.04823368
     1.10981411 1.17257134 1.23539875 1.29823033 1.36106217 1.42389402
     1.48672588 1.54955773 1.61238958 1.67522144 1.73805329 1.80088514
     1.863717   1.92654885 1.9893807  2.05221255 2.11504441 2.17787626
     2.24070811 2.30353997 2.36637182 2.42920367 2.49203553 2.55486738
     2.61769923 2.68053109 2.74336294 2.80619479 2.86902664 2.9318585
     2.99469035 3.0575222  3.12035406 3.18318591 3.24601776 3.30884962
     3.37168147 3.43451332 3.49734518 3.56017703 3.62300888 3.68584073
     3.74867259 3.81150444 3.87433629 3.93716815 4.        ]

```python
#Plotting the initial condition
plt.figure(figsize=(11, 7), dpi=100)
plt.plot(x, u, marker='o', lw=2)
plt.xlim([0, 2 * np.pi])
plt.ylim([0, 10])
plt.show()
```

```python
#Changing u according to the timestep
for n in range(nt):
    un = u.copy()
    for i in range(1, nx-1):
        u[i] = un[i] - un[i] * dt / dx *(un[i] - un[i-1]) + nu * dt / dx**2*(un[i+1] - 2 * un[i] + un[i-1])
    u[0] = un[0] - un[0] * dt / dx * (un[0] - un[-2]) + nu * dt / dx**2 *(un[1] - 2 * un[0] + un[-2])
    u[-1] = u[0]

print(u)
```

    [2.77501411 2.81860393 2.86219375 2.90578357 2.94937339 2.99296321
     3.03655303 3.08014285 3.12373267 3.16732249 3.21091231 3.25450213
     3.29809195 3.34168177 3.38527159 3.42886141 3.47245123 3.51604105
     3.55963087 3.60322069 3.64681051 3.69040033 3.73399015 3.77757997
     3.82116979 3.86475961 3.90834943 3.95193925 3.99552907 4.03911889
     4.08270871 4.12629853 4.16988835 4.21347816 4.25706798 4.3006578
     4.34424762 4.38783744 4.43142726 4.47501708 4.5186069  4.56219672
     4.60578654 4.64937636 4.69296618 4.736556   4.78014582 4.82373564
     4.86732546 4.91091528 4.95450509 4.9980949  5.0416847  5.08527447
     5.12886415 5.17245365 5.21604272 5.25963077 5.30321645 5.34679663
     5.39036412 5.43390242 5.4773738  5.52069226 5.56366213 5.60583953
     5.64622005 5.68253942 5.70972375 5.71653417 5.67868075 5.54649073
     5.23096356 4.62161195 3.7237691  2.83274015 2.25637999 1.99159854
     1.90314957 1.89369951 1.9164822  1.95196924 1.99240748 2.03477056
     2.07788235 2.12128577 2.1648029  2.2083644  2.25194321 2.29552876
     2.33911694 2.38270612 2.4262957  2.46988543 2.51347521 2.55706502
     2.60065484 2.64424465 2.68783447 2.73142429 2.77501411]

```python
#Analytical Solution
u_analytical = np.asarray([ufunc(nt * dt, xi, nu) for xi in x])
print(u_analytical)
```

    [2.77811931 2.82175791 2.8653965  2.9090351  2.95267369 2.99631229
     3.03995089 3.08358948 3.12722808 3.17086667 3.21450527 3.25814387
     3.30178246 3.34542106 3.38905965 3.43269825 3.47633685 3.51997544
     3.56361404 3.60725264 3.65089123 3.69452983 3.73816842 3.78180702
     3.82544562 3.86908421 3.91272281 3.9563614  4.         4.0436386
     4.08727719 4.13091579 4.17455438 4.21819298 4.26183158 4.30547017
     4.34910877 4.39274736 4.43638596 4.48002456 4.52366315 4.56730175
     4.61094035 4.65457894 4.69821754 4.74185613 4.78549473 4.82913333
     4.87277192 4.91641052 4.96004911 5.00368771 5.04732631 5.0909649
     5.1346035  5.17824209 5.22188069 5.26551929 5.30915788 5.35279648
     5.39643507 5.44007367 5.48371227 5.52735086 5.57098946 5.61462805
     5.65826665 5.70190524 5.74554383 5.78918234 5.83282035 5.87645478
     5.92006383 5.96349303 6.00564782 6.0387971  6.00950252 5.59878747
     4.         2.40121253 1.99049748 1.9612029  1.99435218 2.03650697
     2.07993617 2.12354522 2.16717965 2.21081766 2.25445617 2.29809476
     2.34173335 2.38537195 2.42901054 2.47264914 2.51628773 2.55992633
     2.60356493 2.64720352 2.69084212 2.73448071 2.77811931]

```python
plt.figure(figsize=(11, 7), dpi=100)
plt.plot(x, u, marker='o', lw=2, label="Numerical Solution")
plt.plot(x, u_analytical, color='orange', label="Analytical Solution")
plt.xlim(0, 2*np.pi)
plt.ylim(0,10)
plt.legend()
plt.show()
```

```python

```

12 Steps to Navier-Stokes/Step 4 - 1D Burger's Equation.ipynb


```python
from sympy import symbols, Function, solve
from sympy import diff, sin, exp
```

```python
x, y, c1, c2, c3, n = symbols('x y c1 c2 c3 n')
y = Function('y')
```

```python
print(diff((x - c1 - c2*n-c3*n*n)**2, c1))
print(diff((x - c1 - c2*n-c3*n*n)**2, c2))
print(diff((x - c1 - c2*n-c3*n*n)**2, c3))
```

    2*c1 + 2*c2*n + 2*c3*n**2 - 2*x
    -2*n*(-c1 - c2*n - c3*n**2 + x)
    -2*n**2*(-c1 - c2*n - c3*n**2 + x)

```python

```


# Model selection using GpABC

## Problem formulation

This notebook illustrates how to perform model selection using the algorithm described in (Toni et al, 2009). The example here is adapted from the SIR (susceptible-infected-recovered) model in Section 3.3 of the paper.

The example gives two models that describe the evolution of the number of of susceptible ($S$), infected ($I$) and recovered ($R$) individuals in a population (the number of dead individuals is not included in the model).

There are two possible models, both of which are described below. Given some observed data on the number of individuals in each disease state over time we wish to determine which of the two models is better supported by these data. The data was generated using the first model so the algorithm should reflect this.

### Model 1

A basic SIR model assumes that each individual can be infected once and is then able to infect other individuals immediately.

The model is given by

\begin{align}
    \dot{S} &= \alpha - \gamma S I - d S, \\
    \dot{I} &= \gamma S I - v I - d I, \\
    \dot{R} &= v I - d R \, ,
\end{align}

where the notation $\dot{x}$ denotes $\text{d}x/\text{d}t$.

The paramters of this model are the birth rate $\alpha$, the infection rate $\gamma$, the recovery rate $v$ and the death rate $d$. Note that the death rate is constant for all three disease states.

### Model 2

A slightly more sophisticated model includes a time delay between an individual becoming infected and being able to infect others by introducing an additional latent state ($L$). This model is given by

\begin{align}
    \dot{S} &= \alpha - \gamma S I - d S , \\
    \dot{L} &= \gamma S I - \delta L - d L , \\
    \dot{I} &= \delta L - v I - d I , \\
    \dot{R} &= v I - d R \, ,
\end{align}

where $\delta$ is the transition rate from latent to infected.

### References

- Toni, T., Welch, D., Strelkowa, N., Ipsen, A., & Stumpf, M. P. H. (2009). Approximate Bayesian computation scheme for parameter inference and model selection in dynamical systems. *Interface*, (July 2008), 187–202. https://doi.org/10.1098/rsif.2008.0172

## Computation in GpABC

### Preamble

The code below loads the required packages and prepares the observed data into the correct format (this is a 2D array with size (number of trajectories, number of time points).

The observed data contains information the number of susceptible, infected and recovered individuals at 13 time points.

```julia
# Load required packages
using GpABC, DifferentialEquations, Distributions, Distances, Plots
```

    ┌ Info: Recompiling stale cache file /Users/tanhevg/.julia/compiled/v1.0/GpABC/9yA2E.ji for GpABC [e850a1a4-d859-11e8-3d54-a195e6d045d3]
    └ @ Base loading.jl:1187
    ┌ Info: Recompiling stale cache file /Users/tanhevg/.julia/compiled/v1.0/Plots/ld3vC.ji for Plots [91a5bcdd-55d7-5caf-9e0b-520d859cae80]
    └ @ Base loading.jl:1187

```julia
times = [0.0, 0.6, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0]
data = [[ 20.     ,  10.     ,   0.     ],
       [  0.12313,  13.16813,   9.42344],
       [  0.12102,   7.17251,  11.18957],
       [  0.09898,   2.36466,  10.0365 ],
       [  0.37887,   0.92019,   6.87117],
       [  1.00661,   0.61958,   4.44955],
       [  1.20135,   0.17449,   3.01271],
       [  1.46433,   0.28039,   1.76431],
       [  1.37789,   0.0985 ,   1.28868],
       [  1.57073,   0.03343,   0.81813],
       [  1.4647 ,   0.28544,   0.52111],
       [  1.24719,   0.10138,   0.22746],
       [  1.56065,   0.21671,   0.19627]]

data = hcat(data...);
```

Next, we define the models. As in other GpABC functions, the model is represented using a function that takes parameters as an argument and returns trajectories that are the solution to the differential equation system. This is called the `simulator_function`. The solution shuuld have the same shape as the reference data, which is (number of trajectories, number of time points).

This means that for the second model, which has 4 states, we need to modify the `simulator_function` in order to ignore the solution for the latent state $L$ as we have no reference data for that trajectory.

We also need to define different initial conditions for the two models as the first model has 3 states and the second has 4.

```julia
# Need initial conditions for each model since they have different numbers of states
ic1 = [20.0, 10.0, 0.0]
ic2 = [20.0, 0.0, 10.0, 0.0]
ics = [ic1, ic2]

# Define a simulator function for each model. Each function returns the solution of the model
# as a 2D array for some parameter values. The appropriate ODE is solved at the same time
# points as the observed data
function simulator1(params::Array{Float64,1})

    # p = (alpha, gamma, d, v)
    # x = (S, I, R)
    function model1(dx, x, p, t)
        dx[1] = p[1] - p[2]*x[1]*x[2] - p[3]*x[1] # dS/dt = alpha - gamma*S*I - d*S
        dx[2] = p[3]*x[1]*x[2] - p[4]*x[2] - p[3]*x[2] # dI/dt = gamma*S*I - v*I - d*I
        dx[3] = p[4]*x[2] - p[3]*x[3] # dR/dt = v*I - d*R
    end

    return(Array{Float64,2}(
        solve(ODEProblem(model1, ics[1], (times[1], times[end]), params),
                saveat=times, force_dtmin=true)))
end

function simulator2(params::Array{Float64,1})

    # p = (alpha, gamma, d, v, delta)
    # x = (S, L, I, R)
    function model2(dx, x, p, t)
        dx[1] = p[1] - p[2]*x[1]*x[3] - p[3]*x[1] # dS/dt = alpha - gamma*S*I - d*S
        dx[2] = p[2]*x[1]*x[3] - p[5]*x[2] - p[3]*x[2] # dL/dt = gamma*S*I - delta*L - d*L
        dx[3] = p[5]*x[2] - p[4]*x[3] - p[3]*x[3] # dI/dt = delta*L - v*I - d*I
        dx[4] = p[4]*x[3] - p[3]*x[4] # dR/dt = v*I - d*R
    end

    # Model2 contains the species L, which is not measured - we remove it from the returned ODE solution
    # so that it can be compared to the reference data "data", which only contains S, I and R
    return(Array{Float64,2}(
        solve(ODEProblem(model2, ics[2], (times[1], times[end]), params),
                saveat=times, force_dtmin=true)))[[1,3,4],:]
end;
```

We also need a set of priors for each model, as well as a summary statistic and threshold schedule. We will compare the full model solution with the data, which is the "keep_all" summary statistic. The model prior will be the default discrete uniform prior with span 2 (as there are two models).

```julia
#
# Priors and initial conditions - these are model-specfic as each model can
# have different numbers of parameters/species
#
priors1 = [Uniform(0.0, 5.0) for i in 1:4]
priors2 = vcat([Uniform(0.0, 5.0) for i in 1:4], Uniform(0.0, 10.0))
priors3 = vcat([Uniform(0.0, 5.0) for i in 1:4], Uniform(0.0, 10.0))

threshold_schedule = [20, 15, 10, 5, 3, 2.5, 2, 1.7, 1.5]
summary_statistic = "keep_all";
```

We are now ready to perform the model selection computation, accepting 200 particles at each population.

First, we will do this using full model simulations:

```julia
n_particles = 200

ms_sim_result = SimulatedModelSelection(data,
    [simulator1, simulator2],
    [priors1, priors2],
    threshold_schedule,
    n_particles);
```

A plot recipe allows this result to be plotted easily:

```julia
plot(ms_sim_result, title="Model selection result - simulation")
```

The first model is correctly identified as the correct one.

Now do the same computation using emulation. The only difference between the simulation and emulation function call is that we specify the number of design points as the first argument when using emulation.

We will use 200 design points.

```julia
ms_emu_result = EmulatedModelSelection(data,
    [simulator1, simulator2],
    [priors1, priors2],
    threshold_schedule,
    n_particles,
    200;
    summary_statistic = "keep_all",
    distance_function=Distances.euclidean);
```

```julia
plot(ms_emu_result, title="Model selection result - emulation")
```

## References

[Toni, T., Welch, D., Strelkowa, N., Ipsen, A. and Stumpf, M.P., 2009. Approximate Bayesian computation scheme for parameter inference and model selection in dynamical systems. *Journal of the Royal Society Interface, 6*(31), pp.187-202.](http://rsif.royalsocietypublishing.org/content/royinterface/6/31/187.full.pdf)


# Jupyter notebook: Affine equivalences of rational surfaces of translation, and applications to rational minimal surfaces

This is a Jupyter notebook accompanying the paper [1].

[1] J.G. Alcazar, G. Muntingh. _Affine equivalences of rational surfaces of translation, and applications to rational minimal surfaces_, Available at arXiv.

[2] https://groupprops.subwiki.org/wiki/Dihedral_group:D8

```python
!pip install --quiet sympy numpy matplotlib tabulate

import numpy as np
import sympy as sp

import matplotlib.pyplot as plt
import matplotlib as mpl

from utils.minsurf import Weierstrass_Enneper
from utils.plot import Enneper_mesh, symmetry_element_meshes, bridge_meshes

from sympy.assumptions import assuming, Q
sp.init_printing()

from IPython.display import display, Math, Latex, HTML
from tabulate import tabulate
```

```python
def T1(n, x=None):
    """
    :param n: Degree of the Chebyshev polynomial
    :param x: Variable/number
    :return: Chebyshev polynomial of the first kind of degree n in the variable/number x.
    """
    if x is None:
        x = sp.Symbol('x')

    if n == 0:
        return 1
    elif n == 1:
        return x
    elif n >= 2:
        return sp.expand(2*x*T1(n-1, x=x) - T1(n-2, x=x))

def S_enneper(k):
    r, s, t = sp.symbols("r s t")

    x = sp.expand(sp.expand(t - r**(2*k+1) * T1(2*k + 1, x=t/r)/(2*k + 1)).subs(r, sp.sqrt(s**2 + t**2)))
    y = sp.expand(sp.expand(-s - (-1)**k * r**(2*k+1) * T1(2*k + 1, x=s/r)/(2*k + 1)).subs(r, sp.sqrt(s**2 + t**2)))
    z = sp.expand(sp.expand(2*r**(k+1) * T1(k+1, x=t/r)/(k+1)).subs(r, sp.sqrt(s**2 + t**2)))

    return x, y, z

def binomial(n, k):
    assert(0 <= k <= n)

    if n == 0 or n == 1 or k == 0 or k == n:
        return 1
    else:
        return binomial(n-1, k-1) + binomial(n-1, k)

def get_colors():
    c_surfs = tuple([tuple(list(mpl.colors.to_rgb(col))[:3] + [0.8]) for col in ['tab:blue', 'cornflowerblue']])
    c_plane, c_line, c_point = [tuple(list(mpl.colors.to_rgb(col))[:3] + [0.2])
                                for col in ["indianred", "forestgreen", "goldenrod"]]

    return c_surfs, c_plane, c_line, c_point

def plot_enneper(ks=(1, 2, 3, 4), N=20, azimuths=(0, 20), elevations=(90, 30), colors=get_colors(), verbose=True,
                 savefig=False, showfig=True, elements=True):
    """
    :param tuple ks        : Orders of the Enneper surface
    :param int N           : Resolution of each plot
    :param tuple azimuths  : Azimuths for each plot
    :param tuple elevations: Elevations for each plot
    :param tuple colors    : RGBA tuples for colors of the surface, planes, lines, and point
    :param bool verbose    : Whether to output auxilliary information
    :param bool savefig    : Whether to save the figure to a file
    :param bool showfig    : Whether to show the figure
    :param bool elements   : Whether to display elements
    :return:
    """
    c_surfs, c_plane, c_line, c_point = colors
    for k in ks:
        if verbose:
            print(f"Enneper surface of order k={k}")

        # Collect meshes of the Enneper surface, its symmetry planes, symmetry axes, and symmetry point
        X, Y, Z, C = Enneper_mesh(k=k, N=N, colors=c_surfs)

        if elements:
            u_plane = max(np.max(X), np.max(Y))
            u_line = 0.6*np.max(np.sqrt(X**2 + Y**2))
            u_zfactor = np.max(Z) / u_plane

            Xs, Ys, Zs, Cs = symmetry_element_meshes(k, N, u_plane, u_line, u_zfactor, c_plane, c_line, c_point)
            X, Y, Z, C = bridge_meshes([X] + Xs, [Y] + Ys, [Z] + Zs, [C] + Cs)

        # We plot the surface from various view points
        for azimuth, elevation in zip(azimuths, elevations):
            fig = plt.figure()
            ax = fig.gca(projection='3d')
            ax.view_init(elev=elevation, azim=azimuth)

            ax.plot_surface(X, Y, Z, antialiased=True, rstride=1, cstride=1, facecolors=C,
                            shade=True, linewidth=1)

            ax.set_xlim(np.min(X), np.max(X))
            ax.set_ylim(np.min(Y), np.max(Y))
            ax.set_zlim(np.min(Z), np.max(Z))
            plt.axis('off')

            if savefig:
                if elements:
                    fname = f"figs/Enneper-k-{k}-elements-elevation-{elevation}-azimuth-{azimuth}-N-{N}.png"
                else:
                    fname = f"figs/Enneper-k-{k}-elevation-{elevation}-azimuth-{azimuth}-N-{N}.png"

                plt.savefig(fname, bbox_inches='tight', dpi=300)

            if showfig:
                plt.show()

r, s, t = sp.symbols("r s t", real=True)
z = sp.Symbol('z', nonzero=True)
k = sp.Symbol('k', positive=True)
m = sp.Symbol('m', positive=True)
```

# Plotting Enneper surfaces and their symmetry elements

```python
plot_enneper(N=50)
```

## Proposition 7 in [1]

```python
f = 2
g = z**k
psi = Weierstrass_Enneper(f, g, z, imag_unit=sp.I)
print(f"The Weierstrass form with f={f} and g={g} yields the minimal curve")
display(psi)
```

## Proposition 8 in [1] (for the case k=1)

```python
Psi = [[sp.expand(expr.subs(k, 1).diff(z, k0).subs(z, 0)) for expr in psi] for k0 in [1, 2, 3]]
Psi = sp.Matrix(Psi).transpose()
A = sp.Matrix([[sp.I**m, 0, 0], [0, (-1)**m, 0], [0, 0, (-sp.I)**m]])
M = Psi*A*Psi.inv()
print("\nThe matrix Psi=")
display(Psi)
print("\nThe matrix Psi^-1=")
display(Psi.inv())
print("\nThe matrix A=")
display(A)
print("\nThe matrix M=")
display(M)
R1 = M.subs(m, 1)
print("\nThe matrix R1=")
display(R1)

```

    The matrix Psi=

$\displaystyle \left[\begin{matrix}1 & 0 & -2\\i & 0 & 2 i\\0 & 2 & 0\end{matrix}\right]$

    The matrix Psi^-1=

$\displaystyle \left[\begin{matrix}\frac{1}{2} & - \frac{i}{2} & 0\\0 & 0 & \frac{1}{2}\\- \frac{1}{4} & - \frac{i}{4} & 0\end{matrix}\right]$

    The matrix A=

$\displaystyle \left[\begin{matrix}i^{m} & 0 & 0\\0 & \left(-1\right)^{m} & 0\\0 & 0 & \left(- i\right)^{m}\end{matrix}\right]$

    The matrix M=

$\displaystyle \left[\begin{matrix}\frac{i^{m}}{2} + \frac{\left(- i\right)^{m}}{2} & - \frac{i i^{m}}{2} + \frac{i \left(- i\right)^{m}}{2} & 0\\\frac{i i^{m}}{2} - \frac{i \left(- i\right)^{m}}{2} & \frac{i^{m}}{2} + \frac{\left(- i\right)^{m}}{2} & 0\\0 & 0 & \left(-1\right)^{m}\end{matrix}\right]$

    The matrix R1=

$\displaystyle \left[\begin{matrix}0 & 1 & 0\\-1 & 0 & 0\\0 & 0 & -1\end{matrix}\right]$

## Example 1 in [1]

```python
for k in range(1, 4+1):
    print(f"\nThe Enneper surface of order k={k} has parametrization:")
    display(S_enneper(k))
```

## Remark 2 in [1]

```python
for n in range(0, 5):
    k, eps = divmod(n, 2)
    lhs1 = sp.expand(sp.expand(r**n * T1(n, t/r)).subs(r, sp.sqrt(s**2 + t**2)))
    lhs2 = sp.expand(sp.expand(r**n * T1(n, s/r)).subs(r, sp.sqrt(s**2 + t**2)))
    rhs1 = sum([(-1)**(k+m) * binomial(n, 2*m + eps) * s**(n - 2*m - eps) * t**(2*m + eps) for m in range(k+1)])
    rhs2 = sum([(-1)**(k+m) * binomial(n, 2*m + eps) * t**(n - 2*m - eps) * s**(2*m + eps) for m in range(k+1)])
    print(f"For n={n}, the lhs and rhs in Remark 2 match: {lhs1 == rhs1 and lhs2 == rhs2}")
```

    For n=0, the lhs and rhs in Remark 2 match: True
    For n=1, the lhs and rhs in Remark 2 match: True
    For n=2, the lhs and rhs in Remark 2 match: True
    For n=3, the lhs and rhs in Remark 2 match: True
    For n=4, the lhs and rhs in Remark 2 match: True

## Table 1 in [1]
Consider the unitary matrices
$$\mathbf{B} = \begin{bmatrix}0&-i&0\\i&0&0\\0&0&i \end{bmatrix},\quad
\mathbf{C} = \begin{bmatrix}i&0&0\\0&i&0\\0&0&i \end{bmatrix},\quad
\mathbf{S} = \begin{bmatrix}1&0&0\\0&-1&0\\0&0&1\end{bmatrix},\quad
\mathbf{T} = \begin{bmatrix}1&0&0\\0&1&0\\0&0&-1\end{bmatrix},\quad
\mathbf{R}_1 = \begin{bmatrix}0&1&0\\-1&0&0\\0&0&-1 \end{bmatrix}.$$
The matrices in Table 1 are $\mathbf{S}^n \mathbf{B}^s \mathbf{C}^{q+r}$, for $q,r,s \in \mathbb{Z}/4\mathbb{Z}$ and $n\in \mathbb{Z}/2\mathbb{Z}$.

```python
def matrix_to_latex(M):
    s = "$\\begin{bmatrix}" + \
        "".join(["".join([str(M[i])] + (i % 3 != 2)*["&"] + (i % 3 == 2)*["\\\\"])  for i in range(len(C))]) + \
        "\end{bmatrix}$"
    return s

S = sp.Matrix([[1, 0, 0], [0, -1, 0], [0, 0, 1]])
B = sp.Matrix([[0, -sp.I, 0], [sp.I, 0, 0], [0, 0, sp.I]])
C = sp.Matrix([[sp.I, 0, 0], [0, sp.I, 0], [0, 0, sp.I]])
T = sp.Matrix([[1, 0, 0], [0, 1, 0], [0, 0, -1]])
I = S**0
```

```python
headers = ['$M^+_{q+r,s}$', '$q+r\equiv 0$', '$q+r\equiv 1$', '$q+r \equiv 2$', '$q+r \equiv 3$']
table = [ [f"$s\equiv{s}$"] + [matrix_to_latex(B**s * C**qr) for qr in range(4)] for s in range(4)]
HTML(tabulate(table, headers=headers, floatfmt='.3e', tablefmt='html'))
```

$M^+_{q+r,s}$  	$q+r\equiv 0$                                        	$q+r\equiv 1$                                        	$q+r \equiv 2$                                        	$q+r \equiv 3$

$s\equiv0$     	$\begin{bmatrix}1&0&0\\0&1&0\\0&0&1\\\end{bmatrix}$  	$\begin{bmatrix}I&0&0\\0&I&0\\0&0&I\\\end{bmatrix}$  	$\begin{bmatrix}-1&0&0\\0&-1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}-I&0&0\\0&-I&0\\0&0&-I\\\end{bmatrix}$

$s\equiv1$     	$\begin{bmatrix}0&-I&0\\I&0&0\\0&0&I\\\end{bmatrix}$ 	$\begin{bmatrix}0&1&0\\-1&0&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&I&0\\-I&0&0\\0&0&-I\\\end{bmatrix}$ 	$\begin{bmatrix}0&-1&0\\1&0&0\\0&0&1\\\end{bmatrix}$

$s\equiv2$     	$\begin{bmatrix}1&0&0\\0&1&0\\0&0&-1\\\end{bmatrix}$ 	$\begin{bmatrix}I&0&0\\0&I&0\\0&0&-I\\\end{bmatrix}$ 	$\begin{bmatrix}-1&0&0\\0&-1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}-I&0&0\\0&-I&0\\0&0&I\\\end{bmatrix}$

$s\equiv3$     	$\begin{bmatrix}0&-I&0\\I&0&0\\0&0&-I\\\end{bmatrix}$	$\begin{bmatrix}0&1&0\\-1&0&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&I&0\\-I&0&0\\0&0&I\\\end{bmatrix}$  	$\begin{bmatrix}0&-1&0\\1&0&0\\0&0&-1\\\end{bmatrix}$

```python
headers = ['$M^-_{q+r,s}$', '$q+r\equiv 0$', '$q+r\equiv 1$', '$q+r \equiv 2$', '$q+r \equiv 3$']
table = [[f"$s\equiv{s}$"] + [matrix_to_latex(B**s * C**qr *S) for qr in range(4)] for s in range(4)]
HTML(tabulate(table, headers=headers, floatfmt='.3e', tablefmt='html'))
```

$M^-_{q+r,s}$  	$q+r\equiv 0$                                        	$q+r\equiv 1$                                         	$q+r \equiv 2$                                        	$q+r \equiv 3$

$s\equiv0$     	$\begin{bmatrix}1&0&0\\0&-1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}I&0&0\\0&-I&0\\0&0&I\\\end{bmatrix}$  	$\begin{bmatrix}-1&0&0\\0&1&0\\0&0&-1\\\end{bmatrix}$ 	$\begin{bmatrix}-I&0&0\\0&I&0\\0&0&-I\\\end{bmatrix}$

$s\equiv1$     	$\begin{bmatrix}0&I&0\\I&0&0\\0&0&I\\\end{bmatrix}$  	$\begin{bmatrix}0&-1&0\\-1&0&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&-I&0\\-I&0&0\\0&0&-I\\\end{bmatrix}$	$\begin{bmatrix}0&1&0\\1&0&0\\0&0&1\\\end{bmatrix}$

$s\equiv2$     	$\begin{bmatrix}1&0&0\\0&-1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}I&0&0\\0&-I&0\\0&0&-I\\\end{bmatrix}$ 	$\begin{bmatrix}-1&0&0\\0&1&0\\0&0&1\\\end{bmatrix}$  	$\begin{bmatrix}-I&0&0\\0&I&0\\0&0&I\\\end{bmatrix}$

$s\equiv3$     	$\begin{bmatrix}0&I&0\\I&0&0\\0&0&-I\\\end{bmatrix}$ 	$\begin{bmatrix}0&-1&0\\-1&0&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&-I&0\\-I&0&0\\0&0&I\\\end{bmatrix}$ 	$\begin{bmatrix}0&1&0\\1&0&0\\0&0&-1\\\end{bmatrix}$

The matrices $\mathbf{B}$ and $\mathbf{C}$ commute, and their product is the matrix $\mathbf{R}_1$.

```python
print(R1 == B@C == C@B)
```

    True

The matrix $\mathbf{B}$ is a square root of the matrix $T$.

```python
print(B@B == T)
```

    True

## Remark 4 in [1]
Consider the (external) direct product group
$$ D_4 \times \mathbf{Z}/2\mathbf{Z} \simeq \langle \rho,\sigma,\tau\,:\,\rho^4=\sigma^2=\tau^2=e,\ \sigma\rho\sigma=\rho^{-1},\ \tau\rho=\rho\tau,\ \sigma\tau=\tau\sigma \rangle, $$
where $e$ denotes the neutral element. The map
$$ D_4 \times \mathbf{Z}/2\mathbf{Z} \longrightarrow O(3), \qquad \sigma^n \rho^m \tau^p\longmapsto \mathbf{S}^n\mathbf{R}_1^m \mathbf{T}^p $$
is a group monomorphism establishing a group structure on the set of real matrices in the above table.

```python
print("R1 has order 4:", R1@R1 != I and R1@R1@R1 != I and R1@R1@R1@R1 == I)
print("S has order 2:", S@S == I)
print("T has order 2:", T@T == I)
print("S and R1 satisfy the dihedral relation SR_1S = R_1^-1:", S@R1@S == R1.inv())
print("T and R1 commute:", T@R1 == R1@T)
print("S and T commute:", S@T == T@S)
```

    R1 has order 4: True
    S has order 2: True
    T has order 2: True
    S and R1 satisfy the dihedral relation SR_1S = R_1^-1: True
    T and R1 commute: True
    S and T commute: True

Next, we show that these matrices appear in the above table. Since $\mathbf{T} = \mathbf{B}^2$ and $\mathbf{R}_1 = \mathbf{B}\mathbf{C} = \mathbf{C}\mathbf{B}$, one has
$$\mathbf{R}_1^m \mathbf{S}^n \mathbf{T}^p = \mathbf{C}^m \mathbf{B}^{m + 2p} \mathbf{S}^n = \mathbf{C}^{q+r} \mathbf{B}^s \mathbf{S}^n, $$
with $s = m + 2p$ and $m = q + r$.

```python
headers = ['$M^-_{q+r,s}$', '$q+r\equiv 0$', '$q+r\equiv 1$', '$q+r \equiv 2$', '$q+r \equiv 3$']
table = [[f"$s-(q+r)\equiv {2*m}$"] + [matrix_to_latex(R1**p * T**m) for p in range(4)] for m in range(2)]
HTML(tabulate(table, headers=headers, floatfmt='.3e', tablefmt='html'))
```

$M^-_{q+r,s}$    	$q+r\equiv 0$                                       	$q+r\equiv 1$                                        	$q+r \equiv 2$                                        	$q+r \equiv 3$

$s-(q+r)\equiv 0$	$\begin{bmatrix}1&0&0\\0&1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&1&0\\-1&0&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}-1&0&0\\0&-1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&-1&0\\1&0&0\\0&0&-1\\\end{bmatrix}$

$s-(q+r)\equiv 2$	$\begin{bmatrix}1&0&0\\0&1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&1&0\\-1&0&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}-1&0&0\\0&-1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&-1&0\\1&0&0\\0&0&1\\\end{bmatrix}$

```python
headers = ['$M^-_{q+r,s}$', '$q+r\equiv 0$', '$q+r\equiv 1$', '$q+r \equiv 2$', '$q+r \equiv 3$']
table = [[f"$s-(q+r)\equiv {2*m}$"] + [matrix_to_latex(R1**p * T**m * S) for p in range(4)] for m in range(2)]

HTML(tabulate(table, headers=headers, floatfmt='.3e', tablefmt='html'))
```

$M^-_{q+r,s}$    	$q+r\equiv 0$                                        	$q+r\equiv 1$                                         	$q+r \equiv 2$                                       	$q+r \equiv 3$

$s-(q+r)\equiv 0$	$\begin{bmatrix}1&0&0\\0&-1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&-1&0\\-1&0&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}-1&0&0\\0&1&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}0&1&0\\1&0&0\\0&0&-1\\\end{bmatrix}$

$s-(q+r)\equiv 2$	$\begin{bmatrix}1&0&0\\0&-1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&-1&0\\-1&0&0\\0&0&1\\\end{bmatrix}$ 	$\begin{bmatrix}-1&0&0\\0&1&0\\0&0&-1\\\end{bmatrix}$	$\begin{bmatrix}0&1&0\\1&0&0\\0&0&1\\\end{bmatrix}$

## Example 2 in [1]
Let $\varphi^q(z) = i^q(z)$. We show that the diagonal cases $s = q + r$ in Table 1 are symmetries of the Enneper surface of order 1, by verifying the relations
$$ \Psi \circ \varphi^q (z) = \mathbf{f}^+_{q+r,s} \circ \Psi(z) = \mathbf{M}^+_{q+r,s} \Psi(z),$$
$$ \Psi \circ \varphi^q (z) = \mathbf{f}^-_{q+r,s} \circ \overline{\Psi}(z) = \mathbf{M}^-_{q+r,s} \overline{\Psi}(z). $$

```python
k = 1
f = 2 + 0*z
g = z**k
psi = Weierstrass_Enneper(f, g, z, imag_unit=sp.I)

print(f"The Enneper surface of order k={k} has the symmetry:")
for q in range(4):
    for r in range(4):
        for s in range(4):
            # Does the parity-like condition hold for this combination (q,r,s)?
            if sp.I**s * g == g.subs(z, sp.I**q*z) and sp.I**r * f == f.subs(z, sp.I**q*z):
                M = B**s * C**(q+r)
                rhs = M@sp.Matrix(psi)
                lhs = sp.Matrix([expr.subs(z, sp.I**q * z) for expr in psi])
                print("M^+_{"+str(q+r)+","+str(s)+"}:", sp.expand(lhs - rhs) == sp.Matrix([0,0,0]))
                psi_conj = sp.Matrix([expr.conjugate() for expr in psi])
                rhs = S@M@psi_conj
                lhs = sp.Matrix([expr.subs(z, sp.I**(-q) * z) for expr in psi])
                print("M^-_{"+str(q+r)+","+str(s)+"}:", sp.expand(lhs - rhs) == sp.Matrix([0,0,0]))
```

    The Enneper surface of order k=1 has the symmetry:
    M^+_{0,0}: True
    M^-_{0,0}: True
    M^+_{1,1}: True
    M^-_{1,1}: True
    M^+_{2,2}: True
    M^-_{2,2}: True
    M^+_{3,3}: True
    M^-_{3,3}: True


```python
# initial setup
try:
    # settings colab:
    import google.colab
except ModuleNotFoundError:
    # settings local:
    %run "../../../common/0_notebooks_base_setup.py"
```

---

#### Distribución Poisson

La distribución Poisson cuenta **cantidad de eventos en un período de tiempo dado**.

Podemos pensar esta distribución como la probabilidad de que ocurra un determinado número de eventos durante cierto período de tiempo.

Sea la variable aleatoria discreta X el número de veces que determinado evento ocurre en un intervalo de tiempo o espacio.

Entonces X puede ser una variable Poisson que toma valores $x=0,1,2,…$ si cumple con las siguientes condiciones:

1. El número de eventos que ocurren en períodos de tiempo, sin superposición entre períodos, es independiente.

2. La probabilidad de exactamente un evento en un intervalo de tiempo corto de duración h=1/n es aproximadamente h*λ donde n es la cantidad de intervalos dentro del período considerado.

3. La probabilidad de exactamente dos o más eventos en un intervalo de tiempo corto es esencialmente cero.

Si se cumplen esas condiciones X es una variable aleatoria que sigue un proceso de Poisson aproximado con parámetro $\lambda > 0$ por lo que  su función de probabilidad puntual es

\begin{equation}
    P(X = k) = \frac{\lambda^k . e^{-\lambda}}{k!} \\
    \lambda > 0, \hspace{0.2cm} k = 0, 1, 2, ...
\end{equation}

Se puede mostrar que $\lambda$ es la media y la varianza de una variable Poisson.

Función de densidad de probabilidad:

**Ejemplos**:

* X: Cantidad de pacientes que ingresan en la guardia de un hospital en una hora

* X: Cantidad de autos que pasan por una cabina de peaje

* X: Cantidad de llamados que llegan a un operador en un call center en la mañana.

---

Vamos a ver ahora cómo generar datos con esta distibución de probabilidad.

Necesitamos un generador de números aleatorios, que expone métodos para generar números aleatorios con alguna distribución de probabilidad especificada. Construimos este generador de este modo `np.random.default_rng()`

https://docs.scipy.org/doc/numpy/reference/random/generator.html

Estas son las distribuciones de probabilidad disponibles:
https://docs.scipy.org/doc/numpy/reference/random/generator.html#distributions

Vamos a generar datos con distribución empleando el método `poisson`
https://docs.scipy.org/doc/numpy/reference/random/generated/numpy.random.Generator.poisson.html#numpy.random.Generator.poisson

```python
import seaborn as sns
def distribution_plotter(data, label, bins=None):
    sns.set(rc={"figure.figsize": (10, 7)})
    sns.set_style("white")
    dist = sns.distplot(data, bins= bins, hist_kws={'alpha':0.2}, kde_kws={'linewidth':5})
    dist.set_title('Distribucion de ' + label + '\n', fontsize=16)
```

```python
import numpy as np
random_generator = np.random.default_rng()
lambda_value = 10
sample_size = 10000
random_poisson_data = random_generator.poisson(lam=lambda_value, size = sample_size)

distribution_plotter(random_poisson_data, "Poisson")
```

#### Referencias

Gráficos: https://es.wikipedia.org/wiki/Distribuci%C3%B3n_de_Poisson


```python
%matplotlib widget
```

## Python Package Imports

```python
from IPython.core.display import HTML
HTML("""
<style>
.output_png {
    display: table-cell;
    text-align: right;
    vertical-align: middle;
}
</style>
""")
```

```python
%load_ext autoreload
%autoreload 2
#import warnings
#warnings.filterwarnings("always")
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
import matplotlib.cm as cm
from IPython.display import display, Math, clear_output

import sympy  #Sympy1.2 is from conda-forge: [sudo conda install -c conda-forge sympy]
from sympy import *
from sympy.physics.vector import ReferenceFrame, CoordinateSym
from sympy.vector import CoordSys3D, divergence, curl

import ipyvolume as ipv
import time
from ipywidgets import Output

import helpers

np.seterr(divide='ignore', invalid='ignore');
```

## Symbolic Calculation of the Divergence
### (be sure to also try this example by hand)

**Step 1.** Initialize the coordinate system and the vector field; display the equation for the vector field. Note that the subscript **A** simply represents the coordinate system; awkward, but don't let it throw you.

```python
A = CoordSys3D('A')  # initialize a 3D coordinate system
F = A.x*A.y*A.z*(A.i + A.j + A.k)
display(Math(r'\textbf{F} = ' + latex(F)))
```

**Step 2.** Symbolically calculate the divergence

```python
div = divergence(F)
display(Math(r'\nabla \cdot \textbf{F} = ' + latex(div)))
```

### Above we have the divergence of the vector field F

### A Few Notes:
* The *vector field* has three vector components at each point in space (i.e. x,y,z)
* The divergence of the vector field is a *scalar field*. At each point in space there is only a single scalar value.

## Numerical Evaluation of the Divergence for Display

**Step 3.** Create a function from the symbolic divergence (to evaluate at any x,y,z point)

```python
vars = symbols('A.x A.y A.z')  # seems unnecessary but is to fix a lambdify "bug"
div_func = lambdify(vars, div.subs(dict(zip([A.x, A.y, A.z], vars))), modules='numpy')
```

**Step 4.** Create a field (i.e. x, y, z arrays) and evaluate the divergence over the field.
Note: the ordering of x,y,z combined with meshgrid is [not obvious](https://github.com/maartenbreddels/ipyvolume/issues/79)

```python
x = np.linspace(0, 10, 41)
y = np.linspace(0, 10, 41)
z = np.linspace(0, 10, 41)
X, Y, Z = np.meshgrid(x,y,z, indexing='ij')
divergence_data = helpers.check_lambdify_out(div_func(X, Y, Z), X, Y, Z)
```

**Step 5.** Let's first check a few points of the divergence. Recall the expression we found:

```python
display(Math(r'\nabla \cdot \textbf{F} = ' + latex(div)))
```

```python
def my_func_eval(func, indx_x, indx_y, indx_z):
    display(Math(r'\nabla \cdot \textbf{F}' + '({},{},{}) = {}'.format(indx_x, indx_y, indx_z, func[indx_x, indx_y,indx_z])))
    return func[indx_x,indx_y,indx_z]

x = my_func_eval(divergence_data, 0, 0, 0);
x = my_func_eval(divergence_data, 0, 40, 40);
x = my_func_eval(divergence_data, 40, 0, 40);
x = my_func_eval(divergence_data, 40, 40, 0);
x = my_func_eval(divergence_data, 40, 40, 40);
```

**Step 5.** Display the divergence along the xy plane (z=0)

```python
x = np.linspace(0, 10, 41)
y = np.linspace(0, 10, 41)
z = np.linspace(0, 0, 41)
X, Y, Z = np.meshgrid(x,y,z, indexing='ij')
divergence_data = helpers.check_lambdify_out(div_func(X, Y, Z), X, Y, Z)

plt.figure()
plt.imshow(divergence_data[:,:,0])
plt.colorbar()
plt.show()
```

**Step 6.** Now display the divergence in 3D

```python
# plot a 3d figure
ipv.figure()
ipv.volshow(divergence_data, opacity=0.03, level_width=20)
# ipv.view(azimuth = 115, elevation = 90)
ipv.show()
```

### Let's Go Back to the Vector Field and Display as a Quiver Plot (a quiver plot shows magnitude and direction at each point in a field)

Recall the vector field:

```python
display(Math(r'\textbf{F} = ' + latex(F)))
```

```python
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 10)
z = np.linspace(-10, 10, 10)
X, Y, Z = np.meshgrid(x,y,z, indexing='ij');

u,v,w = helpers.evaluate_field_components(A, F, X,Y,Z);

ipv.figure()
quiver = ipv.quickquiver(X.flatten(), Y.flatten(), Z.flatten(), u.flatten(), v.flatten(), w.flatten(), size=2);
ipv.show()
```

## Sympy also supports spherical or cylindrical coordinates

## Try an example in spherical coordinates

```python
A = CoordSys3D('A', transformation='spherical')
r, theta, phi, a, b, c = symbols('r \theta phi a b c')

F = a*A.r**2*A.i + c*cos(A.theta)/A.r*A.j + b * A.k
div = divergence(F)
```

```python
display(Math(r'\textbf{F} = ' + latex(F)))
```

```python
display(Math(r'\nabla \cdot \textbf{F} = ' + latex(div)))
```

## You should check this with a hand calculation of the divergence


 Convolutional Neural Nets

# Filtering

```python
!pip install opencv-contrib-python-headless
```

    Requirement already satisfied: opencv-contrib-python-headless in /envs/pictionary/lib/python3.8/site-packages (4.2.0.34)
    Requirement already satisfied: numpy>=1.17.3 in /envs/pictionary/lib/python3.8/site-packages (from opencv-contrib-python-headless) (1.18.4)

```python
import cv2
import numpy as np
import matplotlib.pyplot as plt
```

```python
def apply_filer(img, filter):
    oimg = cv2.filter2D(img, -1, filter)
    return oimg
```

### Detection using filters

We already know that we can use filters to detect arbitarily complex shapes.
Let's take the example of a square detector.

```python
img = np.ones((100,100))
for i in range(1,4):
    img[i*20:i*20+10, i*20:i*20+10] = 0

plt.imshow(img, cmap="gray")

```

```python
# all_zeros = np.zeros((11,11))
square_detector = -np.ones((11,11))
square_detector[-1, :] = 1
square_detector[0, :] = 1
square_detector[:, -1] = 1
square_detector[:, 0] = 1
square_detector
```

    array([[ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1., -1., -1., -1., -1., -1., -1., -1., -1., -1.,  1.],
           [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.]])

```python
oimg = apply_filer(img, square_detector)
oimg = oimg * (oimg>0)
plt.imshow(oimg, cmap="gray")
```

### Heirarchical Features

Another approach to solve the above problem is to use very simple filters and then use the information captured by each of them using another simple filter, to finally get the desired output.

```python
verticle_filter = np.array([[-1, 0, 1], [-1, 0, 1], [-1, 0, 1]])
verticle_filter
```

    array([[-1,  0,  1],
           [-1,  0,  1],
           [-1,  0,  1]])

```python
oimg1 = apply_filer(img, verticle_filter)
oimg1 = oimg1 * (oimg1 >=0)
plt.imshow(oimg1, cmap="gray")
```

```python
horizontal_filter = np.array([[-1, -1, -1], [0, 0, 0], [1, 1, 1]])
horizontal_filter
```

    array([[-1, -1, -1],
           [ 0,  0,  0],
           [ 1,  1,  1]])

```python
oimg2 = apply_filer(img, horizontal_filter)
oimg2 = oimg2 * (oimg2 >=0)
plt.imshow(oimg2, cmap="gray")
```

```python
verticle_filter = np.array([[1, 0, -1], [1, 0, -1], [1, 0, -1]])
print(verticle_filter)
oimg3 = apply_filer(img, verticle_filter)
oimg3 = oimg3 * (oimg3 >=0)
plt.imshow(oimg3, cmap="gray")
```

```python
fimg = np.dstack((oimg1/oimg1.max(), oimg2/oimg2.max(), oimg3/oimg3.max()))
plt.imshow(fimg)
fimg.shape
```

**Filter for bottom right corner detection**

*R channel*
$$
\begin{pmatrix}
-1 & 1 & 0 \\
-1 & 0 & 0 \\
-1 & -1 & -1
\end{pmatrix}
$$

*G channel*
$$
\begin{pmatrix}
-1 & -1 & -1 \\
1 & 0 & -1 \\
0 & 0 & -1
\end{pmatrix}
$$

*B channel*
$$
\begin{pmatrix}
0 & 0 & 0 \\
0 & 0 & 0 \\
0 & 0 & 0
\end{pmatrix}
$$

```python
corner_filter = np.zeros((3,3,3))
corner_filter[:,0,0] = -1
corner_filter[2,:,0] = -1
corner_filter[0,1,0] = 1

corner_filter[0,:,1] = -1
corner_filter[:,2,1] = -1
corner_filter[1,0,1] = 1
print(corner_filter[:, :, 1])

ffimg = np.zeros((fimg.shape[0], fimg.shape[1]))
for i in range(fimg.shape[0]-2):
    for j in range(fimg.shape[1]-2):
        ffimg[i, j] = np.sum(fimg[i:i+3, j:j+3, :]*corner_filter)

ffimg = ffimg * (ffimg>0)
plt.imshow((ffimg-fimg.min())/(fimg.max()-fimg.min()), cmap="gray")
ffimg.shape
```

So if we use the information by the 3 filters together our detectors will be able to work better

### Conclusion
1. We can have arbitarily complex filters to detect various features.
2. We can stack the filtered images to pass on more information to processes downstream

# Why we need CNNs? Why MLPs might be bad?

## Problems

### 1. MLPs get too big with too many parameters.

Let's say we have images of size 28*28 (MNIST). We use the following architecture on it. Roughly how many parameters are there?

So basically, if we try to make the network even slightly deeper, we end up with too many params.
Depth might be desirable due to heirarchy of features. But if there are too many params, then overfitting can happen.

### 2. MLPs are bad at capturing spatial relations.

$$
\begin{equation}
\begin{pmatrix}
1 & 2 & 3 & 4 \\
5 & 6 & 7 & 8
\end{pmatrix} \rightarrow
\begin{pmatrix}
1 & 2 & 3 & 4 & 5 & 6 & 7 & 8
\end{pmatrix}
\end{equation}
$$

## Solutions
What if we just learn the filters?

### Weight Sharing.
We don't have different weights for each input neuron. We are just learning the filters. So we are "sharing" weights between inputs.

### Pooling.
This helps in making features "position invariant". Let's say we are detecting a bird.
We would like the head filter to give a high response to both the heads (even though they are flipped). Also, we want the full bird detector to give a high respnse for both the birds, even though they are completely flipped.

# General Architecture of CNNs

The main thing to note here is that, we have fully connected layers at the end. Which is your basic MLP. So instead of directly flattening the input image and appling MLP on it, we are applying some "filters" (which are learned by the network), and then getting some "features" (Which are high level and position invariant). We use these "features" as input to a standard mlp. So CNNs are basically feature extractors.

```python

```


```python
%matplotlib inline
```

```python
import numpy as np
import matplotlib.pyplot as plt
#import librosa
from scipy import fftpack
from scipy import signal
#from ipywidgets import interactive
#from IPython.display import Audio, display
```

# 592B, Class 6.2 (10/07/21).  Convolution

## Intro
Last time we said that windowing, filtering, and smoothing, are the same operations from a mathematical perspective. That operation is called **convolution**. The convolution of two functions $f(t)$ and $g(t)$ is defined as:

$ f * g = \int_{-\infty}^{\infty} f(\tau)g(t-\tau)d\tau $

The motto that goes with this is: flip and shift and compute the overlapping area.

Here are some examples:

A rectangular pulse with itself:

A rectangular pulse with itself, again:

A rectangular pulse with a spiky function:

A rectangular pulse with a spiky function, again:

And a Gaussian with a Gaussian:

If we look at the [source code for the Gaussian window function scipy.signal.gaussian](https://github.com/scipy/scipy/blob/v1.0.0/scipy/signal/windows.py#L1159-L1219), it's basically just this (where $M$ is the window length):

```python

n = np.arange(0, M) - (M - 1.0) / 2.0
sig2 = 2 * std * std
w = np.exp(-n ** 2 / sig2)

```
And in the Elegant Scipy tutorial, the windowing happens with this:

```python

t = np.linspace(0, 1, 500)
x = np.sin(49 * np.pi * t)

X = fftpack.fft(x)
win = np.kaiser(len(t), 5)
X_win = fftpack.fft(x * win)

plt.plot(fftpack.fftfreq(len(t)), np.abs(X_win))
plt.ylim(0, 190);

```

As we noticed last time, the windowing is happening in the frequency domain, via plain multiplication, taking advantage of the Convolution Theorem: the Fourier transform of the convolution of two functions in the time domain is equivalent to the pointwise multiplication of the Fourier transforms of the two functions in the frequency domain:

$$ \mathcal{F}(g \ast f)(s) = \mathcal{F}g(s)\mathcal{F}f(s)$$

Moreover, we can also show (see Osgood 2010 notes, p. 98):

$$ \mathcal{F}(gf)(s) = (\mathcal{F}g \ast \mathcal{F} f)(s) $$

## Review questions

1. Compute the convolution of a Gaussian function, using `np.convolve`. You can use the Gaussian window code above as a starting point.  What does the result look like? And what is the convolution of the result with itself?

2. Try to explain why there is an inverse relationship between window length and frequency resolution when you compute a spectrum (or spectrogram).

Sample code from [fft convolve docs](https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.fftconvolve.html#scipy.signal.fftconvolve)

```python
from scipy import misc

face = misc.face(gray=True)
kernel = np.outer(signal.windows.gaussian(70, 8),signal.windows.gaussian(70, 8))
blurred = signal.fftconvolve(face, kernel, mode='same')
```

```python
fig, (ax_orig, ax_kernel, ax_blurred) = plt.subplots(3, 1, figsize=(6, 15))

ax_orig.imshow(face, cmap='gray')
ax_orig.set_title('Original')
ax_orig.set_axis_off()

ax_kernel.imshow(kernel, cmap='gray')
ax_kernel.set_title('Gaussian kernel')
ax_kernel.set_axis_off()

ax_blurred.imshow(blurred, cmap='gray')
ax_blurred.set_title('Blurred')
ax_blurred.set_axis_off()

fig.show()

```

```python
M = 128; std = 8
n = np.arange(0, M) - (M - 1.0) / 2.0
sig2 = 2 * std * std
w = np.exp(-n ** 2 / sig2)

w2 = np.convolve(w,w)

w4 = np.convolve(w2,w2)

plt.plot(w2)

plt.figure()
plt.plot(w4)

```

## Example of Convolution Theorem with digital convolution

```python

x = np.array([5, 6, 8, 2, 5])
y = np.array([6, -1, 3, 5, 1])

x1 = np.concatenate((x, np.zeros(4)), axis=0)
y1 = np.concatenate((y, np.zeros(4)), axis=0)

c1 = fftpack.ifft(fftpack.fft(x1) * fftpack.fft(y1));
c2 = np.convolve(x,y);

print(c1)
print(c2)
```

    [30.+0.00000000e+00j 31.-3.15796771e-15j 57.+3.15796771e-15j
     47.-3.41860033e-16j 87.-1.15589641e-15j 47.-1.57898386e-15j
     33.+3.41860033e-16j 27.+4.31386412e-15j  5.-1.57898386e-15j]
    [30 31 57 47 87 47 33 27  5]

## Filters

The code below is adapted from this tutorial [signal filtering demo](https://www.swharden.com/wp/2009-01-21-signal-filtering-with-python/) (with some corrections).

The key point to get from this is to look at how filtering works from a frequency domain perspective, and to also see how that is related to what is going on in the time domain.

```python
t_s = 0.01 # sampling interval
fs = 1/t_s # sampling rate
print("Sampling rate: ", fs, " Hz")

t = np.arange(1,100,t_s)

f_sig = 0.3/(2*np.pi)   # frequency of signal
f_n1 = 2.33/(2*np.pi)   # frequency of noise source 1
f_n2 = 2.77/(2*np.pi)    # frequency of noise source 2

print("Frequency of signal: ", f_sig, " Hz")
print("Frequency of electrical noise 1: ", f_n1, " Hz")
print("Frequency of electrical noise 2: ", f_n2, " Hz")

signal = np.sin(2*np.pi*f_sig*t) #(A)

sin_n1 = 0.333* np.sin(2*np.pi*f_n1*t) # (B) noise source 1
sin_n2 = 0.777* np.sin(2*np.pi*f_n2*t) # (B) noise source 2

noise = sin_n1 + sin_n2 # (C)  Combined noise sources

static = (np.random.random_sample((len(t)))-.5)*2 # (D)  Uniform distibution noise, 0.5 to 0.5

sigstat = static + signal # (E) Signal and static

rawsignal = sigstat + noise # (F) Signal + all sources of noise

```

    Sampling rate:  100.0  Hz
    Frequency of signal:  0.0477464829275686  Hz
    Frequency of electrical noise 1:  0.37083101740411617  Hz
    Frequency of electrical noise 2:  0.4408591923645501  Hz

```python
#pylab.subplot(h,w,1);pylab.title("(A) Original Signal")
plt.plot(t,signal)
plt.title("(A) Original Signal")
plt.show()

plt.title("(B) Electrical Noise Sources (2 Sine Waves)")
plt.plot(t,sin_n1,label="sin_n1")
plt.plot(t,sin_n2,label="sin_n2")
plt.legend()
plt.show()

plt.title("(C) Electrical Noise (2 sine waves added together)")
plt.plot(t,noise)
plt.show()

plt.title("(D) Static (random noise)")
plt.plot(t,static)
plt.axis([None,None,-1,1])
plt.show()

plt.title("(E) Signal + Static")
plt.plot(t,sigstat)
plt.show()

plt.title("(F) Recording (Signal + Static + Electrical Noise)")
plt.plot(t,rawsignal)
```

```python
fft_sig = fftpack.fft(rawsignal) # (G) and (H)

plt.title("(G) FFT of Recording")
plt.plot(np.abs(fft_sig))

plt.figure()
plt.title("(G1) First 300 points")
plt.plot(np.abs(fft_sig)[0:300])

plt.figure()
plt.title("(G2) Last 300 points")
plt.plot(np.abs(fft_sig)[-300:-1])
```

```python
freqs = fftpack.fftfreq(len(rawsignal)) * fs
plt.plot(freqs)

plt.figure()
plt.title("(G) FFT of Recording, frequencies shifted")
plt.plot(freqs,np.abs(fft_sig))
plt.axis([-fs/2,fs/2,None,None])  # Why fs/2?
```

```python
plt.title("(H) FFT zoomed in to low freqs")
plt.plot(freqs,np.abs(fft_sig))
plt.axis([-1,1,None,None])
plt.show()
# Where is the static? Try increasing amplitude of Gaussian noise (increasing variance)
```

Let's low pass filter to remove the static and noise sources.
The definition of a low pass filter that passes frequencies $s < \nu_c$ in the frequency domain is the following:

$$Low(s)  = \left\{
        \begin{array}{ll}
            1 & \quad \lvert s \rvert < \nu_c \\
            0 & \quad otherwise
        \end{array}
         \right.$$

 So we need $\nu_c$ to be smaller than those noise sources.

 ***What should we set $\nu_c$ to be?***

Note also that this is just a scaled rect function, i.e., $$\Pi\left(\frac{s}{2\nu_c}\right)$$ and so in the time domain, we have:

 $$low(t) = 2\nu_c sinc(2\nu_ct)$$

```python
x = np.linspace(-4, 4, 41)
plt.plot(x,np.sinc(x))
plt.title("Sinc functions")

fc = 2  # set critical frequency

h = 2 * fc* np.sinc(2 * fc * x)
plt.plot(x,h)
```

```python
lowpass_filter = np.zeros(len(rawsignal))

# Set first 25 and last 25 values to 1
lowpass_filter[0:25] = 1
lowpass_filter[-25:-1] = 1

fil_fft_sig = fft_sig * lowpass_filter

plt.title("(H) Filtered FFT of Recording")
plt.plot(np.abs(fil_fft_sig))

plt.figure()
plt.title("(H1) First 300 points")
plt.plot(np.abs(fil_fft_sig)[0:300])

```

Now we can take the inverse Fourier transform to get back to the time domain!

```python
inv_fil_fft = fftpack.ifft(fil_fft_sig)
plt.title("(I) Inverse FFT")
plt.plot(t,inv_fil_fft)
plt.show()
```

## Time-limited signals and window length

Do the exercises I found [here](https://www.gaussianwaves.com/2011/01/fft-and-spectral-leakage-2/). Note that the code is in Matlab; you'll need to port to Python and modify as necessary.

Things to think about and answer:

- What causes spectral leakage?

## DFT and frequency/time trade-off

Remember that we ended up with the Fourier series of $g(t)$ defined as $T \rightarrow \infty$, expressed as a double integral:

\begin{equation}
g(t) = \int_{-\infty}^{\infty} \int_{-\infty}^{\infty}e^{-2\pi ift}g(t)dt e^{2\pi ift}df
\end{equation}

and then we derived $\mathcal{F}g$, where $\mathcal{F}g$ is defined as the Fourier transform of a function $g(t)$:

\begin{equation}
\mathcal{F}g = \int_{-\infty}^{\infty}e^{-2\pi ift} g(t)dt
\end{equation}

and the inverse Fourier transform $\mathcal{F}^{-1}(t)$ of a function $\mathcal{F}g(f)$ is:

\begin{equation}
g(t) = \int_{-\infty}^{\infty}e^{2\pi ift} \mathcal{F}gdf
\end{equation}

Then we briefly introduced the **discrete Fourier transform (DFT)**: this is what we use when we're computing the Fourier Transform in python because we are working with *digital* signals.

## The Discrete Fourier Transform

Today we're going to work on understanding the DFT a little more, because it will help us understand what's going on when we invoke `fftpack.fft`. The derivation here follows Osgood Chapter 6.

Our goal is to find a discrete version of $\mathcal{F}g(f)$, the Fourier transform of a function $g(t)$. (Note: by writing $\mathcal{F}g(f)$, we mean that the function $\mathcal{F}g$, the Fourier transform of $g(t)$, is a function of frequency, $f$. We start with $g(t)$, which is a function of time $t$, but once we take the Fourier transform of $g(t)$, $\mathcal{F}g$, we have a function of frequency $f$.).

To do this, we need to find three things:
- A discrete version of $g(t)$ that reasonably approximates $g(t)$
- A discrete version of $\mathcal{F}g(f)$ that reasonably approximates $\mathcal{F}g(f)$
- A way in which these two discrete versions are related, which approximates the relation between the continuous versions

We start by assuming that $g(t)$ is:
- *time-limited*, meaning that $g(t)$ is zero outside of $0\leq t \leq L$, where $L$ is the length of the signal (in time)
- *band-limited*, meaning that $\mathcal{F}g(f)$ vanishes outside of $0 \lt f \lt 2B$, where $B$ is the *bandwidth* of the signal.

By the Sampling Theorem, if we sample $g(t)$ at the Nyquist rate of $2B$ samples/second, we can reconstruct $g(t)$ perfectly. This sampled version of $g(t)$, call it $g_{discrete}(t)$, is just a list of $N$ sampled values:

$$ g(t_0), g(t_1), \ldots, g(t_{N-1}) $$,

where $N=2BL$ and the timepoints are evenly spaced apart by $\frac{1}{2B}$.

***In-class exercise: Why is $N=2BL$ and why does $\Delta t_n = \frac{1}{2B}$?***

---

### The Dirac comb

We can re-express $g_{discrete}(t)$ using the Dirac comb $III(t)$, defined as:

$$III(t) = \displaystyle\sum_{n=0}^{N-1} \delta(t-t_n) $$

All this is is a "train" of pulses, a comb of "lollipops" with amplitude 1, where the pulses occur exactly at the sampled points $t_0, t_1, \ldots, t_{N-1}$.

Here's an image of a Dirac comb from Wikipedia. In our case, $T = 1/2B$.

And here's an example of [one way to define a Dirac comb function](https://scipython.com/book/chapter-6-numpy/examples/a-comb-function/), from [scipython.com](https://scipython.com).

```python
N, n = 101, 5
def f(i):
    return (i % n == 0) * 1

comb = np.fromfunction(f, (N,), dtype=int)
print(comb)

```

    [1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0
     0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0
     0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1]

***In-class discussion: Does the Dirac comb form an orthogonal basis set?***

---

Using $III(t)$, we can now express $g_{discrete}(t)$ as:

$$g_{discrete}(t) = g(t) \displaystyle\sum_{n=0}^{N-1} \delta(t-t_n) = \displaystyle\sum_{n=0}^{N-1} g(t) \delta(t-t_n) $$

And the Fourier transform of $g_{discrete}(t)$ is:

\begin{equation}
\mathcal{F}g_{discrete}(f) = \displaystyle\sum_{n=0}^{N-1} = \mathcal{F}g(t_n) \delta(t-t_n) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi ift_n}
\end{equation}

This gives us the continuous Fourier transform of the sampled version of $g(t)$.

Now let's think about $g(t)$ in the frequency domain. Remember by assumption that $g(t)$ is time-limited so $g(t)$ is zero outside of $0\leq t \leq L$, where $L$ is the length of the signal (in time). So we can apply the Sampling Theorem to reconstruct $\mathcal{F}g(f)$ in the frequency domain. The sampling rate we need (the Nyquist rate) for perfect reconstruction is $L$ samples/Hz and the spacing between sampling points is $1/L$.

Since $\mathcal{F}g(f)$ is band-limited by assumption and  vanishes outside of $0 \lt f \lt 2B$, we sample $\mathcal{F}g(f)$ over $0 \lt f \lt 2B$, with points $1/L$ Hz apart.

---

***In-class exercise: Why is the sampling rate $L$ samples/Hz and why is the interval between sampling points $1/L$ Hz? What is the total number of sampling points, $N$***

---

This sampled version of $\mathcal{F}g(f)$, call it $\mathcal{F}g_{discrete}(f)$, is just a list of $N$ sampled values, of the form $m/L$, where $m$ is a non-negative integer:

$$ f_0=0,\, f_1 = \frac{1}{L},\, \ldots, f_{N-1} = \frac{N-1}{L} $$,

And if we want the discrete version of $g_{discrete}(t)$, then we want to take $[\mathcal{F}(g_{discrete})](f)$, call this $F(f)$ for short.

Taking our definition of the Fourier transform of $g_{discrete}(t)$,

$$\mathcal{F}g(t_n) \delta(t-t_n) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi ift_n}$$

this will give us the list:

$$ F(f_0) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi if_0t_n}, \ldots, F(f_{N-1})=\displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi if_{N-1}t_n}  $$

And so now we have a way to go from $g_{discrete}(t)$ to $\mathcal{F}g_{discrete}(f)$, for each $m$ from $m=0$ to $m=N-1$:

$$F(f_m) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi if_mt_n} $$

Recalling that $t_n = \frac{n}{2B}$ and $f_m = \frac{m}{L}$ and $N=2BL$, we can re-write this as:

$$F(f_m) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi inm/N} $$

***In-class exercise: derive our final expression of $F(f_m)$.***

---

At this point, let's come back to one of our starting questions and discuss. You should have more insight on this now! What is the "grid" spacing in the time-domain? The frequency domain? How are they related?

> Why is the computation of the spectrum affected by the "window length" over which it is computed, and how is it affected?

---

## Positive and negative frequencies (Osgood 2010, p. 260)

Given our discrete Fourier transform $[\mathcal{F}(g_{discrete})](f)$, call this $F(f)$:

$$F(f_m) = \displaystyle\sum_{n=0}^{N-1} g(t_n) e^{-2\pi inm/N} $$

it turns out that the spectrum *splits* at $N/2$. See Osgood (2010) for the derivation, but due to some periodicity relations:

$$ F[\frac{N}{2} + 1] = \overline{F[\frac{N}{2} - 1]}$$
$$ F[\frac{N}{2} + 2] = \overline{F[\frac{N}{2} - 2]}$$
$$ \vdots$$

***In-class exercise: What is F[0]? What do the periodicity relations mean geometrically?***

So because of this, the convention is to say, for a spectrum indexed from 0 to $N-1$:
- The frequencies from $m=1$ to $m= N/2-1$ are the "positive" frequencies
- The frequencies from $m=N/2+1$ to $m= N-1$ are the "negative" frequencies

For a real signal, all the information you need is in the positive frequencies and the first component $F[0]$.


# Periodic homogenization of linear elastic materials

## Introduction

This tour will show how to perform periodic homogenization of linear elastic materials. The considered 2D plane strain problem deals with a skewed unit cell of dimensions $1\times \sqrt{3}/2$ consisting of circular inclusions (numbered $1$) of radius $R$ with elastic properties $(E_r, \nu_r)$ and embedded in a matrix material (numbered $0$) of properties $(E_m, \nu_m)$ following an hexagonal pattern. A classical result of homogenization theory ensures that the resulting overall behavior will be isotropic, a property that will be numerically verified later.

```python
from __future__ import print_function
from dolfin import *
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib notebook
%matplotlib inline
```

```python
a = 1.         # unit cell width
b = sqrt(3.)/2. # unit cell height
c = 0.5        # horizontal offset of top boundary
R = 0.2        # inclusion radius
vol = a*b      # unit cell volume
# we define the unit cell vertices coordinates for later use
vertices = np.array([[0, 0.],
                     [a, 0.],
                     [a+c, b],
                     [c, b]])
fname = "hexag_incl"
mesh = Mesh(fname + ".xml")
subdomains = MeshFunction("size_t", mesh, fname + "_physical_region.xml")
facets = MeshFunction("size_t", mesh, fname + "_facet_region.xml")
plt.figure(dpi=80)
plt.subplot(1,2,1)
plot(subdomains)
plt.subplot(1,2,2)
plot(mesh)
```

**Remark**: `mshr` does not allow to generate a meshed domain with perfectly matching vertices on opposite boundaries as would be required when imposing periodic boundary conditions. For this reason, we used a `Gmsh`-generated mesh.

## Periodic homogenization framework

The goal of homogenization theory consists in computing the apparent elastic moduli of the homogenized medium associated with a given microstructure. In a linear elastic setting, this amounts to solving the following auxiliary problem defined on the unit cell $\mathcal{A}$:

$$\begin{equation}\begin{cases}\operatorname{div} \boldsymbol{\sigma} = \boldsymbol{0} & \text{in } \mathcal{A} \\
\boldsymbol{\sigma} = \mathbb{C}(\boldsymbol{y}):\boldsymbol{\varepsilon} & \text{for }\boldsymbol{y}\in\mathcal{A} \\
\boldsymbol{\varepsilon} = \boldsymbol{E} + \nabla^s \boldsymbol{v} & \text{in } \mathcal{A} \\
\boldsymbol{v} & \text{is } \mathcal{A}\text{-periodic} \\
\boldsymbol{T}=\boldsymbol{\sigma}\cdot\boldsymbol{n} & \text{is } \mathcal{A}\text{-antiperiodic}
\end{cases} \label{auxiliary-problem}
\end{equation}$$

where $\boldsymbol{E}$ is the **given** macroscopic strain, $\boldsymbol{v}$ a periodic fluctuation and $\mathbb{C}(\boldsymbol{y})$ is the heterogeneous elasticity tensor depending on the microscopic space variable $\boldsymbol{y}\in\mathcal{A}$. By construction, the local microscopic strain is equal on average to the macroscopic strain: $\langle \boldsymbol{\varepsilon} \rangle = \boldsymbol{E}$. Upon defining the macroscopic stress $\boldsymbol{\Sigma}$ as the microscopic stress average: $\langle \boldsymbol{\sigma} \rangle = \boldsymbol{\Sigma}$, there will be a linear relationship between the auxiliary problem loading parameters $\boldsymbol{E}$ and the resulting average stress:

$$\boldsymbol{\Sigma}  = \mathbb{C}^{hom}:\boldsymbol{E}$$

where $\mathbb{C}^{hom}$ represents the apparent elastic moduli of the homogenized medium. Hence, its components can be computed by solving elementary load cases corresponding to the different components of $\boldsymbol{E}$ and performing a unit cell average of the resulting microscopic stress components.

### Total displacement as the main unknown

The previous problem can also be reformulated by using the total displacement $\boldsymbol{u} = \boldsymbol{E}\cdot\boldsymbol{y} + \boldsymbol{v}$ as the main unknown with now $\boldsymbol{\varepsilon} = \nabla^s \boldsymbol{u}$. The periodicity condition is therefore equivalent to the following constraint:

$$\boldsymbol{u}(\boldsymbol{y}^+)-\boldsymbol{u}(\boldsymbol{y}^-) = \boldsymbol{E}\cdot(\boldsymbol{y}^+-\boldsymbol{y}^-)$$

where $\boldsymbol{y}^{\pm}$ are opposite points on the unit cell boundary related by the periodicity condition. This formulation is widely used in solid mechanics FE software as it does not require specific change of the problem formulation but just adding tying constraints between some degrees of freedom.

This formulation is however not easy to translate in FEniCS. It would indeed require introducing Lagrange multipliers defined on some part of the border only, a feature which does not seem to be available at the moment.

### Periodic fluctuation as the main unknown

Instead, we will keep the initial formulation and consider the periodic fluctuation $\boldsymbol{v}$ as the main unknown. The periodicity constraint on $\boldsymbol{v}$ will be imposed in the definition of the associated FunctionSpace using the ``constrained_domain`` optional keyword. To do so, one must define the periodic map linking the different unit cell boundaries. Here the unit cell is 2D and its boundary is represented by a parallelogram of vertices ``vertices`` and the corresponding base vectors `a1` and `a2` are computed. The right part is then mapped onto the left part, the top part onto the bottom part and the top-right corner onto the bottom-left one.

```python
# class used to define the periodic boundary map
class PeriodicBoundary(SubDomain):
    def __init__(self, vertices, tolerance=DOLFIN_EPS):
        """ vertices stores the coordinates of the 4 unit cell corners"""
        SubDomain.__init__(self, tolerance)
        self.tol = tolerance
        self.vv = vertices
        self.a1 = self.vv[1,:]-self.vv[0,:] # first vector generating periodicity
        self.a2 = self.vv[3,:]-self.vv[0,:] # second vector generating periodicity
        # check if UC vertices form indeed a parallelogram
        assert np.linalg.norm(self.vv[2, :]-self.vv[3, :] - self.a1) <= self.tol
        assert np.linalg.norm(self.vv[2, :]-self.vv[1, :] - self.a2) <= self.tol

    def inside(self, x, on_boundary):
        # return True if on left or bottom boundary AND NOT on one of the
        # bottom-right or top-left vertices
        return bool((near(x[0], self.vv[0,0] + x[1]*self.a2[0]/self.vv[3,1], self.tol) or
                     near(x[1], self.vv[0,1] + x[0]*self.a1[1]/self.vv[1,0], self.tol)) and
                     (not ((near(x[0], self.vv[1,0], self.tol) and near(x[1], self.vv[1,1], self.tol)) or
                     (near(x[0], self.vv[3,0], self.tol) and near(x[1], self.vv[3,1], self.tol)))) and on_boundary)

    def map(self, x, y):
        if near(x[0], self.vv[2,0], self.tol) and near(x[1], self.vv[2,1], self.tol): # if on top-right corner
            y[0] = x[0] - (self.a1[0]+self.a2[0])
            y[1] = x[1] - (self.a1[1]+self.a2[1])
        elif near(x[0], self.vv[1,0] + x[1]*self.a2[0]/self.vv[2,1], self.tol): # if on right boundary
            y[0] = x[0] - self.a1[0]
            y[1] = x[1] - self.a1[1]
        else:   # should be on top boundary
            y[0] = x[0] - self.a2[0]
            y[1] = x[1] - self.a2[1]
```

We now define the constitutive law for both phases:

```python
Em = 50e3
num = 0.2
Er = 210e3
nur = 0.3
material_parameters = [(Em, num), (Er, nur)]
nphases = len(material_parameters)
def eps(v):
    return sym(grad(v))
def sigma(v, i, Eps):
    E, nu = material_parameters[i]
    lmbda = E*nu/(1+nu)/(1-2*nu)
    mu = E/2/(1+nu)
    return lmbda*tr(eps(v) + Eps)*Identity(2) + 2*mu*(eps(v)+Eps)
```

## Variational formulation

The previous problem is very similar to a standard linear elasticity problem, except for the periodicity constraint which has now been included in the FunctionSpace definition and for the presence of an eigenstrain term $\boldsymbol{E}$. It can easily be shown that the variational formulation of the previous problem reads as: Find $\boldsymbol{v}\in V$ such that:

$$\begin{equation}
F(\boldsymbol{v},\widehat{\boldsymbol{v}}) = \int_{\mathcal{A}} (\boldsymbol{E}+\nabla^s\boldsymbol{v}):\mathbb{C}(\boldsymbol{y}):\nabla^s\widehat{\boldsymbol{v}}\text{ d} \Omega = 0 \quad \forall \widehat{\boldsymbol{v}}\in V
\end{equation}$$

The above problem is not well-posed because of the existence of rigid body translations. One way to circumvent this issue would be to fix one point but instead we will add an additional constraint of zero-average of the fluctuation field $v$ as is classically done in homogenization theory. This is done by considering an additional vectorial Lagrange multiplier $\lambda$ and considering the following variational problem (see the [pure Neumann boundary conditions FEniCS demo](https://fenicsproject.org/docs/dolfin/2019.1.0/python/demos/neumann-poisson/demo_neumann-poisson.py.html) for a similar formulation): Find $(\boldsymbol{v},\boldsymbol{\lambda})\in V\times \mathbb{R}^2$ such that:

$$\begin{equation}
\int_{\mathcal{A}} (\boldsymbol{E}+\nabla^s\boldsymbol{v}):\mathbb{C}(\boldsymbol{y}):\nabla^s\widehat{\boldsymbol{v}}\text{ d} \Omega + \int_{\mathcal{A}} \boldsymbol{\lambda}\cdot\widehat{\boldsymbol{v}} \text{ d} \Omega + \int_{\mathcal{A}} \widehat{\boldsymbol{\lambda}}\cdot\boldsymbol{v} \text{ d} \Omega = 0 \quad \forall (\widehat{\boldsymbol{v}}, \widehat{\boldsymbol{\lambda}})\in V\times\mathbb{R}^2
\end{equation}$$

Which can be summarized as:

$$\begin{equation}
a(\boldsymbol{v},\widehat{\boldsymbol{v}}) + b(\boldsymbol{\lambda},\widehat{\boldsymbol{v}}) + b(\widehat{\boldsymbol{\lambda}}, \boldsymbol{v}) = L(\widehat{\boldsymbol{v}}) \quad \forall (\widehat{\boldsymbol{v}}, \widehat{\boldsymbol{\lambda}})\in V\times\mathbb{R}^2
\end{equation}$$

This readily translates into the following FEniCS code:

```python
Ve = VectorElement("CG", mesh.ufl_cell(), 2)
Re = VectorElement("R", mesh.ufl_cell(), 0)
W = FunctionSpace(mesh, MixedElement([Ve, Re]), constrained_domain=PeriodicBoundary(vertices, tolerance=1e-10))
V = FunctionSpace(mesh, Ve)

v_,lamb_ = TestFunctions(W)
dv, dlamb = TrialFunctions(W)
w = Function(W)
dx = Measure('dx')(subdomain_data=subdomains)

Eps = Constant(((0, 0), (0, 0)))
F = sum([inner(sigma(dv, i, Eps), eps(v_))*dx(i) for i in range(nphases)])
a, L = lhs(F), rhs(F)
a += dot(lamb_,dv)*dx + dot(dlamb,v_)*dx
```

We have used a general implementation using a sum over the different phases for the functional `F`. We then used the `lhs` and `rhs` functions to respectively extract the corresponding bilinear $a$ and linear $L$ forms.

## Resolution

The resolution of the auxiliary problem is performed for elementary load cases consisting of uniaxial strain and pure shear sollicitations by assigning unit values of the correspnonding $E_{ij}$ components. For each load case, the average stress $\boldsymbol{\Sigma}$ is computed components by components and the macroscopic stiffness components $\mathbb{C}^{hom}$ are then printed.

```python
def macro_strain(i):
    """returns the macroscopic strain for the 3 elementary load cases"""
    Eps_Voigt = np.zeros((3,))
    Eps_Voigt[i] = 1
    return np.array([[Eps_Voigt[0], Eps_Voigt[2]/2.],
                    [Eps_Voigt[2]/2., Eps_Voigt[1]]])
def stress2Voigt(s):
    return as_vector([s[0,0], s[1,1], s[0,1]])

Chom = np.zeros((3, 3))
for (j, case) in enumerate(["Exx", "Eyy", "Exy"]):
    print("Solving {} case...".format(case))
    Eps.assign(Constant(macro_strain(j)))
    F = sum([inner(sigma(dv, i, Constant(macro_strain(j))), eps(v_))*dx(i) for i in range(nphases)]) + dot(lamb_,dv)*dx + dot(dlamb,v_)*dx
    a, L = lhs(F), rhs(F)
    solve(a == L, w, [])
    (v, lamb) = split(w)
    Sigma = np.zeros((3,))
    for k in range(3):
        Sigma[k] = assemble(sum([stress2Voigt(sigma(v, i, Eps))[k]*dx(i) for i in range(nphases)]))/vol
    Chom[j, :] = Sigma

print(np.array_str(Chom, precision=2))
```

    Solving Exx case...
    Solving Eyy case...
    Solving Exy case...
    [[ 6.56e+04  1.74e+04 -2.43e-02]
     [ 1.74e+04  6.56e+04 -4.10e-02]
     [-2.43e-02 -4.10e-02  2.41e+04]]

```python
for (j, case) in enumerate(["Exx", "Eyy", "Exy"]):
    print("Solving {} case...".format(case))
    print(j)
```

It can first be verified that the obtained macroscopic stiffness is indeed symmetric and that the corresponding behaviour is quasi-isotropic (up to the finite element discretization error). Indeed, if $\lambda^{hom} = \mathbb{C}_{xxyy}$ and $\mu^{hom} = \mathbb{C}_{xyxy}$ we have that $\mathbb{C}_{xxxx}\approx\mathbb{C}_{yyyy}\approx \mathbb{C}_{xxyy}+2\mathbb{C}_{xyxy} = \lambda^{hom}+2\mu^{hom}$.

> **Note:** The macroscopic stiffness is not exactly symmetric because we computed it from the average stress which is not stricly verifying local equilibrium on the unit cell due to the FE discretization. A truly symmetric version can be obtained from the computation of the bilinear form for a pair of solutions to the elementary load cases.

```python
lmbda_hom = Chom[0, 1]
mu_hom = Chom[2, 2]
print(Chom[0, 0], lmbda_hom + 2*mu_hom)
```

We thus deduce that $E^{hom} = \mu^{hom}\dfrac{3\lambda^{hom}+2\mu^{hom}}{\lambda^{hom}+\mu^{hom}}$ and $\nu^{hom} = \dfrac{\lambda^{hom}}{2(\lambda^{hom}+\mu^{hom})}$ that is:

```python
E_hom = mu_hom*(3*lmbda_hom + 2*mu_hom)/(lmbda_hom + mu_hom)
nu_hom = lmbda_hom/(lmbda_hom + mu_hom)/2
print("Apparent Young modulus:", E_hom)
print("Apparent Poisson ratio:", nu_hom)
```

```python
# plotting deformed unit cell with total displacement u = Eps*y + v
y = SpatialCoordinate(mesh)
plt.figure()
p = plot(0.5*(dot(Eps, y)+v), mode="displacement", title=case)
plt.colorbar(p)
plt.show()
```

```python

```


# 第二讲：监督学习应用-线性回归

我们从监督学习的一些例子开始讲起。假设我们有一个经调查得到的数据集，内容是关于俄勒冈州波特兰地区$47$间出租公寓的面积与价格之间的关系表：

$$\begin{array}{c|c}living\ area(feet^2)&price(1000\$s)\\\hline2104&400\\1600&330\\2400&369\\1416&232\\3000&540\\\vdots&\vdots\end{array}$$

我们可以画出一张关于面积-价格的图像：

有了这样的数据，我们怎样才能预测波特兰地区其他出租公寓的价格，比如推导出一个从面积得出价格的函数。

为了今后的课程书写方便，我们约定一些未来将要用到的符号：

* 使用$x^{(i)}$来表示**输入变量**，也称作**输入特征（feature）**，即本例中的实用面积（living area）；
* 使用$y^{(i)}$来表示**输入变量**或**目标变量（target）**，也就是我们尝试做出预测的值；
* 一对$(x^{(i)}),\ y^{(i)}$称作一个**训练样本（training example）**；
* 用于做学习的数据集，也就是由$m$个$(x^{(i)}),\ y^{(i)}$组成的列表$\left\{\left(x^{(i)},\ y^{(i)}\right);\ i=1,\cdots,m\right\}$，称作**训练集（training set）**；

    需要注意的是，标记中的上标$^{(i)}$仅用于在训练集中标识训练样本的序号，并不表示指数。

* $X$表示**输入值空间**，$Y$表示**输出值空间**，在本例中$X=Y=\mathbb{R}$（即都是一维实向量空间）。

更加正式的描述监督学习问题：我们的目标是，通过一个给定的训练集，训练一个函数$h:\ X\to Y$，如果$h(x)$能够通过较为准确的预测而得到结果$y$，则我们称这个$h(x)$是“好的”。因为一些历史原因，函数$h$被称为假设（hypothesis），监督学习用流程图表示如下：

当我们尝试预测的目标变量是连续的（就像我们公寓租金的例子一样），我们就说这是一个回归问题（regression problem）。

当$y$只能取几个分立的值（例如已知房间面积大小，我们需要判断这是一栋房屋还是一套公寓）时，我们称其为分类问题（classification problem）。

# 第一部分：线性回归（Linear Regression）

为了让例子更有趣，我们丰富一下数据集，在已知条件里再加上卧室的数量：

$$\begin{array}{c|c|c}living\ area(feet^2)&\#bedrooms&price(1000\$s)\\\hline2104&3&400\\1600&3&330\\2400&3&369\\1416&2&232\\3000&4&540\\\vdots&\vdots&\vdots\end{array}$$

此时$x$是一个在$\mathbb{R}^2$向量空间中的二维向量，即$x_1^{(i)}$表示第$i$个房间样本的实用面积，$x_2^{(i)}$表示第$i$个房间样本的卧室的数量。（通常在设计学习算法时，我们都是自行决定选取哪些特征。假设我们在其他地方做房屋价格调研，那么我们可能会将“是否每个房间都有壁炉”、“卫生间数量”等其他因素纳入输入特征。）

要实现一个监督学习算法，我们必须确定函数（假设）$h$的形式，作为本课程的第一个例子，我们选择使用一个简单的线性函数来逼近$y$：

$$h_\theta(x)=\theta_0+\theta_1x_1+\theta_2x_2$$

在式子里的$\theta_i$叫做参数（或权重），用于参数化$X\to Y$的线性函数映射空间。在不产生混淆的情况下，我们会省略$h_\theta(x)$中的下标$\theta$，简单记为$h(x)$。在上式中我们省略了第一项中的$x_0=1$（即截距项），补上这一项后有：

$$h(x)=\sum_{i=1}^n\theta_ix_i=\theta^Tx$$

上式最右侧中的$\theta$和$x$都是向量，$n$代表输入向量中元素个数（不包含$x_0$）。

现在，有了训练集，我们应该如何选择（训练）参数$\theta$呢？其中一种合理的方法是使得计算出的$h(x)$与真实结果$y$尽量接近（至少对训练集中的已有数据做到尽量接近）。为了形式化这个“尽量接近”的问题，我们定义一个关于$\theta$的函数，使其能够描述$h(x^{(i)})$与相应$y^{(i)}$间的差距，并称其为**成本函数（cost function）**：

$$J(\theta)=\frac{1}{2}\sum_{i=1}^n\left(h_\theta\left(x^{(i)}\right)-y^{(i)}\right)^2\tag{J}$$

以前见过线性回归的同学应该熟悉上式，这就是最小二乘回归模型中常见的最小二乘成本函数。随着课程的进行，我们会发现最小二乘法仅是它所在的一族算法中的特例。

## 1. 最小均方法（LMS: Least mean squares algorithm）

我们需要选取合适的$\theta$以使得$J(\theta)$取最小值。为了达到这种效果，我们可以使用一种搜索算法，算法起始于某个关于$\theta$的“初始猜测值”，然后不断的修改$\theta$以使$J(\theta)$减小，直到最终$\theta$收敛于使得$J(\theta)$取最小处。特别的，我们讲一下梯度下降（gradient descent）法：算法从某个初始的$\theta$起，不断更新$\theta_j:=\theta_j-\alpha\frac{\partial}{\partial\theta_j}J(\theta)$（$:=$表示赋值，该赋值运算同时作用于所有的$\theta_j,\ j=0,\cdots,n$）。赋值运算中的$\alpha$称为学习速率，通常采用手动设置（learning rate，在本例中，它决定了我们“向下山走”时每一步的大小，过小的话收敛太慢，过大的话可能错过最小值）。这是一种很自然的算法，每一步总是寻找使$J$下降最“陡”的方向（就像找最快下山的路一样）。

为了实现这是算法，我们需要找出偏导数是什么。我们从最简单的情况入手，假设只有一个训练样本$(x,\ y)$，这样我们就可以忽略$(J)$式中的求和步骤：

$$\begin{align}\frac{\partial}{\partial\theta_j}J(\theta)&=\frac{\partial}{\partial\theta_j}\frac{1}{2}\left(h_\theta(x)-y\right)^2\\&=2\cdot\frac{1}{2}\left(h_\theta(x)-y\right)\cdot\frac{\partial}{\partial\theta_j}\left(h_\theta(x)-y\right)\\&=\left(h_\theta(x)-y\right)\cdot\frac{\partial}{\partial\theta_j}\left(\sum_{i=0}^n\theta_ix_i-y\right)\\&=\left(h_\theta(x)-y\right)x_j\end{align}$$

训练集仅有一条数据的情况给了我们更新规则：$\theta_j:=\theta_j+\alpha\left(y^{(i)}-h_\theta(x^{(i)})\right)x_j^{(i)}$。

这条规则称为**最小均方（LMS: Least mean squares）**更新规则，也被称作Widrow-Hoff学习规则。这条规则一些看上去很直观、自然的特征，比如更新的幅度与**误差项**$\left(y^{(i)}-h_\theta(x^{(i)})\right)$的大小呈正比。因此，如果我们拿到一条训练向量，其目标量$y^{(i)}$与我们函数的预测值$h_\theta(x^{(i)}$非常接近，则在这条规则作用下，函数的参数$\theta_j$只需要很小的调整；相反的，如果预测值与目标量误差较大，则会得到一个较大的参数调整。

上面，我们使用仅含有一条训练样例的训练集推导出了LMS规则，我们需要将规则推广以适用于正常的训练集，介绍两种方法：

* 一种方法是：

    ```
    Repeat until convergence {
    ```
    $\qquad\quad\quad\theta_j:=\theta_j+\alpha\underbrace{\displaystyle\sum_{i=1}^m\left(y^{(i)}-h_\theta(x^{(i)})\right)x_j^{(i)}}\qquad\quad(for\ every\ j)$
    ```
    }
    ```

对于所有的$j$，重复迭代$\theta_j:=\theta_j+\alpha\displaystyle\sum_{i=1}^m\left(y^{(i)}-h_\theta(x^{(i)})\right)x_j^{(i)}$直到$\theta_j$收敛。

从这个式子里容易看出，大括号中的求和过程就是$\frac{\partial J(\theta)}{\partial\theta_j}$（在有$m$个训练样本时，观察$(J)$式）。所以，这个方法其实就是对$J(\theta)$原始的成本函数做了简单的梯度下降。该方法在每一步搜索“最陡方向”时都会遍历整个训练集，所以也被称作**批量梯度下降（batch
gradient descent）**。值得注意的是，梯度下降可以容易的达到局部最小值，而且我们此处的优化问题也只有一个全局最优解（没有局部最小值）。因此，在本例中，梯度下降总是收敛于全局最小值（当然学习速率$\alpha$不能过大）。况且成本函数$J$确实是一个凸二次函数（仅有一个全局最小值）。

图中的椭圆为某二次函数的等高线，同时也显示了梯度下降法从初始值$(48,\ 30)$到最小值中间的轨迹。图中的“×”标记了梯度下降过程中经过的一系列$\theta$值。

关于收敛，我们可以查看两侧迭代是否相差很多，如果相差无几则可以判断收敛；更常用的方法是检查$J(\theta)$，如果这个值不再发生较大变化时，也可以判断收敛。关于找如何“找”最陡下山路径，其实求偏导本身就已经给出了最陡路径。

我们运行批量梯度下降算法用$\theta$拟合“公寓租金”训练集，以求得根据面积预测价格的函数，最终得到$\theta_0=71.27,\ \theta_1=0.1345$。当我们在最开始的“面积-价格”图中画出关于面积$x$的函数$h_\theta(x)$时，有：

如果将卧室数量也纳入输入特征，则会得到$\theta_0=89.60,\ \theta_1=0.1392,\ \theta_2=-8.738$。这个结果也是通过批量梯度下降求得。

* 我们接下来介绍第二种方法：

    ```
    Loop {
        for i=1 to m, {
    ```
    $\qquad\qquad\quad\quad\theta_j:=\theta_j+\alpha\left(y^{(i)}-h_\theta(x^{(i)})\right)x_j^{(i)}\qquad\qquad\quad(for\ every\ j)$
    ```
        }
    }
    ```

在这个方法中，我们每次仅使用一个训练样本，根据由这个样本得到的误差梯度来更新参数$\theta$。整个算法运行完毕时，对每一套$\theta$参数，每个样本只使用了一次。这就是**随机梯度下降（stochastic gradient descent）**，也称作**增量梯度下降（incremental
gradient descent）**。相对于批量梯度下降每走一步都需要遍历整个训练集（如果训练集样本很多，即$m$很大时，就是非常繁重的计算），随机梯度下降每一步就轻松很多，之后就是不断地根据遇到的每一个样本调整参数。通常情况下，随机梯度下降能够比批量梯度下降更快的使$\theta$接近最优解。（然而，值得注意的是，随机梯度下降可能永远都不会收敛于最小值点，参数$\theta$将在$J(\theta)$最小值附近持续摆动。不过，在实践中，最小值附近的解通常都足够接近最小值。另外，在随机梯度下降的实际操作中，随着迭代步骤的进行，我们会慢慢减小$\alpha$的值至$0$，这样也可以保证参数收敛于全局最小值，而不是在其附近持续摆动）。也是因为效率原因，在遇到训练集中包含大量样本的情况下，我们通常会选用随机梯度下降法。

## 2. 正规方程组（The normal equations）

梯度下降给了我们一种最小化$J$的途径，下面我们看另一种方法。这次我们直接通过计算实现最小化，不再依靠类似梯度下降中使用的分步迭代算法。此方法会直接对$J$求$\theta$的偏导，并使偏导为零，以求$J$的极值点。正规方程组会涉及较多的线性代数，为了避免大篇幅的矩阵求导等运算，我们引入一些新的记号。

### 2.1 矩阵求导

定义函数$f:\ \mathbb{R}^{m\times n}\to\mathbb{R}$，这是一个从$m\times n$阶矩阵到实数的映射，我们定义对“$f$关于$A$的函数”求关于$A$的导数：

$$\nabla_Af(A)=\begin{bmatrix}\frac{\partial f}{\partial A_{11}}&\cdots&\frac{\partial f}{\partial A_{1n}}\\\vdots&\ddots&\vdots\\\frac{\partial f}{\partial A_{m1}}&\cdots&\frac{\partial f}{\partial A_{mn}}\end{bmatrix}$$

可以发现，$\nabla_Af(A)$的梯度是一个$m\times n$阶矩阵，它的第$(i,\ j)$个元素为$\frac{\partial f}{\partial A_{ij}}$。举个例子，矩阵$A=\begin{bmatrix}A_{11}&A_{12}\\A_{21}&A_{22}\end{bmatrix}$是一个二阶方阵，定义函数$f:\ \mathbb{R}^{2\times 2}\to \mathbb{R}$，有$f(A)=\frac{3}{2}A_{11}+5A_{12}^2+A_{21}A_{22}$，于是有$\nabla_Af(A)=\begin{bmatrix}\frac{3}{2}&10A_{12}\\A_{22}&A_{21}\end{bmatrix}$。

在引入$\mathrm{tr}$来表示矩阵的**迹（trace）**。对于一个$n$阶方阵$A$，定义$A$的迹为其对角线元素之和：

$$\mathrm{tr}A=\sum_{i=1}^nA_{ii}$$

实数的迹就是它本身，$a\in\mathbb{R},\ \mathrm{tr}\ a=a$。（如果以前没遇到过这种“操作符记号”，可以把$A$的迹看做作用在$A$上的"迹"函数$\mathrm{tr}(A)$，而对于迹，我们通常不写括号）。

矩阵的迹符合“按次序”的交换律：

$$\mathrm{tr}AB=\mathrm{tr}BA,\quad \mathrm{tr}ABC=\mathrm{tr}CAB=\mathrm{tr}BCA,\quad \mathrm{tr}ABCD=\mathrm{tr}DABC=\mathrm{tr}CDAB=\mathrm{tr}BCDA$$

下面的推论中$A,\ B$皆为方阵：

$$\begin{align}\mathrm{tr}A&=\mathrm{tr}A^T\\\mathrm{tr}(A+B)&=\mathrm{tr}A+\mathrm{tr}B\\\mathrm{tr}\ aA&=a\mathrm{tr}A\end{align}$$

我们就不证明上面这些推论了（有些性质我们会在后面的章节使用）。接下来是关于迹和导数的推论：

$$\begin{align}\nabla_A\mathrm{tr}AB&=B^T\tag{1}\\\nabla_{A^T}f(A)&=\left(\nabla_Af(A)\right)^T\tag{2}\\\nabla_A\mathrm{tr}ABA^TC&=CAB+C^TAB^T\tag{3}\\\nabla_A\lvert A\rvert&=\lvert A\rvert\left(A^{-1}\right)^T\tag{4}\end{align}$$

结合推论$(2),\ (3)$有：

$$\nabla_{A^T}\ \mathrm{tr}ABA^TC=B^TA^TC^T+BA^TC\tag{5}$$

推论$(4)$只用于非奇异方阵，$\lvert A\rvert$表示矩阵的行列式。

为了使这些推论更形象，我们详细解释一下等式$(1)$为何成立。设有矩阵$B\in\mathbb{R}^{n\times m}$，我们可以以$f(A)=\mathrm{tr}AB$为原型定义函数$f:\ \mathbb{R}^{m\times n}\to\mathbb{R}$。因为$A\in\mathbb{R}^{m\times n}$，所以$AB\in\mathbb{R}^{m\times m}$为方阵，所以我们对其求迹是可行的。因此函数$f$的确做了$\mathbb{R}^{m\times n}$到$\mathbb{R}$的映射。然后我们再对其应用一开始定义的矩阵求导，计算$\nabla_Af(A)$，根据定义可知，结果仍是一个$m\times n$的矩阵。根据矩阵求导定义，$\mathrm{tr}AB$对$A$求导后，结果矩阵中第$(i,\ j)$个元素应为$B_{ji}$，也就是$B^T$的第$(i,\ j)$个元素。

推论$(1)-(3)$都较为简单，而推论$(4)$可以从逆的伴随矩阵表示法得出：定义$A'$是方阵$A$的代数余子式（其第$(i,\ j)$元素为$(-1)^{i+j}$乘以“从$A$中删掉第$i$行和第$j$列所得到的方阵”的行列式）于是可以证明$A^{-1}=\frac{\left(A'\right)^T}{\lvert A\rvert}$（可以找一个二阶方阵来验证，如果想要看完整证明可以参考中高阶线代课本，如Charles Curtis, 1991, Linear Algebra, Springer。也可以参考[MIT线性代数第二十讲笔记](http://nbviewer.jupyter.org/github/zlotus/notes-linear-algebra/blob/master/chapter20.ipynb)）这个结论表明$A'=\lvert A\rvert\left(A^{-1}\right)^T$，同时，行列式可以表示为$\lvert A\rvert=\sum_jA_{ij}A'_{ij}$（按第$j$列做代数余子式列展开），注意到$\left(A'\right)_{ij}$并不依赖元素$A_{ij}$（从代数余子式的定义得到），这说明$\frac{\partial\lvert A\rvert}{A_{ij}}=A'_{ij}$，综上得$(4)$。

### 2.2 再看最小二乘

有了矩阵求导，我们再来看如何形式化“使$J(\theta)$能够取到最小值”的$\theta$。我们将$J$重新按照矩阵及向量的标记法表示。

对于已知训练集，定义**设计矩阵（design matrix）**$X$，这是一个$m\times n$矩阵（如果算上截距项，则应是$m\times (n+1)$矩阵，第一项为$\theta_0x_0,\ x_0=1$），矩阵以各训练样本作为行向量：$X=\begin{bmatrix}—\left(x^{(1)}\right)^T—\\—\left(x^{(2)}\right)^T—\\\vdots\\—\left(x^{(m)}\right)^T—\end{bmatrix}$。

同样的，将训练集中的所有目标值放入$m$维列向量$\vec y=\begin{bmatrix}y^{(1)}\\y^{(2)}\\\vdots\\y^{(m)}\end{bmatrix}$。

再将$h_\theta\left(x^{(i)}\right)=\left(x^{(i)}\right)^T\theta$表示为向量形式有$X\theta-\vec y=\begin{bmatrix}\left(x^{(1)}\right)^T\theta\\\left(x^{(2)}\right)^T\theta\\\vdots\\\left(x^{(m)}\right)^T\theta\end{bmatrix}-\begin{bmatrix}y^{(1)}\\y^{(2)}\\\vdots\\y^{(m)}\end{bmatrix}=\begin{bmatrix}h_\theta\left(x^{(1)}\right)-y^{(1)}\\h_\theta\left(x^{(2)}\right)-y^{(2)}\\\vdots\\h_\theta\left(x^{(m)}\right)-y^{(m)}\end{bmatrix}$。

在线性代数中我们知道，向量与自身做内积得到向量长度的平方，即$z^Tz=\sum_iz_i^2$，则有：

$$\begin{align}\frac{1}{2}\left(X\theta-\vec y\right)^T\left(X\theta-\vec y\right)&=\frac{1}{2}\sum_{i=1}^m\left(h_\theta\left(x^{(i)}\right)-y^{(i)}\right)^2\\&=J(\theta)\end{align}$$

最后，为了最小化$J$，我们来求$J$关于$\theta$的导数，于是有：

$$\begin{align}\nabla_\theta J(\theta)&=\nabla_\theta\frac{1}{2}\left(X\theta-\vec y\right)^T\left(X\theta-\vec y\right)\\&=\frac{1}{2}\nabla_\theta\left(\theta^TX^TX\theta-\theta^TX^T\vec y-\vec{y}^TX\theta+\vec{y}^T\vec y\right)\\&=\frac{1}{2}\nabla_\theta\mathrm{tr}\left(\theta^TX^TX\theta-\theta^TX^T\vec y-\vec{y}^TX\theta+\vec{y}^T\vec y\right)\\&=\frac{1}{2}\nabla_\theta\left(\mathrm{tr}\theta^TX^TX\theta-2\mathrm{tr}\ \vec{y}^TX\theta\right)\\&=\frac{1}{2}\left(X^TX\theta+X^TX\theta-2X^T\vec y\right)\\&=X^TX\theta-X^T\vec y\end{align}$$

* 第二步括号内的运算其实是一个数（$\left(X\theta-\vec y\right)$长度的平方），所以可以在第三步中使用性质“实数的迹就是它本身”；
* 第三步到第四部使用了推论$\mathrm{tr}A=\mathrm{tr}A^T$合并项；
* 第四步到第五步使用推论$(5):\ \nabla_{A^T}\ \mathrm{tr}ABA^TC=B^TA^TC^T+BA^TC$，其中$A^T=\theta,\ B=B^T=X^TX,\ C=I$；以及推论$(1):\ \nabla_A\mathrm{tr}AB=B^T$

最终得到**正规方程组（normal equations）**：

$$\bbox[30px,border:2px solid red]{X^TX\theta=X^T\vec y}$$

因此，我们得到形式化的使得“使$J(\theta)$能够取到最小值”的 $\theta=\left(X^TX\right)^{-1}X^T\vec y$。

另外，在[MIT线性代数第二十讲笔记](http://nbviewer.jupyter.org/github/zlotus/notes-linear-algebra/blob/master/chapter16.ipynb)有关于正规方程组几何意义（向量空间投影）的推导。


# Notebook 7: Logistic Regression and SoftMax for MNIST

## Learning Goal

The goal of this notebook is to familiarize the reader with SoftMax regression (a generalization of logistic regression to more than two categories), categorical predictions, and the MNIST handwritten dataset. The reader will understand how to use the Scikit Logistic regression package and visualize learned weights.

## Overview
### The MNIST dataset:
The MNIST classification problem is one of the classical ML problems for learning classification on high-dimensional data with a fairly sizable number of examples (60000). Yann LeCun and collaborators collected and processed $70000$ handwritten digits (60000 are used for training and 10000 for testing) to produce what became known as one of the most widely used datasets in ML: the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset. Each handwritten digit comes in a grayscale square image in the shape of a $28\times 28$ pixel grid. Every pixel takes a value in the range $[0,255]$, representing $256$ nuances of the gray color. The problem of image classification finds applications in a wide range of fields and is important for numerous industry applications of ML.
### SoftMax regression:
We will use SoftMax regression, which can be thought of as a statistical model which assigns a probability that a given input image corresponds to any of the 10 handwritten digits. The model is a generalization of the logistic regression and reads:
\begin{align}
p(y=i|\boldsymbol{x};W) = \frac{e^{\boldsymbol{w}_i^T \boldsymbol{x}}}{\sum_{j=0}^9 e^{\boldsymbol{w}_j^T}},
\end{align}
Where $p(y=i|\boldsymbol{x};W)$ is the probability that input $\boldsymbol{x}$ is the $i$-th digit, $i\in[0,9]$.
The model also has 10 weight vectors $\boldsymbol{w}_i$ which we will train below. Finally, one can use this information for prediction by taking the value of $y$ for which this probability is maximized:
\begin{align}
y_{pred}=\arg\max_i p(y=i|\boldsymbol{x})
\end{align}

## Numerical Experiments

The reader is invited to check out the code below to build up their intuition about SoftMax regression. The following notebook is a slight modification of [this Scikit tutorial](http://scikit-learn.org/dev/auto_examples/linear_model/plot_sparse_logistic_regression_mnist.html) by Arthur Mensch on studying the MNIST problem using Logistic Regression.

```python
import time
import numpy as np
import random

from sklearn.datasets import fetch_openml # MNIST data
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.utils import check_random_state

print(__doc__)
```

    Automatically created module for IPython interactive environment

```python
# Turn down for faster convergence
t0 = time.time()
train_size = 50000
test_size = 10000

### load MNIST data from https://www.openml.org/d/554
X, y = fetch_openml('mnist_784', version=1, return_X_y=True)

# shuffle data
random_state = check_random_state(0)
permutation = random_state.permutation(X.shape[0])
X = X[permutation]
y = y[permutation]
X = X.reshape((X.shape[0], -1))

# pick training and test data sets
X_train, X_test, y_train, y_test = train_test_split(X,y,train_size=train_size,test_size=test_size)

X_test_unscaled=  X_test.copy()

# scale data to have zero mean and unit variance [required by regressor]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# apply logistic regressor with 'sag' solver, C is the inverse regularization strength
clf = LogisticRegression(C=1e5,
                         multi_class='multinomial',
                         penalty='l2', solver='sag', tol=0.1)
# fit data
clf.fit(X_train, y_train)
# percentage of nonzero weights
sparsity = np.mean(clf.coef_ == 0) * 100
# compute accuracy
score = clf.score(X_test, y_test)

#display run time
run_time = time.time() - t0
print('Example run in %.3f s' % run_time)

print("Sparsity with L2 penalty: %.2f%%" % sparsity)
print("Test score with L2 penalty: %.4f" % score)
```

    Example run in 38.831 s
    Sparsity with L2 penalty: 9.18%
    Test score with L2 penalty: 0.8958

```python
%matplotlib inline
import matplotlib.pyplot as plt

# plot weights vs the pixel position
coef = clf.coef_.copy()
plt.figure(figsize=(10, 5))
scale = np.abs(coef).max()
for i in range(10):
    l2_plot = plt.subplot(2, 5, i + 1)
    l2_plot.imshow(coef[i].reshape(28, 28), interpolation='nearest',
                   cmap=plt.cm.Greys, vmin=-scale, vmax=scale)
    l2_plot.set_xticks(())
    l2_plot.set_yticks(())
    l2_plot.set_xlabel('Class %i' % i)
plt.suptitle('classification weights vector $w_j$ for digit class $j$')

plt.show()
```

### Exercises: ###

 The snippet above invoked the "sag" solver which only deals with $L2$ regularization. Try another solver, e.g. 'liblinear', in scikit that supports $L1$ regularization. You can find more details on  http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html.

 Visualize the misclassified samples. Analyze on how logistic regression performs on MNIST, and note this since we will study the MNIST problem later on using deep neural nets.

 Try to vary the size of the training set and see how that affects the accuracy.

  [Theory] In this notebook, we considered a $K=10$ multi-class logistic regression (MLR) also known as the SoftMax regression. Recall that the our model reads:
$$
p(y=i|\boldsymbol{x};W) = \frac{e^{\boldsymbol{w}_i^T \boldsymbol{x}}}{\sum_{j=0}^9 e^{\boldsymbol{w}_j^T \boldsymbol{x}}}
$$
Use this to construct the likelihood function $P(y|\boldsymbol{x})$.

 [Theory] Now consider we're given a set of i.i.d. data $\mathcal{D}=\{(\boldsymbol{x}_1,y_1),\cdots, (\boldsymbol{x}_n,y_n)\}$. As we argued in the main text, the cross-entropy error used in logistic regression is motivated by maximizing the likelihood (or equivalently, minimizing the negative log-likelihood), find the corresponding error function $E(\boldsymbol{w}_0,\cdots,\boldsymbol{w}_9)$ in MLR using the hypothesis above.

 [Theory] Using the error function derived above, calculate its gradient $\frac{\partial E}{\partial \boldsymbol{w}_k}$. Can you find an elegant way of writing the gradient?

```python
yhat=clf.predict(X_test)
fails = X_test_unscaled[yhat!=y_test]
fails_ys = y_test[yhat!=y_test]
fails_yhats = yhat[yhat!=y_test]
```

```python
numimages = 10

Is = [random.randint(0,fails.shape[0]) for j in range(numimages)]
for i in Is:
    plt.figure()
    plt.imshow(fails[i].reshape(28, 28), interpolation='nearest',
                   cmap='gray_r',vmin=0, vmax=255)
    plt.show()
    print("Predicted: "+fails_yhats[i]+", Actual: "+fails_ys[i])
#     vmin=np.abs(fails[i]).min(), vmax=np.abs(fails[i]).max()
```

```python
fails.shape
```

    (1042, 784)

```python
# # Turn down for faster convergence
# t0 = time.time()
# train_size = 50000
# test_size = 10000

# ### load MNIST data from https://www.openml.org/d/554
# X, y = fetch_openml('mnist_784', version=1, return_X_y=True)

# # shuffle data
# random_state = check_random_state(0)
# permutation = random_state.permutation(X.shape[0])
# X = X[permutation]
# y = y[permutation]
# X = X.reshape((X.shape[0], -1))

# # pick training and test data sets
# X_train, X_test, y_train, y_test = train_test_split(X,y,train_size=train_size,test_size=test_size)

# X_test_unscaled=  X_test.copy()

# # scale data to have zero mean and unit variance [required by regressor]
# scaler = StandardScaler()
# X_train = scaler.fit_transform(X_train)
# X_test = scaler.transform(X_test)

# # apply logistic regressor with 'sag' solver, C is the inverse regularization strength
# clf = LogisticRegression(C=1e5,
# #                          multi_class='multinomial',
#                          penalty='l1', solver='liblinear', tol=0.1)
# # fit data
# clf.fit(X_train, y_train)
# # percentage of nonzero weights
# sparsity = np.mean(clf.coef_ == 0) * 100
# # compute accuracy
# score = clf.score(X_test, y_test)

# #display run time
# run_time = time.time() - t0
# print('Example run in %.3f s' % run_time)

# print("Sparsity with L2 penalty: %.2f%%" % sparsity)
# print("Test score with L2 penalty: %.4f" % score)
```

    Automatically created module for IPython interactive environment

    /Users/will/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.
      "this warning.", FutureWarning)

```python

```


```python
from pycalphad import equilibrium, Database, Model, variables as v
import sympy
import numpy as np

TDB = """
 ELEMENT A    GRAPHITE                   12.011     1054.0      5.7423 !
 ELEMENT B   BCC_A2                     55.847     4489.0     27.2797 !
 ELEMENT C   BCC_A2                     55.847     4489.0     27.2797 !
 TYPE_DEFINITION % SEQ * !
 PHASE TEST % 1 1 !
 CONSTITUENT TEST : A,B,C: !
"""
my_phases = ['TEST']
comps = ['A', 'B','C']
comps = sorted(comps)
conds = dict({v.T: 1000, v.P: 101325, v.N: 1})

dbf = Database(TDB)

mod = Model(dbf, ['A', 'B', 'C'], 'TEST')

NP = sympy.Symbol('NP', real=True)
total_moles = sum([NP*mod.moles(c) for c in comps])
total_moles = NP

variables = [v.N, v.P, v.T] + mod.site_fractions + [NP]

mass_cons = [v.N, v.P, v.T]
mass_cons.extend(mod.get_internal_constraints())
mass_cons.extend(NP*mod.moles(c) for c in comps)

mass_jac = []
for cons in mass_cons:
    mass_jac.append([cons.diff(x) for x in variables])

energy_grad = [(total_moles*mod.GM).diff(x) for x in variables]
```

```python
mass_cons
```

    [N,
     P,
     T,
     TEST0A + TEST0B + TEST0C - 1,
     1.0*TEST0A*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C),
     1.0*TEST0B*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C),
     1.0*TEST0C*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)]

```python
mass_jac
```

    [[1, 0, 0, 0, 0, 0, 0],
     [0, 1, 0, 0, 0, 0, 0],
     [0, 0, 1, 0, 0, 0, 0],
     [0, 0, 0, 1, 1, 1, 0],
     [0,
      0,
      0,
      -1.0*TEST0A*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2 + 1.0*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C),
      -1.0*TEST0A*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      -1.0*TEST0A*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      1.0*TEST0A/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)],
     [0,
      0,
      0,
      -1.0*TEST0B*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      -1.0*TEST0B*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2 + 1.0*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C),
      -1.0*TEST0B*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      1.0*TEST0B/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)],
     [0,
      0,
      0,
      -1.0*TEST0C*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      -1.0*TEST0C*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2,
      -1.0*TEST0C*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)**2 + 1.0*NP/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C),
      1.0*TEST0C/(1.0*TEST0A + 1.0*TEST0B + 1.0*TEST0C)]]

```python
A = sympy.Matrix(mass_jac).T.pinv()
x = A * sympy.Matrix(energy_grad)
```

```python
from pycalphad.codegen.sympydiff_utils import build_functions

mu_a = build_functions(x[4], variables, include_grad=True, include_hess=False)
mu_b = build_functions(x[5], variables, include_grad=True, include_hess=False)
mu_c = build_functions(x[6], variables, include_grad=True, include_hess=False)

energy = build_functions(mod.GM, variables, include_grad=True)
```

```python
print(x[4].free_symbols)
```

    {TEST0A, TEST0B, NP, T, TEST0C}

```python
mu_a.func([1, 1e5, 1000, 0.4, 0.6, 1e-12, 1e-6])
```

    array(-7618.14919886)

```python
np.array(mu_a.grad([1, 1e5, 1000, 0.4, 0.6, 1e-12, 1])) - np.array(mu_b.grad([1, 1e5, 1000, 0.4, 0.6, 1e-12, 1]))
```

    array([ 0.00000000e+00,  0.00000000e+00, -3.37123964e+00,  2.07862500e+04,
           -1.38575000e+04,  9.23094935e-01, -9.44368765e-10])

```python
from pycalphad.core.solver import InteriorPointSolver

class ProblemSaver(InteriorPointSolver):
    saved_problem = [None]
    def solve(self, prob):
        self.saved_problem[0] = prob
        self.verbose = True
        return super(ProblemSaver, self).solve(prob)

eq = equilibrium(dbf, ['A', 'B', 'C'], ['TEST'],
                     {v.MU('B'): -1000, v.X('A'): 0.1, v.T: 800, v.P: 101325}, solver=ProblemSaver())
```

    Chemical Potentials [-15315.87500456  -1000.         -21480.14360388]
    [0. 0. 0. 0. 0. 0. 0.]
    [1.00000000e+00 1.01325000e+05 8.00000000e+02 1.00000000e-01
     8.60415585e-01 3.95844148e-02 1.00000000e+00]
    Status: 0 b'Algorithm terminated successfully at a locally optimal point, satisfying the convergence tolerances (can be specified by options).'

```python
ProblemSaver.saved_problem[0].jacobian([1, 1e5, 800, 0.1, 8.60415585e-01, 3.95844148e-2, 1.0])[-1]
```

    array([ 0.00000000e+00,  0.00000000e+00, -1.25000000e+00,  2.59387889e-09,
            7.73068284e+03,  1.93540473e-09, -8.77662387e-10])

```python
mu_b.grad([1, 1e5, 800, 0.1, 8.60415585e-01, 3.95844148e-2, 1.0])
```

    [array(0.),
     array(0.),
     array(-1.25),
     array(-3.52429197e-12),
     array(7730.68284206),
     array(2.79669621e-11),
     array(2.33626452e-11)]

```python

```