# import module

```python
import numpy as np
import numba
import xarray as xr
```

# Continuous Equation

$$
    \begin{align}
        \frac{\partial h}{\partial t}+\frac{\partial q_x}{\partial x} +\frac{\partial q_y}{\partial y} = 0
    \end{align}
$$

```python
@numba.jit(nopython=True, parallel=False)
def conEq(dep, qx, qy, zb, dt, dx, dy, hmin, hbuf):
    imax, jmax = len(dep), len(dep[0])
    depn = np.empty_like(dep)
    fluxx = np.zeros((imax+1, jmax))
    fluxy = np.zeros((imax, jmax+1))
    gravity = 9.8

    f = lambda Qp, Qm : Qm if Qp >= 0.0 and Qm >= 0.0 else (Qp if Qp <= 0.0 and Qm <= 0.0 else 0.5*Qp+0.5*Qm )

    def flux(Qp, Qm, depp, depm, zbp, zbm) :
        r = f(Qp, Qm)

        if ( (depm + zbm) < zbp ) and (depp <= hbuf) : r = 0.0
        if ( (depp + zbp) < zbm ) and (depm <= hbuf) : r = 0.0

        return r

    for i in numba.prange( 1, imax ):
        for j in numba.prange( jmax ):
            c, xm = (i,j), (i-1,j)
            fluxx[c] = flux(qx[c], qx[xm], dep[c], dep[xm], zb[c], zb[xm])

# boundary : not use
    fluxx[-1,:] = -9999
    fluxx[0,:] = -9999

    for i in numba.prange( imax ):
        for j in numba.prange( 1, jmax ):
            c, ym = (i,j), (i,j-1)
            fluxy[c] = flux(qy[c], qy[ym], dep[c], dep[ym], zb[c], zb[ym])

# wall boundary
    fluxy[:,0] = 0.0 #fluxy[:,0]
    fluxy[:,-1] = 0.0 #fluxy[:,0]

    n = 0
    for i in numba.prange(1, imax):
        for j in numba.prange(jmax):
            c, xp, yp = (i, j), (i+1, j), (i, j+1)
            depn[c] = dep[c] - dt*(fluxx[xp] - fluxx[c])/dx - dt*(fluxy[yp] - fluxy[c])/dy
            if depn[c] < hmin :
                n += 1
                depn[c] = hmin

# upstream boundary
    depn[0][:] = depn[1][:]

# downstream boundary
    depn[-1][:] = depn[-2][:]

    return depn, n
```

# Momentum Equation

$$
    \begin{align}
        \frac{\partial q_x}{\partial t}+\frac{\partial u q_x}{\partial x}+\frac{\partial v q_x}{\partial y}+gh\frac{\partial H}{\partial x}+\frac{\tau_{0x}}{\rho}
        - \nu_t h \left(\frac{\partial^2 u}{\partial x^2}+\frac{\partial^2 u}{\partial y^2} \right)= 0 \\
        \frac{\partial q_y}{\partial t}+\frac{\partial u q_y}{\partial x}+\frac{\partial v q_y}{\partial y}+gh\frac{\partial H}{\partial y}+\frac{\tau_{0y}}{\rho}- \nu_t h \left(\frac{\partial^2 v}{\partial x^2}+\frac{\partial^2 v}{\partial y^2} \right)
        = 0
    \end{align}
$$

```python
@numba.jit(nopython=True, parallel=False)
def momentEq(dep, qx, qy, depn, zb, Qup, dt, dx, dy, hmin, hbuf, direction):
    #direction = 1:x, 2:y
    gravity = 9.8
    cManing = 0.03

    q = qx if direction == 1 else qy
    u, v = qx/dep, qy/dep
    Vdir = q/dep

    imax, jmax = len(q), len(q[0])

    qn = np.empty_like(q)
    fluxx = np.zeros((imax+1, jmax))
    fluxy = np.zeros((imax, jmax+1))

#     f = lambda vp,vm,qp,qm : vm*qm if vp > 0.0 and vm > 0.0 else \
#         (vp*qp if vp < 0.0 and vm < 0.0 else (0.5*vp+0.5*vm)*(0.5*qp+0.5*qm) )

    f = lambda vp,vm,qp,qm : vm*qm if vp >= 0.0 and vm >= 0.0 else \
        (vp*qp if vp <= 0.0 and vm <= 0.0 else (0.5*vp+0.5*vm)*(0.5*qp+0.5*qm) )

    def flux1(vp, vm, qp, qm, depp, depm, zbp, zbm) :
        if ( (depm + zbm) < zbp ) and (depp <= hbuf) : r = 0.0
        if ( (depp + zbp) < zbm ) and (depm <= hbuf) : r = 0.0

        r = f(vp,vm,qp,qm)
        return r

    for i in numba.prange( 1, imax ):
        for j in numba.prange( jmax ):
            c, xm = (i,j), (i-1,j)
            fluxx[c] = flux1( u[c], u[xm], q[c], q[xm], dep[c], dep[xm], zb[c], zb[xm] )

# boundary : not use
    fluxx[-1,:] = -9999
    fluxx[0,:]  = -9999

    for i in numba.prange( imax ):
        for j in numba.prange( 1, jmax ):
            c, ym = (i,j), (i,j-1)
            fluxy[c] = flux1( v[c], v[ym], q[c], q[ym], dep[c], dep[ym], zb[c], zb[ym] )

# wall boundary
    fluxy[:,0]  = -fluxy[:,1]
    fluxy[:,-1] = -fluxy[:,-2]

    for i in numba.prange(1, imax-1):
        for j in numba.prange(jmax):
            c = (i, j)
            if depn[c] <= hbuf :
                qn[c] = 0.0
            else:
            # pressure & gravity term
                if direction == 2 and ((j == 0) or (j == jmax-1)) :
                    if j == 0 :
                        c, yp = (i, j), (i, j+1)
                        Hc, Hp = depn[c]+zb[c], depn[yp]+zb[yp]

                        if Hc < zb[yp] and depn[yp] <= hbuf :
                            dHdx = 0.0
                        else :
                            dHdx = ( Hp - Hc )/dy
                    elif j == jmax-1 :
                        c, ym = (i, j), (i, j-1)
                        Hc, Hm = depn[c]+zb[c], depn[ym]+zb[ym]

                        if Hc < zb[ym] and depn[ym] <= hbuf :
                            dHdx = 0.0
                        else :
                            dHdx = ( Hc - Hm )/dy

                elif direction == 1 and i == imax-1 :
                    pass
                else :
                    c  = (i, j)
                    if direction == 1 :
                        xp = (i+1, j)
                        xm = (i-1, j)
                        dp, dm, delta, dib = xp, xm, dx, 0.0
                    else :
                        yp = (i, j+1)
                        ym = (i, j-1)
                        dp, dm, delta, dib = yp, ym, dy, 0.0

                    Vc, Vp, Vm = q[c]/dep[c], q[dp]/dep[dp], q[dm]/dep[dm]
                    Hc, Hp, Hm = depn[c]+zb[c], depn[dp]+zb[dp], depn[dm]+zb[dm]

                    if Hc < zb[dp] and depn[dp] <= hbuf :
                        if Hc < zb[dm] and depn[dm] <= hbuf :
                            dHdx = 0.0
                        else:
                            dHdx = (Hc-Hm)/delta
                    elif Hc < zb[dm] and depn[dm] <= hbuf :
                        dHdx = (Hp-Hc)/delta
                    else :
                        if Vc > 0.0 and Vp > 0.0 and Vm > 0.0:
                            Cr1, Cr2 = 0.5*(abs(Vc)+abs(Vp))*dt/delta, 0.5*(abs(Vc)+abs(Vm))*dt/delta
                            dHdx1, dHdx2 = (Hp-Hc)/delta-dib, (Hc-Hm)/delta-dib
                        elif Vc < 0.0 and Vp < 0.0 and Vm < 0.0:
                            Cr1, Cr2 = 0.5*(abs(Vc)+abs(Vm))*dt/delta, 0.5*(abs(Vc)+abs(Vp))*dt/delta
                            dHdx1, dHdx2 = (Hc-Hm)/delta-dib, (Hp-Hc)/delta-dib
                        else:
                            Cr1 = Cr2 = 0.5*(abs(0.5*(Vc+Vp))+abs(0.5*(Vc+Vm)))*dt/delta
                            dHdx1 = dHdx2 = (0.5*(Hc+Hp) - 0.5*(Hc+Hm)) / delta - dib

                        w1, w2 = 1-Cr1**0.5, Cr2**0.5
                        dHdx = w1 * dHdx1 + w2 * dHdx2

# viscous sublayer
                Cf = gravity*cManing**2.0/dep[c]**(1.0/3.0)
                Vnorm = np.sqrt(u[c]**2.0+v[c]**2.0)
                Vis = Cf * Vnorm * u[c] if direction == 1 else  Cf * Vnorm * v[c]

# turbulence
#                kenergy = 2.07*Cf*Vnorm**2
                nut = 0.4/6.0*dep[c]*np.sqrt(Cf)*np.abs(Vnorm)

# side boundary : non-slip condition
                if i == imax-1:
                    turb = 0.0
                elif j == 0 :
                    if direction == 1:
                        turb = nut * ( Vdir[xp] - 2.0*Vdir[c] + Vdir[xm] )/ dx**2 \
                             + nut * ( Vdir[yp] - 3.0*Vdir[c] )/ dy**2
                    elif direction == 2:
                        turb = nut * ( Vdir[xp] - 2.0*Vdir[c] + Vdir[xm] )/ dx**2 \
                             + nut * ( Vdir[yp] - 3.0*Vdir[c] )/ dy**2
                elif j == jmax-1 :
                    if direction == 1:
                        turb = nut * ( Vdir[xp] - 2.0*Vdir[c] + Vdir[xm] )/ dx**2 \
                             + nut * (  - 3.0*Vdir[c] + Vdir[ym] )/ dy**2
                    elif direction == 2:
                        turb = nut * ( Vdir[xp] - 2.0*Vdir[c] + Vdir[xm] )/ dx**2 \
                             + nut * (  - 3.0*Vdir[c] + Vdir[ym] )/ dy**2
                else :
                    turb = nut * ( Vdir[xp] - 2.0*Vdir[c] + Vdir[xm] )/ dx**2 \
                         + nut * ( Vdir[yp] - 2.0*Vdir[c] + Vdir[ym] )/ dy**2

                c, xp, yp = (i,j), (i+1,j), (i,j+1)
                sourcet = Vis - dep[c] * turb

                qn[c] = q[c] - dt * ( fluxx[xp] - fluxx[c] ) / dx \
                             - dt * ( fluxy[yp] - fluxy[c] ) / dy \
                             - dt * gravity * depn[c] * dHdx \
                             - dt * sourcet

# upstream boundary
    if direction == 2 :
        qn[0,:] = 0.0
    else :
        updep = depn.copy()
        for j in range(jmax):
            updep[0,j] = 0.0 if updep[0,j] <= hmin else updep[0,j]

        alpha = Qup / dy / np.sum( updep[0,:]**(5/3) )
        qn[0,:] = alpha * updep[0,:]**(5/3)

# downstream boundary
    qn[-1,:] = qn[-2,:]

    return qn
```

```python
@numba.jit(nopython=True, parallel=False)
def simulation(dep, qx, qy, zb, Qup, dt, dx, dy, hmin, hbuf):
    depn, count = conEq(dep, qx, qy, zb, dt, dx, dy, hmin, hbuf)
    qxn = momentEq(dep, qx, qy, depn, zb, Qup, dt, dx, dy, hmin, hbuf, 1)
    qyn = momentEq(dep, qx, qy, depn, zb, Qup, dt, dx, dy, hmin, hbuf, 2)
    CFL = ((np.abs(qxn/depn))/dx + (np.abs(qyn/depn))/dy)*dt
    CFLmin = np.max(CFL)
    return depn, qxn, qyn, CFLmin, count
```

# bed variation

$$
    \begin{align}
       (1-\lambda) \frac{\partial z_b}{\partial t}+\frac{\partial q_{bx}}{\partial x} +\frac{\partial q_{by}}{\partial y} = 0 \\
        q_b : M.P.M.-formula
    \end{align}
$$

```python
@numba.jit(nopython=True, parallel=False)
def bedvarEq(dzb, dep, qx, qy, zb, dt, dx, dy, hmin, hbuf):
    imax, jmax = len(dep), len(dep[0])
    dzbn = np.empty_like(dep)
    qbx = np.empty_like(dep)
    qby = np.empty_like(dep)
    fluxx = np.zeros((imax+1, jmax))
    fluxy = np.zeros((imax, jmax+1))
    gravity = 9.8
    cManning = 0.03
    rhosw = 1.65
    porosity = 0.4
    di = 5.0 / 1000.0
    tsc = 0.05

    f = lambda Qp, Qm : Qm if Qp >= 0.0 and Qm >= 0.0 else (Qp if Qp <= 0.0 and Qm <= 0.0 else 0.5*Qp+0.5*Qm )

    def flux(Qp, Qm, depp, depm, zbp, zbm) :
        r = f(Qp, Qm)

        # Dry bed
        if ( (depm + zbm) < zbp ) and (depp <= hbuf) : r = 0.0
        if ( (depp + zbp) < zbm ) and (depm <= hbuf) : r = 0.0

        return r

    for i in numba.prange(imax):
        for j in numba.prange(jmax):
            c = (i, j)
            u = qx[c]/dep[c]
            v = qy[c]/dep[c]
            Cf = gravity * cManning**2.0 / dep[c]**(1.0/3.0)
            Vnorm = np.sqrt( u**2 + v**2 )
            us = Vnorm*np.sqrt(Cf)
            ts = us**2.0/rhosw/gravity/di

            if ts/tsc > 1.0 :
            # MPM
                qb = np.sqrt(rhosw*gravity*di**3.0) * 8.0*ts**1.5 * (1.0 - tsc/ts)**1.5
                qbx[c] = qb * u/Vnorm
                qby[c] = qb * v/Vnorm
            else :
                qbx[c] = 0.0
                qby[c] = 0.0

    for i in numba.prange( 1, imax ):
        for j in numba.prange( jmax ):
            c, xm = (i,j), (i-1,j)
            fluxx[c] = flux(qbx[c], qbx[xm], dep[c], dep[xm], zb[c]+dzb[c], zb[xm]+dzb[xm])

# boundary
    fluxx[-1,:] = qbx[-1, :]
    fluxx[0,:] = fluxx[1,:] # equilibrium supply

    for i in numba.prange( imax ):
        for j in numba.prange( 1, jmax ):
            c, ym = (i,j), (i,j-1)
            fluxy[c] = flux(qby[c], qby[ym], dep[c], dep[ym], zb[c]+dzb[c], zb[ym]+dzb[ym])

# wall boundary
    fluxy[:,0] = 0.0
    fluxy[:,-1] = 0.0

    for i in numba.prange(imax):
        for j in numba.prange(jmax):
            c, xp, yp = (i, j), (i+1, j), (i, j+1)
            dzbn[c] = dzb[c] - dt/(1-porosity)*(fluxx[xp] - fluxx[c])/dx - dt/(1-porosity)*(fluxy[yp] - fluxy[c])/dy

# upstream boundary
    dzbn[0,:] = dzbn[1,:]

    return dzbn, qbx, qby
```

# Main routine

```python
def d2dmodel(dep, qx, qy, zb, dzb, Qup, tmax, trunup):
    hmin = 10.0**(-5)
    hbuf = 10.0**(-2)
#     dzb = np.zeros_like(dep)
    qbx = np.zeros_like(dep)
    qby = np.zeros_like(dep)
    nxmax = len(zb)
    nymax = len(zb[0])
    dx = 5.0
    dy = 5.0
    dt = 0.02
    dtout= 5*3600.0
    irunup = -int(trunup/dt)

    def timeGenerater():
        it, itout = irunup, 0
        isTiter, isOut = True, False
        isSedimentcal = False
        yield it*dt, isTiter, isOut, isSedimentcal

        while True:
            it += 1
            if it*dt >= itout*dtout:
                itout += 1
                isOut = True
            else:
                isOut = False

            if it*dt >= tmax: isTiter = False
            if it > 0 : isSedimentcal = True

            yield it*dt, isTiter, isOut, isSedimentcal

    tgen = timeGenerater()
    t, isTiter, isOut, isSedimentcal = next(tgen)

    tarr = []
    deparr = np.empty((0,nxmax,nymax))
    uarr   = np.empty((0,nxmax,nymax))
    varr   = np.empty((0,nxmax,nymax))
    dzbarr = np.empty((0,nxmax,nymax))

    nout= 1
    while isTiter:
        t, isTiter, isOut, isSedimentcal = next(tgen)

        dep, qx, qy, CFLmin, count = simulation(dep, qx, qy, zb+dzb, Qup, dt, dx, dy, hmin, hbuf)
        if isSedimentcal : dzb, qbx, qby = bedvarEq(dzb, dep, qx, qy, zb, dt, dx, dy, hmin, hbuf)

        if isOut:
            tarr.append(round(t,2))
            deparr = np.append(deparr, np.array([dep]), axis=0)
            uarr = np.append(uarr, np.array([qx/dep]), axis=0)
            varr = np.append(varr, np.array([qy/dep]), axis=0)
            dzbarr = np.append(dzbarr, np.array([dzb]), axis=0)
            print(t, CFLmin, count)
            nout += 1
            dss = xr.Dataset( { 'depth': (['t','x','y'], deparr), 'u': (['t','x','y'], uarr), 'v': (['t','x','y'], varr)
                               , 'dzb': (['t','x','y'], dzbarr) }
                             , coords={'x': np.arange(nxmax)*dx + 0.5*dx , 'y':np.arange(nymax)*dy + 0.5*dy, 't': tarr} )

    dss.to_netcdf('output.nc')
    dss.close()
    return dep, qx, qy, dzb, qbx, qby
```

# calculation

```python
%%time
dx = 5
hmin = 10.0**(-5)

nxmax, nymax = 200, 20
zb = np.zeros((nxmax, nymax))
qx = np.zeros((nxmax,nymax))
qy = np.zeros_like(qx)

ib = 0.001
Hs = 1.0
for i in range(nxmax):
    zb[i,:] = Hs - ib * i * dx

zb[:60, :5]  += 5.0
zb[:60, 15:] += 5.0

dep = np.full_like(zb, 1.0)
dep[:60, :5]  = hmin
dep[:60, 15:] = hmin

qy = np.full_like(zb, 0.0)
qx = 1/0.03*ib**0.5*dep**(5/3)

qx[:60, :5]  = 0.0
qx[:60, 15:] = 0.0
Qup = np.sum(qx[0])*dx

dep, qx, qy, dzb, qbx, qby = d2dmodel(dep, qx, qy, zb, dzb, Qup, tmax=500*3600, trunup=3600)
```


The dynamical equations are given as

\begin{align}
\dot{S_{ni}}	    &=-\lambda_{ni}(t)S_{ni}+\sigma_{ni},\\
\dot{E}_{ni}	    &=\lambda_{i}(t)S_{ni}-\gamma_{E}E_{ni},\\
\dot{I}_{ni}^{a}	&=\alpha\gamma_{E}E_{ni}-\gamma_{I^{a}}I_{ni}^{a},\\
\dot{I}_{ni}^{s}	&=\bar{\alpha}\gamma_{E}E_{ni}-\gamma_{I^{s}}I_{ni}^{s},\\
\dot{I}_{ni}^{h}	&=h_{i}\gamma_{I^{s}}I_{ni}^{s}-\gamma_{I^{h}}I_{ni}^{h},\\
\dot{I}_{ni}^{c}	&=c_{i}\gamma_{I^{h}}I_{ni}^{h}-\gamma_{I^{c}}I_{ni}^{c},\\
\dot{I}_{ni}^{m}	&=m_{i}\gamma_{I^{c}}I_{ni}^{c},\\
\dot{N}_{ni}	    &=\sigma_{ni}-m_{ni}\gamma_{I^{c}}I_{ni}^{c}
\end{align}

The key idea is to note that the rate of infection of a susceptible individual in age group `i` at the node `n` is given as

\begin{align}
\lambda_{ni}(t)=\lambda^0_{ni}(t)=\beta\sum_{j=1}^{M}\left(C_{ij}\frac{I_{nj}^{a}}{N_{j}}+f_{sa}C_{ij}\frac{I_{nj}^{s}}{N_{j}}\right)
\end{align}

Commute can then be modeled by changing this to

\begin{align}
\lambda_{ni}^{T}(t)=\beta\sum_{j=1}^{M}\sum_{n=1,n\neq m}^{Nd}r_{T}T_{nm}\left(C_{ij}\frac{I_{mj}^{a}}{N_{j}}+f_{sa}C_{ij}\frac{I_{mj}^{s}}{N_{j}}\right).
\end{align}

while workplace interaction can be modeled as

\begin{align}
\lambda_{ni}^{W}(t)=\beta\sum_{j=1}^{M}\sum_{n=1,n\neq m}^{Nd}r_{W}W_{nm}\left(C_{ij}\frac{I_{mj}^{a}}{N_{j}}+f_{sa}C_{ij}\frac{I_{mj}^{s}}{N_{j}}\right).
\end{align}

The simulation then runs by choosing appropriate $\lambda$ as a function of time. The constants $r_T$ and $r_W$ control the strength of interactions and the matrices $W_{nm}$ and $T_{nm}$ specify which `nm` nodes interact.

```python
%%capture
## compile for this notebook
import os
owd = os.getcwd()
os.chdir('../')
%run setup.py install
os.chdir(owd)
```

```python
%matplotlib inline
import numpy as np
import pandas as pd

import nodgeo
import matplotlib.pyplot as plt
#from matplotlib import rc; rc('text', usetex=True)
```

```python
M=16  # number of age groups

# load age structure data
my_data = np.genfromtxt('data/age_structures/UK.csv', delimiter=',', skip_header=1)
aM, aF = my_data[:, 1], my_data[:, 2]

# set age groups
Ni0=aM+aF;   Ni0=Ni0[0:M];
```

```python
my_data = pd.read_excel('data/contact_matrices_152_countries/MUestimates_home_2.xlsx', sheet_name='United Kingdom of Great Britain')
CH0 = np.array(my_data)

my_data = pd.read_excel('data/contact_matrices_152_countries/MUestimates_work_2.xlsx', sheet_name='United Kingdom of Great Britain',index_col=None)
CW0 = np.array(my_data)

my_data = pd.read_excel('data/contact_matrices_152_countries/MUestimates_school_2.xlsx', sheet_name='United Kingdom of Great Britain',index_col=None)
CS0 = np.array(my_data)

my_data = pd.read_excel('data/contact_matrices_152_countries/MUestimates_other_locations_2.xlsx', sheet_name='United Kingdom of Great Britain',index_col=None)
CO0 = np.array(my_data)

CH = np.zeros((16, 16))
CH[0,:]= np.array((0.478812799633172, 0.55185413960287,0.334323605154544,0.132361228266194,0.138531587861408,0.281604887066586,0.406440258772792,0.493947983343078,0.113301080935514,0.0746826413664804,0.0419640342896305,0.0179831987029717,0.00553694264516568,0.00142187285266089,0,0.000505582193632659))
for i in range(15):
    CH[i+1, :] = CH0[i, :]

CW = np.zeros((16, 16))
CW[0,:]= np.array((0,0,0,0,0,0,0,0,0,0,0,0,0,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000820604524144799,0.0000120585150153575,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000316436833811157))
for i in range(15):
    CW[i+1, :] = CW0[i, :]

CS = np.zeros((16, 16))
CS[0,:]= np.array((0.974577996106766,0.151369805263473,0.00874880925953218,0.0262790907947637,0.0111281607429249,0.0891043051294382,0.125477587043249,0.0883182775274553,0.0371824197201174,0.0294092695284747,0.0000000000000000000000000000000000000510911446027435,0.0000000000000000000000000000000113982464440009,0.00758428705895781,0.00151636767747242,0.0000000000000000000000000000000000000000000000000123262013953524,0.000000000000000000000000000000000000000000000000000000000000000597486362181075))
for i in range(15):
    CS[i+1, :] = CS0[i, :]

CO = np.zeros((16, 16))
CO[0,:]= np.array((0.257847576361162,0.100135168376607,0.0458036773638843,0.127084549151753,0.187303683093508,0.257979214509792,0.193228849121415,0.336594916946786,0.309223290169635,0.070538522966953,0.152218422246435,0.113554851510519,0.0615771477785246,0.040429874099682,0.0373564987094767,0.00669781557624776))
for i in range(15):
    CO[i+1, :] = CO0[i, :]

## matrix of total contacts
C=CH+CW+CS+CO

fig,aCF =  plt.subplots(2,2);
aCF[0][0].pcolor(CH, cmap=plt.cm.get_cmap('GnBu', 10));
aCF[0][1].pcolor(CW, cmap=plt.cm.get_cmap('GnBu', 10));
aCF[1][0].pcolor(CS, cmap=plt.cm.get_cmap('GnBu', 10));
aCF[1][1].pcolor(CO, cmap=plt.cm.get_cmap('GnBu', 10));

```

```python
beta  = 0.4/24            # infection rate

gE    = (1.0/5)/24
gIa   = (1.0/7)/24                # recovery rate of asymptomatic infectives
gIs   = (1.0/7)/24                # recovery rate of symptomatic infectives
alpha = 0.3                 # fraction of asymptomatic infectives
fsa   = 1                 # the self-isolation parameter

gIh   = (1.0/14)/24
gIc   = (1.0/1)/24

sa    = 100*np.ones(M)      #  rate of additional/removal of population by birth etc
sa[0]     = 1500            # birth
sa[12:16] = -300            # mortality

hh   = 0.1*np.ones(M)       # fraction which goes from Is to hospital
cc   = 0.05*np.ones(M)      # fraction which goes from  hospital to ICU
mm   = np.zeros(M)       # mortality from IC
mm = np.array((0,0,.0,1,1,1,1,1,1,3.5,3.5,3.5,3.5,6,6,14.2))/100

```

```python

beta  = 1/24            # infection rate
gE    = (1.0/5)/24
gIa   = (1.0/7)/24                # recovery rate of asymptomatic infectives
gIs   = (1.0/7)/24                # recovery rate of symptomatic infectives
alpha = 0.3                 # fraction of asymptomatic infectives
fsa   = 1                 # the self-isolation parameter

gIh   = (1.0/7)/24
gIc   = (1.0/7)/24

sa    = 0*np.ones(M)      #  rate of additional/removal of population by birth etc

hh   = 0.5*np.ones(M)       # fraction which goes from Is to hospital
cc   = np.array((0,0,.0,1,1,1,1,1,1,3.5,3.5,3.5,3.5,6,6,14.2))/100
mm   = 0.5*cc      # fraction which goes from  hospital to ICU

Nd=21

N0= np.zeros((M*Nd))
for i in range(Nd):
    N0[i*M:(i+1)*M]=Ni0/Nd
Ni = N0

E0   = np.zeros((M*Nd));
Ia0  = 0*np.ones((M*Nd));  #   Ia0[6:13]=4;  Ia0[2:6]=2; Ia0[13:16]=2
Is0  = 0*np.ones((M*Nd));  #Is_0[6:13]=8;  Is_0[2:6]=4; Is_0[13:16]=4

for i in range(int(Nd/3)):
    Ia0[3*M*i:3*M*i+10] = 10
Is0 = Ia0

Ih0  = np.zeros((M*Nd))
Ic0  = np.zeros((M*Nd))
Im0  = np.zeros((M*Nd))
R0   = np.zeros((M*Nd))

S0  = Ni - (E0 + Ia0 + Is0 + Ih0 + Ic0 + Im0 + R0)

tL, wL = 6, 6# int(Nd/5), int(Nd/5)
Tr = np.zeros((Nd, tL))
Wo = np.zeros((Nd, wL))

for i in range(Nd):
    Tr[i,:] = np.linspace(i+1, i+tL, tL)%(Nd-1)#np.random.randint(Nd, size=tL)
    Wo[i,:] = np.linspace(i+1, i+wL, wL)%(Nd-1)#np.random.randint(Nd, size=wL)

rW, rT = .1, .1

def contactMatrix(t):
    return C

# duration of simulation and data file
Tf = 24*60;  Nt=120;

# instantiate model
parameters = {'alpha':alpha,'beta':beta, 'gIa':gIa,'gIs':gIs,
              'gIh':gIh,'gIc':gIc,  'gE':gE,
              'fsa':fsa,  'rW':rW, 'rT':rT,
              'hh':hh, 'cc':cc, 'mm':mm}
model = nodgeo.deterministic.SEI5R(parameters, Nd, M, Ni, Wo, Tr)

# simulate model
#data = model.simulate(S0, E0, Ia0, Is0, Ih0, Ic0, Im0, contactMatrix, Tf, Nt, nodeInteraction='False')

```

## Non-interacting nodes

```python
data = model.simulate(S0, E0, Ia0, Is0, Ih0, Ic0, Im0, contactMatrix, Tf, Nt, nodeInteraction='False')

## first M*Nd points are S, then M*Nd points are E, then M*Nd points are Ia,  Is, Ih, Ic, Im, Ni
## thus a total of 8*M*Nd points

radius = np.sqrt(np.arange(Nd)/float(Nd))
golden_angle = np.pi * (3 - np.sqrt(5)); theta = golden_angle * np.arange(Nd)
points = np.zeros((Nd, 2)); points[:,0] = np.cos(theta); points[:,1] = np.sin(theta)
points *= radius.reshape((Nd, 1));  points = points*4*np.sqrt(Nd)

plt.rcParams.update({'font.size': 26})

M1=5*M*Nd
f = plt.figure(figsize=(16, 15));
for ti in range(1):
    ti=Nt-1
    IM1=np.zeros((Nd));  IM2=np.zeros((Nd));  IM3=np.zeros((Nd));  IM4=np.zeros((Nd))
    IS=np.zeros((Nd));

    for n in range(Nd):
        for i in range(3):
            IM1[n] += data['X'][ti, M1+n*M+i]
            IS[n] += data['X'][ti, 3*M*Nd+n*M+i]
        for i in range(6):
            IM2[n] += data['X'][ti, M1+n*M+3+i]
        for i in range(4):
            IM3[n] += data['X'][ti, M1+n*M+9+i]
        for i in range(3):
            IM4[n] += data['X'][ti, M1+n*M+12+i]

    print(IM4[0:4])
    #print(IS[0:4])
    Imm = (np.max((IM1, IM2, IM3, IM4))) + 0.000001
    IM1 = IM1/Imm
    IM2 = IM2/Imm
    IM3 = IM3/Imm
    IM4 = IM4/Imm

    sz=200
    sp =  f.add_subplot(2,2,1);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM1, cmap=plt.cm.Reds)
    plt.clim(0, 1);  plt.title('0-15 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,2);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM2, cmap=plt.cm.Reds)
    plt.clim(0, 1);   plt.title('15-45 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,3);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM3, cmap=plt.cm.Reds)
    plt.clim(0, 1);    plt.title('45-65 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,4);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM4, cmap=plt.cm.Reds)
    plt.clim(0, 1); plt.title('65-80 Years'); plt.axis('square'); plt.axis('off'); plt.colorbar();

    #plt.savefig('this%05d_.png'%(ti))
    #plt.clf()

```

## Interacting nodes

```python
data = model.simulate(S0, E0, Ia0, Is0, Ih0, Ic0, Im0, contactMatrix, Tf, Nt, nodeInteraction='True')

## first M*Nd points are S, then M*Nd points are E, then M*Nd points are Ia,  Is, Ih, Ic, Im, Ni
## thus a total of 8*M*Nd points

radius = np.sqrt(np.arange(Nd)/float(Nd))
golden_angle = np.pi * (3 - np.sqrt(5)); theta = golden_angle * np.arange(Nd)
points = np.zeros((Nd, 2)); points[:,0] = np.cos(theta); points[:,1] = np.sin(theta)
points *= radius.reshape((Nd, 1));  points = points*4*np.sqrt(Nd)

M1=5*M*Nd
f = plt.figure(figsize=(16, 15));
for ti in range(1):
    ti=Nt-1
    IM1=np.zeros((Nd));  IM2=np.zeros((Nd));  IM3=np.zeros((Nd));  IM4=np.zeros((Nd))
    IS=np.zeros((Nd));

    for n in range(Nd):
        for i in range(3):
            IM1[n] += data['X'][ti, M1+n*M+i]
            IS[n] += data['X'][ti, 3*M*Nd+n*M+i]
        for i in range(6):
            IM2[n] += data['X'][ti, M1+n*M+3+i]
        for i in range(4):
            IM3[n] += data['X'][ti, M1+n*M+9+i]
        for i in range(3):
            IM4[n] += data['X'][ti, M1+n*M+12+i]

    print(IM4[0:4])
    #print(IS[0:4])
    Imm = (np.max((IM1, IM2, IM3, IM4))) + 0.000001
    IM1 = IM1/Imm
    IM2 = IM2/Imm
    IM3 = IM3/Imm
    IM4 = IM4/Imm

    sz=200
    sp =  f.add_subplot(2,2,1);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM1, cmap=plt.cm.Reds)
    plt.clim(0, 1);  plt.title('0-15 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,2);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM2, cmap=plt.cm.Reds)
    plt.clim(0, 1);   plt.title('15-45 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,3);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM3, cmap=plt.cm.Reds)
    plt.clim(0, 1);    plt.title('45-65 Years'); plt.axis('square'); plt.axis('off');

    sp =  f.add_subplot(2,2,4);
    plt.scatter(points[:,0], points[:,1], s=sz*np.ones(Nd), c=IM4, cmap=plt.cm.Reds)
    plt.clim(0, 1); plt.title('65-80 Years'); plt.axis('square'); plt.axis('off'); #plt.colorbar();

    #plt.savefig('this%05d_.png'%(ti))
    #plt.clf()
```


# Extended Kalman Filter Raspberry Pi Mouse Version

EKF(Extended Kalman Filter、拡張カルマンフィルタ）はガウシアンフィルタの1つです。

位置推定を行う際にも用いられます。

本ページでは、Raspberry Pi Mouseを用いてロボットがランドマークを観測しながら（拡張）カルマンフィルタで自己位置を推定します。
[raspimouse_http_controller](https://github.com/Tiryoh/raspimouse_http_controller)と組み合わせて使用します。

注がある場合を除き、本ページに掲載されているコードは[MIT](https://tiryoh.mit-license.org/)ライセンスに、文章は[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.ja)ライセンスに基づいて公開します。

## Requirements

このコードを動作させるにはPython 3を必要とします。

## References

S. Thrun, W. Burgard and D. Fox (著), 上田 (訳): 確率ロボティクス, 毎日コミュニケーションズ, 2007

このJupyter Notebookでは [1D_EKF](https://github.com/Tiryoh/probrobo_note/blob/master/ekf/1D_Extended_Kalman_Filter.ipynb) や [2D_EKF](https://github.com/Tiryoh/probrobo_note/blob/master/ekf/1D_Extended_Kalman_Filter.ipynb) を読んだ前提で処理が進められていきます。
まだ読んでいない場合はそちらから先に読んでください。
実際に動かすことに重きを置いていますので、アルゴリズムの詳細な説明は一部省略されています。

# モジュールのインポート

Jupyter Notebookに必要な各種モジュールをインポートします。

```python
# 数値計算及び描画に必要なモジュール
import numpy as np
import math
from  matplotlib import pyplot as plt
from  matplotlib import patches as patches
from copy import copy
import random

# 通信用
import urllib.request
import time
import argparse
import json
import threading

# JupyterNotebook用モジュール
from matplotlib import animation
from IPython.display import HTML
from IPython import display
plt.rcParams.update({'figure.max_open_warning': 0}) #20枚以上画像を描画しても警告を出さないように設定

# デバッグ用
from pprint import pprint
```

# ランドマーククラス
ロボットにとって目印となるランドマークを宣言します。
ロボットはこのランドマークの座標をあらかじめ知っています。

```python
class Landmarks:
    """
    landmarks = Landmarks([x1, x2, ... , x])
    引数にランドマークのX座標とY座標を配列で指定
    """
    def __init__(self, array):
        """
        ランドマークの座標を配列で取得
        あとで計算処理しやすくするため、numpyのarrayに変換
        """
        self.positions = np.array(array)

    def draw(self, subplot = plt):
        """
        取得したすべてのランドマークの座標をプロット
        """
        xs = [ e for e in self.positions ]
        ys = [ 0.1 for e in self.positions ]
        #subplot.scatter(xs, ys, s=600, marker='*', label='landmarks', color='orange')
        subplot.scatter(xs, ys, s = 200, marker = '^',color = 'black', label = 'landmark')
```

# ロボット実機を制御するためのクラス

受信用のクラス(Receiver)と送信用のクラス(Sender)を作成します。ラズパイマウスとTCP通信を行います。

```python
class Receiver(object):
    def __init__(self, ip, port):
        print("receiver init")
        self.url = "http://{}:{}".format(ip, port)
        print ("connecting to " + self.url)

    def get_sensor_val(self):
        start = time.time()
        request = urllib.request.Request(self.url)
        response = urllib.request.urlopen(request)
        sensor_dict = json.loads(response.read().decode('utf-8'))
        return sensor_dict

class Sender(object):
    def __init__(self, ip, port):
        print("sender init")
        self.url = "http://{}:{}".format(ip, port)
        print ("connecting to " + self.url)

    def set_motor_power(self, mode):
        response_data = []
        request = urllib.request.Request(self.url)
        obj = {
        "motor_power" : mode
        }
        json_data = json.dumps(obj).encode('utf-8')
        headers = {"Content-Type" : "application/json"}
        request = urllib.request.Request(self.url, data=json_data, headers=headers, method='POST')
        with urllib.request.urlopen(request) as response:
            response_body = response.read().decode('utf-8')
            response_data.append(response_body)
        return response_data

    def set_motor_speed(self, left, right):
        response_data = []
        request = urllib.request.Request(self.url)
        obj = {
        "motor" : {
            "l" : int(left),
            "r" : int(right)
        }
        }
        json_data = json.dumps(obj).encode('utf-8')
        headers = {"Content-Type" : "application/json"}
        request = urllib.request.Request(self.url, data=json_data, headers=headers, method='POST')
        with urllib.request.urlopen(request) as response:
            response_body = response.read().decode('utf-8')
            response_data.append(response_body)
        return response_data
```

# 通信用クラスの初期化

ラズパイマウスのIPアドレスと通信用のポートを指定します。

```python
r = Receiver("192.168.22.5", 5000)
s = Sender("192.168.22.5", 5000)
#r = Receiver("127.0.0.1", 5000)
#s = Sender("127.0.0.1", 5000)
```

    receiver init
    connecting to http://192.168.22.5:5000
    sender init
    connecting to http://192.168.22.5:5000

# ロボットのクラス

ロボットを操作するためのクラスです。
今回は説明用のグラフ描画のためのメソッド（`draw_graph`）もこのクラスで取り扱っています。

状態空間モデルの状態遷移確率 : $ {x}_t = g({u}_t, {x}_{t-1})+{\epsilon}_t $

## ロボットを操作するためのクラス

実際のロボットは自身の絶対座標を知るすべはありません。そのため、ロボットを操作するためのクラスではロボットの座標を把握していません。

ロボットを操作するためのクラスに与えることができるのは制御入力ベクトルの${u}_t$です。`move`メソッドに`distance`を引数として与えます。

```python
class Robot:
    """
    ロボットの動作を決めるためのクラス
    """
    def __init__(self, x = 0.0):
        self.ekf = EKF(robot_pos = x)
        self.expected_poses = [np.array(x)]

    def move(self, distance):
        # 理論上の位置を計算
        self.expected_poses.append(np.array(self.expected_poses[-1] + distance))
        # ラズパイマウスを0.05秒だけ進ませる
        speed = 400 * distance / 6
        s.set_motor_power(1)
        s.set_motor_speed(speed, speed)
        time.sleep(0.05)
        s.set_motor_speed(0, 0)
        s.set_motor_power(0)
        # 制御入力(distance)に基づき、EKFの予測ステップを適用
        self.ekf.prediction_step(distance)

    def observation(self, landmarks):
        """
        ランドマーク観測の際に使用
        """
        obss = []
        # すべてのランドマークについて、観測できるか確認
        for i,landmark in enumerate(landmarks.positions):
            # ランドマークの観測結果を保存
            obs = Observation(landmark)
            # センサの範囲内だったランドマークをリストに追加
            if obs.distance != None:
                obss.append([obs, i])
        for obs in obss:
            # 観測結果を一つずつ使いながらEKFの修正ステップを適用
            self.ekf.update_step(obs[0].distance, landmarks.positions[obs[1]])
        return obss

    def draw_graph(self, subplot, obss):
        x = np.arange(-50, 250, 0.1)
        subplot.scatter(self.expected_poses[-1], 0.1, s = 200, marker = '^',color = 'red',label = 'expected robot motion')
        subplot.plot(x, self.generate_gaussian(self.ekf.bel_x_prev, self.ekf.cov_x_prev), color = 'green', label='bel_x (t-1)')
        subplot.plot(x, self.generate_gaussian(self.ekf.priori_bel_x, self.ekf.priori_cov_x), color = 'blue', label='priori_bel_x')
        subplot.plot(x, self.generate_gaussian(self.ekf.posteriori_bel_x, self.ekf.posteriori_cov_x), color = 'orange', label='posteriori_bel_x ')

    def generate_gaussian(self, bel, cov):
        x = np.arange(-50, 250, 0.1)
        return np.exp(-(bel - x) ** 2 / (2 * cov)) / np.sqrt(2 * np.pi * cov)
```

# 観測のためのクラス

ランドマークを観測するためのクラスです。
ロボットの現在地からセンサの観測範囲内にランドマークがあるかどうかをチェックします。

状態空間モデルの計測確率: $ {z}_t = h({x}_t ) + {\delta}_t $

`measure`メソッドの中の`actual_distance`が $z_t$ に、`distance`が $h({x}_t )$ に該当します。

```python
class Observation:
    """
    ロボットの観測
    """
    def __init__(self, sensor_min_value = 15, sensor_max_value = 2000):
        # センサ値の上限と下限を設定
        self.sensor_max_value = sensor_max_value
        self.sensor_min_value = sensor_min_value

        # 観測できない場合は観測結果をNoneとし、観測できた場合はこの値を上書きする
        self.distance, self.direction, self.error_ellipse = None, None, None

        self.measure()

    def measure(self):
        """
        センサの値を取得して対象物があるかチェックする
        """
        # センサ値が有効値内かどうか確認する
        sensor_data = r.get_sensor_val()
        time.sleep(0.05)
        sensor_average = (sensor_data["lightsensor"]["0"] + sensor_data["lightsensor"]["3"]) / 2
        if not sensor_average < self.sensor_max_value and sensor_av > self.sensor_min_value:
            print('sensor data out of range')
            return # 範囲外だった場合はNoneのまま終了

        # センサの値を距離（mm）に変換
        distance = sensor_average ** -0.57 * 3435.3

        # センサ値を保存
        self.distance = distance
```

# カルマンフィルタのクラス

本ページでは直線上を移動するロボットの自己位置推定を行っています。
直線上の移動の場合、非線形関数を線形近似する段階がありません。一般的なカルマンフィルタと同じ計算をしています。

## 予測ステップ
予測ステップでは、1つ前の時間($t-1$)の状態を元に、
事前推定値と事前誤差共分散を求めます。
1つ前の時間($t-1$)の
事後推定値を${{\mu}}_{t-1}$
、
事後誤差共分散を${{\Sigma}_{{\bar{x}}_{t-1}}}$
と表します。
事前推定値を$\bar{{\mu}}_t$、
事前誤差共分散を${{\Sigma}_{\bar{{x}}_t}}$
とすると、

カルマンフィルタの場合は、

\begin{align}
    \bar{{\mu}}_t
    &=& {A}_t {{\mu}}_{t-1} + {B}_t {u}_{t} \\
    {{\Sigma}_{\bar{{x}}_t}}
    &=& {A}_t {{\Sigma}_{{{x}}_{t-1}}} {{A}_t}^T+ {R}_t
\end{align}

と表すことができます。

拡張カルマンフィルタの場合は、

\begin{align}
    \bar{{\mu}}_t
    &=& g({u}_t, {\mu}_{t-1}) \\
    {{\Sigma}_{\bar{{x}}_t}}
    &=& {G}_t {{\Sigma}_{{{x}}_{t-1}}} {{G}_t}^T+ {R}_t
\end{align}

と表すことができます。
なお、${G}_t$は、状態遷移確率${x}_t$を求める非線形関数$g$のヤコビ行列です。

## 修正ステップ

修正ステップでは、予測ステップを元に、カルマンゲイン、事後推定値、事後誤差共分散を求めます。
カルマンゲインを${K}_t$、
事後推定値を${\mu}_t$、
事後誤差共分散を${\Sigma}_{{x}_t}$
とすると、

カルマンフィルタの場合、

\begin{align}
    {K}_t
    &=& {{\Sigma}_{\bar{{x}}_t}}{{C}_t}^T({C}_t {{\Sigma}_{\bar{{x}}_t}} {{C}_t}^T + {{Q}_t})^{-1} \\
    {\mu}_t
    &=& \bar{{\mu}}_t + {K}_t ({z}_t - {{C}_t} \bar{{\mu}}_t) \\
    {{\Sigma}_{{{x}}_t}}
    &=& (1 - {K}_t {C}_t) {{\Sigma}_{\bar{{x}}_t}}
\end{align}

と表すことができます。

拡張カルマンフィルタの場合、

\begin{align}
    {K}_t
    &=& {{\Sigma}_{\bar{{x}}_t}}{{H}_t}^T
    ({H}_t {{\Sigma}_{\bar{{x}}_t}} {{H}_t}^T
    + {{Q}_t})^{-1} \\
    {\mu}_t
    &=& \bar{{\mu}}_t + {K}_t ({z}_t - h( \bar{{\mu}}_t)) \\
    {{\Sigma}_{{{x}}_t}}
    &=& (1 - {K}_t {H}_t)
    {{\Sigma}_{{\bar{x}}_t}}
\end{align}

と表すことができます。
${H}_t$ は、ロボットの計測確率 ${z}_t$ を求める非線形関数$h$のヤコビ行列です。

ここで求められた
事後推定値${\mu}_t$
と
事後誤差共分散${{\Sigma}_{{{x}}_t}}$
が、時刻$t$におけるカルマンフィルタ適用後の
事後信念です。

$K$と`kalman_gain` 、$\mu_t$ と `posteriori_bel_x` 、${\Sigma}_{{{x}}_t}$ と`posteriori_cov_x`がそれぞれ対応しています。

カルマンゲインの
${{H}_t {{\Sigma}_{{\bar{x}}_t}} {{H}_t}^T + {{Q}_t}}$
は計測予測の不確かさを表しています。
計測がより確かであれば、不確かさは小さくなり、
カルマンゲインは大きくます。
今回の環境においては、
ロボットがランドマークを観測できると、
計測の不確かさが小さくます。
EKFの修正ステップにおける修正幅が大きくなり、
事後推定値がより真のロボットの位置姿勢に近づいていきます。

```python
class EKF:
    """
    EKFのクラス
    """
    def __init__(self, robot_pos, cov = 0.1, sigma_robot_translation_error = 0.5, sigma_robot_observation_error = 0.9):
        # ロボットの雑音：ロボットのクラスで定義されている値と近いほど正確な位置予測ができる
        # ロボットの制御入力に対する雑音の標準偏差
        self.sigma_robot_translation_error = sigma_robot_translation_error
        # ロボットの観測に対する雑音の標準偏差
        self.sigma_robot_observation_error = sigma_robot_observation_error
        # 初回の事前推定値を計算するための事後推定値
        # 推定位置の初期位置（初期信念）
        self.posteriori_bel_x = robot_pos
        # 初回の事前誤差共分散を計算するための事後誤差共分散
        # 推定値の初期位置の分散（標準偏差**2）
        self.posteriori_cov_x = cov

    def prediction_step(self, distance):
        """
        予測ステップ
        """
        # 1つ前の時間の状態を保存
        self.bel_x_prev = self.posteriori_bel_x
        self.cov_x_prev = self.posteriori_cov_x
        # 1つ前の時間の事後推定値を元に、事前推定値を計算
        self.priori_bel_x = self.bel_x_prev + random.gauss(distance, distance * self.sigma_robot_translation_error)
        # 1つ前の時間の事後誤差共分散を元に、事前誤差共分散を計算
        self.priori_cov_x = self.cov_x_prev + self.sigma_robot_translation_error ** 2

        # ランドマークを観測できなかったとき用
        self.posteriori_bel_x = self.priori_bel_x
        self.posteriori_cov_x = self.priori_cov_x

    def update_step(self, obs, landmark):
        """
        修正ステップ
        """
        # カルマンゲイン
        kalman_gain = self.priori_cov_x / (self.priori_cov_x + self.sigma_robot_observation_error ** 2)
        # 事後推定値
        self.posteriori_bel_x = self.priori_bel_x + kalman_gain * (abs(landmark - self.priori_bel_x) - obs)
        #print(obs ,landmark, self.priori_bel_x, (landmark - self.priori_bel_x))
        # 事後誤差共分散
        self.posteriori_cov_x = (1 - kalman_gain) * self.priori_cov_x
```

# ロボット操作用コードの実行

1つ前の時間の推定値（信念）を `bel_x(t-1)` として、予測ステップ後の事前推定値を `priori_bel_x` として、修正ステップ後の事後推定値を `posteriori_bel_x` として表します。

```python
# ランドマークの座標を指定
landmarks = Landmarks([200.0])

# ロボットの初期座標を指定
robot = Robot(0.0)

# アニメーション作成の更新処理
def update(i):
    # 新規ウィンドウと座標軸用subplotを用意
    # タイトルは個別につけるので今は指定しない
    fig, graph = plt.subplots(figsize=(12, 6))
    # 座標軸の範囲を指定
    graph.set_xlim(-50, 250)
    graph.set_ylim(0.0, 3.0)
    # ランドマークを描画
    landmarks.draw(graph)
    # ロボットを移動
    robot.move(5)
    # 移動後の位置でランドマークを観測
    obss = robot.observation(landmarks)
    # ロボットを描画
    robot.draw_graph(graph, obss)
    # タイトルを追加
    graph.set_title("Robot Position Estimation : frame{0:04d}".format(i))
    # 左上に凡例を追加
    graph.legend(loc='upper left')

# アニメーションの作成
for i in range(0, 20):
    update(i)

    # gifアニメーションのように毎回表示内容を更新する場合は plt.show ではなくこちらを使用
    display.clear_output(wait=True)
    display.display(plt.gcf())

    # 履歴を全表示する場合は display.display ではなくこちらを使用
    #plt.show()
```


```python
# Copyright (c) 2020-2021 CertifAI Sdn. Bhd.
#
# This program is part of OSRFramework. You can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
```

# Statistics and Probability: Fundamentals

```python
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
```

# Standardisation

We consider the grades dataset, which contains Physics, Biology and Maths scores for 26 students:

```python
grades = pd.read_csv("https://archive.org/download/ml-fundamentals-data/machine-learning-fundamentals-data/grades.csv")
print(grades.shape)
grades.agg(["mean", "var", "std"])
```

    (26, 4)

      Physics
      Biology
      Maths

      mean
      63.769231
      67.038462
      69.769231

      var
      517.224615
      456.758462
      300.664615

      std
      22.742573
      21.371908
      17.339683

```python
g = sns.FacetGrid(grades.melt("Name"), col="variable")
g.map(plt.hist, "value")
```

Suppose the instructor wanted to provide the students with an idea of how well they are performing relative to their peers, but not their exact score. This can be done by **standardising** the scores. This is done by subtracting the mean and re-scaling the data to be in units of standard deviations.

```python
grades["Physics_std"] = (grades["Physics"] - grades["Physics"].mean()) / grades["Physics"].std()
```

```python
grades.head()
```

      Name
      Physics
      Biology
      Maths
      Physics_std

      0
      Arnold
      80
      78
      70
      0.713673

      1
      Bob
      60
      98
      45
      -0.165735

      2
      Cassandra
      34
      56
      65
      -1.308965

      3
      Donovan
      56
      65
      32
      -0.341616

      4
      Emily
      98
      23
      55
      1.505141

What can we conclude?

Firstly, Arnold and Emily are above average Physics students, whereas Bob, Cassandra and Donovan are below average.

Also, Emily's scores are **1.5 standard deviations above the mean**, whereas Cassandra's scores are **1.3 standard deviations below the mean**.

More importantly, standardisation allows us to compare the results from different exams. Standardisation allows each of the variables to be expressed in common units: their respective standard deviation.

We can do the same for Math scores, which have a lower variance than Physics scores:

```python
grades["Maths_std"] = (grades["Maths"] - grades["Maths"].mean()) / grades["Maths"].std()
grades.head()
```

      Name
      Physics
      Biology
      Maths
      Physics_std
      Maths_std

      0
      Arnold
      80
      78
      70
      0.713673
      0.013309

      1
      Bob
      60
      98
      45
      -0.165735
      -1.428471

      2
      Cassandra
      34
      56
      65
      -1.308965
      -0.275047

      3
      Donovan
      56
      65
      32
      -0.341616
      -2.178196

      4
      Emily
      98
      23
      55
      1.505141
      -0.851759

What do we observe?

 - While Arnold is doing really well in Physics, he's barely above average in Maths.
 - Bob, Cassandra and Donovan are below average in both Physics and Maths.
 - While Emily seems to be an above average Physics student, she's quite the opposite when it comes to Maths.

# Fundamentals of Probability

For your reference, the rules of probability are reproduced as follows:
1. The probability of an event A is between 0 and 1 (inclusive): $0 \leq p(A)\leq 1$
2. The sum of the probability of all possible events (the *sample space*) is 1. $\Sigma^n_{i=0} p(A_i)=1$
3. The sum of the probability of an event and its complement is 1. $p(A^C)+p(A) = 1$
4. (Addition Rule) The probability of event A *or* event B happening is equal to the sum of the probabilities of event A and event B happening respectively, *less* the probability of event A **and** event B. $p(A \cup B)= p(A) + p(B) - p(AB)$
5. (Multiplicative Rule) If events A and B are independent, then the probability of event A and B jointly occuring is the product of the probabilities of events A and B respectively. $p(AB) = p(A)p(B)$

📝 Note: Events A and B are independent if the occurence of event A doesn't change the likelihood of Event B occuring. Do not confuse this with mutually exclusive events. If events A and B are mutually exclusive (disjoint), they cannot both happen.

## Conditional Probability
|Sex/Handedness|Right handed| Left handed|
|--------------|-----------:|-----------:|
|Male          |43          | 9          |
|Female        |44          | 4          |

The contingency table above summarises some fake data collected from 100 respondents on genders and handedness.

**What is the probability that any one respondent in our survey is female?** To calculate the probability, divide the number of events by the sample space: $\frac{44+4}{100}=0.48$. It is a little more likely that our respondents are male than female.

However, what is the probability that a female in our sample is **left-handed**? Quite unlikely. To quantify the likelihood, we could divide the number of left-handed females by the number of females in our sample: $\frac{4}{48}=\frac{1}{12}$.

$\frac{1}{12}$ is a **conditional probability**: a probability under a condition. Note that to calculate the conditional probability, we restrict the sample space under consideration to a condition, and then recalculate the probability as if the condition were the entire sample space.

A general formula for conditional probability is as follows. The conditional probability of A given B is

$$P(A|B)=\frac{P(A\cap B)}{P(B)}$$

We take the joint probability $P(A\cap B)$ and 'inflate' it with the probability of $P(B)$.

Conditional probabilities are fundamental to the idea of **independence**. Recall that we defined independence of two events A and B as

> the probability of event A is not changed by the occurence of event B.

We can restate the definition above as

$$P(A|B)=P(A)$$

Consider whether handedness is independent of sex. The probability of being left-handed is $11/100$, or 0.11. As you might expect, it's close to the conditional probability $P(left$-$handed|female)=0.12$.

## Law of Total Probability

Let $A_1, \dots A_n$ be mutually exclusive and exhaustive events, where $P(A)\neq 0$ for all $A$. Then, the probability of $P(B)$ can be defined as

$$P(B)=\sum^n_{i=1}P(B|A_i)P(A_i)$$

### Example

The probability that any given person in the population has a disease is 0.1. The likelihood of testing positive for a disease if you have it is 0.95. The probability of testing positive in the absence of the disease (a false positive) is 0.10.

What is the probability of testing positive for the disease?

$$P(T = 1|D = 1) = 0.95$$
$$P(T = 1|D = 0) = 0.10$$
$$P(D = 1) = 0.1$$

$$\begin{align}
P(T=1)&=P(T=1|D=1)P(D=1) + P(T=1|D=0)P(D=0)\\
&= 0.95\times 0.1 + 0.10*0.90\\
&=0.14
\end{align}$$

# Bayes' Theorem

$$P(A|B)=\frac{P(B|A)P(A)}{P(B)}$$

$$Posterior=\frac{Likelihood\times Prior}{Evidence}$$

An informal way of putting Bayes' Theorem:

$$P(pred|data)=\frac{P(data|actual)P(actual)}{P(data)}$$

The proof for Bayes' Theorem follows from the definition of conditional independence. We know that

$$\begin{align}
P(A\cap B)&=P(A|B)\times P(B)\\
&=P(B|A)\times P(A)
\end{align}$$

This can be rearranged to state Bayes's Theorem:

$$P(A|B)=\frac{P(B|A)P(A)}{P(B)}$$

## Example 1

You want to go for a picnic today but it is cloudy in the morning. From experience, you know that:

- the probability of raining on any given day is 10%,
- the probability that it is cloudy on any given day is 40%, and
- 50% of all rainy days start off cloudy.

What is the probability of raining given that it is cloudy?

$P(Rain | Cloud) = \frac {P(Cloud | rain) \times P(rain)}{P(Cloud}=\frac{0.5\times 0.1}{0.4}=0.125$

## Example 2

The probability that any given person in the population has a disease is 0.1. The likelihood of testing positive for a disease if you have it is 0.95. The probability of testing positive in the absence of the disease (a false positive) is 0.10.

If you have been tested positive for a disease. What is the probability that you actually have the disease?

What is the probability of actually getting the disease given tested positive?

$$P(T = 1|D = 1) = 0.95$$
$$P(T = 1|D = 0) = 0.10$$
$$P(D = 1) = 0.1$$

As we've done earlier, we can use the law of total probability to find $P(T=1)$:

$$\begin{align}
P(T=1)&=P(T=1|D=1)P(D=1) + P(T=1|D=0)P(D=0)\\
&= 0.95\times 0.1 + 0.10*0.90\\
&=0.14\end{align}$$

Note that $$P(T = 1|D = 1) = 0.95$$ is the likelihood and $$P(D = 1) = 0.1$$ is the prior.

$\begin{align}
P(D=1|T=1)&=\frac{P(T=1|D=1)P(D=1)}{P(T=1)}\\
&=\frac{0.95\times 0.1}{0.14}\\
&=0.68
\end{align}$

# Probability Distributions
A random variable is a variable whose possible values are the outcomes of a non-deterministic process.

## Random variables
- A **continuous random variable** takes on infinitely many values/real numbers. In fact, the probability of a continuous random variable taking on any one value is 0.
- A **discrete random variable** takes on a finite set of values, e.g. counts.

## Binomial distribution

We start with the **binomial distribution**. The binomial distribution models the probability of $k$ successes in $n$ independent trials.

Let's redo the lecture example with `scipy`. Import the `binom` class from `scipy.stats`:

```python
from scipy.stats import binom
```

If the chance of winning a game is 0.75, what is the probability of losing all 5 games?

Assume the chance of winning each game is independent. Then we can solve this problem using the binomial distribution.

We use the `.pmf` method of the `binom` class. `pmf` stands for **probability mass function**: a function that states the probability that a discrete random variable is equal to any one value.

```python
binom.pmf(k=5, n=5, p=0.25)
```

    0.0009765625

The answer is what we should expect. If we win 3 in 4 games, it's unlikely we lose 5 in 5 games.

Now, suppose we want to know what is the probability of winning 3 games or less. We want to find $P(X \leq 3)$.

Note $P(X \leq 3) = P(X=0)+ P(X=1)+P(X=2)+P(X=3)$.

By the **complement rule**, $P(X \leq 3) = 1 - P(X=5)-P(X=4)$.

```python
1 - binom.pmf(k=4, n=5, p=0.75) - binom.pmf(k=5, n=5, p=0.75)
```

    0.36718749999999994

Note that we could also answer this by referring to the value of the **cumulative density function** at $X=3$.

The cumulative density function, $F(x)$, is defined as

$$F(x)=P(X\leq x)$$

```python
binom.cdf(3, n=5, p=0.75)
```

    0.3671875

## Normal distribution

We move on to the **normal distribution**. There are some key properties that are important to know about this distribution:

- The mean, median and mode of the distribution is the same.
- The curve is bell-shaped.
- The total area under the curve is 1.
- Symmetric about the mean.

There are two key **shape parameters** that define the location and spread of the normal distribution: the mean, commonly denoted as $\mu$; and the standard deviation, commonly denoted as $\sigma$.

Again, start by importing the `norm` class from `scipy.stats`:

```python
from scipy.stats import norm
```

### Standard normal distribution

A normally distributed random variable with mean 0 and standard deviation 1 is said to follow the standard normal distribution, also known as the **z-distribution**.

A normally distributed random variable can be converted into the standard normal distribution by standardisation:

$$Z = \frac{X-\mu}{\sigma}$$

    Note: The .pdf method refers to the probability density function, which is the counterpart of the probability mass function for continuous random variables.

```python
x = np.linspace(-10, 10, 100)
f1 = norm.pdf(x, loc=0, scale=1)
f2 = norm.pdf(x, loc=1, scale=1)
f3 = norm.pdf(x, loc=0, scale=2)

plt.plot(x, f1, label="$\mu=0, \sigma=1$")
plt.plot(x, f2, label="$\mu=1, \sigma=1$")
plt.plot(x, f3, label="$\mu=0, \sigma=2$")
plt.vlines(x=0, ymin=0, ymax=0.5, linestyles="dotted")

plt.legend()
plt.title("$\mu$ and $\sigma$ affect the shape of the normal distribution")
```

With continuous distributions, we do not want to find the probability of a random variable at any given value. Since the range is infinite, the probability that a continuous random variable takes on any given value is zero.

We are usually interested in finding out the probability that the random variable lies between two values.

### Programmatic solutions: percent point function

Recall the definition of the **cumulative density function**:

$$F(x)=P(X\leq x)$$

The CDF returns the probability that a random variable is less than $x$. For instance, because of the symmetric shape of the normal distribution, the CDF of the standard normal distribution at 0 is expected to be 0.5.

```python
norm.cdf(0)
```

    0.5

We will also use the **percent point function**, available via the `ppf()` method. The percent point function does the inverse of the cumulative density function. It returns the percent point given the value of the CDF:

```python
norm.ppf(0.5)
```

    0.0

### Standard deviation rule/empirical rule

A useful rule of thumb is the **standard deviation rule**, or empirical rule.

This rule states that for a normally distributed random variable,
- 68% of data lies within 1 standard deviation of the mean.
- 95% of data lies within 2 standard deviations of the mean, and
- 99.7% of data lies within 3 standard deviations of the mean.

### Example: height

Maalysian males' heights are normally distributed with mean 169.57 and standard deviation 7.57. Answer the following questions programmatically and using the empirical rule.

- 84% of Malaysians have height of less than ... cm.

```python
norm.ppf(0.84, loc=169.57, scale=7.57)
```

    177.09804617589782

```python
169.57 + 7.57
```

    177.14

- The tallest 2.5% of Malaysians are taller than ... cm

```python
norm.ppf(0.975, loc=169.57, scale=7.57)
```

    184.4069273629682

```python
169.57 + 2 * 7.57
```

    184.70999999999998

- The shortest 0.15% of Malaysians are shorter than ... cm

```python
norm.ppf(0.0015, loc=169.57, scale=7.57)
```

    147.10422390516268

```python
169.57 - 3 * 7.57
```

    146.85999999999999

## Poisson distribution

The Poisson distribution models the number of events that occur in a fixed time interval.

Use the Poisson distribution when:
- events occur at random points of time and space, and
- our interest lies is the number of occurrences of the event.

The most important property of the Poisson variable is that the waiting time between events is **memoryless**:
- the time when events occur do not affect the number of successes in an interval;
- only the length of time affects the number of successes.

The Poisson distribution has a single parameter: lambda ($\lambda$). The mean and variance of the Poisson distribution coincides with the shape parameter $\lambda$.

The probability mass function of the Poisson distribution is as follows:

$$p(X=k) = \frac{e^{-\lambda}\lambda^{k}}{k!}$$

```python
from scipy.stats import poisson
```

```python
# a Poisson distribution with lambda=1 can be interpreted as:
# on average, 1 event occurs within the set time interval.
# naturally, it is less likely that more than 1 event occurs.

x = np.arange(10)
f1 = poisson.pmf(x, mu=1)
f2 = poisson.pmf(x, mu=2)
f3 = poisson.pmf(x, mu=3)

plt.bar(x, f1, label="$\lambda=1$")
plt.legend()

```

```python
# the average can be calculated by multiplying the inputs with the values of the PMF
x @ f1
# see that this is equal to lambda
```

    0.999998874797402

```python
# for lambda=2, having 2 events is a lot more likely

plt.bar(x, f2, label="$\lambda=2$")
plt.legend()
```

```python
x @ f2
```

    1.999525105343477

```python
plt.bar(x, f3, label="$\lambda=3$")
plt.legend()
```

```python
x @ f3
```

    2.9885910238149727

## Example

Suppose emergency calls per hour follow a Poisson distribution and occur at a rate of 5 calls per hour. What is the probability of receiving 3 calls in any given hour?

```python
poisson.pmf(3, 5)
```

    0.1403738958142805

# Central Limit Theorem

The Central Limit Theorem helps us reason about realisations of sample means.

Specifically, it tells us that in many repeated samples, the distribution of the sample means of a random variable is normal regardless of the underlying population distribution.

```python
from scipy.stats import poisson
```

```python
bins = np.arange(0, 40)
p_pois = poisson.pmf(bins, mu=3)
plt.plot(bins, p_pois)
```

```python
def sample_pois(_):
    samples = poisson.rvs(mu=3, size=20)
    return np.mean(samples)

means = list(map(sample_pois, np.arange(400)))
plt.hist(means)
```

# Statistic and Probability in Deep Learning - Using Normal Distribution in Weight Initialization

**Normal Distribution** could be very helpful when it comes to weight initialization of deep learning task.

In deep neural network, we may encounter vanishing/exploding gradients when the derivaties become very small or very big. The activations and derivatives will be decreased/increased exponentially as a function of number of layers. This will make training difficult, especially when gradients are exponentially smaller, then gradient descent wiill take tiny little steps. Consequently it will take a long time for gradient descent to learn anything.

Therefore, using a normal distributed weight initiallization with mean equals to 0 and a standard deviation of 1 could be helpful in resolving the issue.

For example, when we using ReLU as our activation function, we would do the following steps:

- Generate random sample of weights from a Normal Distribution having mean equals 0 and a standard deviation of 1.
- Multiply the sample with $\sqrt{(\frac{2}{n_i})}$, where $n_i$ is number of input units for that layer.

```np.random.rand(shape) * np.sqrt(2/n[l-1])```, where `l` is the total number of input units.

> This initialization is also called **Xavier Initialization**.

This sort of initialization helps to set the weight matrix neither too bigger than 1, nor too smaller than 1. Thus it doesn’t explode or vanish gradients respectively.


## Dimensionality Reduction##

## Independent Component Analysis (ICA)

Independent component analysis (ICA) solves the ``cocktail party problem'', where there are multiple microphones situated through out a room containing
$N$ people. Each microphone picks up a linear combination of the $N$
voices. The goal of ICA is to use the concept of statistical
independence to isolate (or unmix) the individual signals.

Each spectrum, $x_i(k)$, can be described by

>$x_1(k) = a_{11}s_1(k) + a_{12}s_2(k) + a_{13}s_3(k) + \cdots\\
x_2(k) = a_{21}s_1(k) + a_{22}s_2(k) + a_{23}s_3(k) + \cdots \\
x_3(k) = a_{31}s_1(k) + a_{32}s_2(k) + a_{33}s_3(k) + \cdots \\
$

where $s_i(k)$ are the individual  spectra and $a_{ij}$
the appropriate mixing amplitudes.

In matrix form,
>$ X = A S$,

 Extracting these signal spectra is equivalent to estimating the appropriate weight matrix,
$W$, such that
>$ S = W X$.

Underlying ICA is the observation that the
input signals, $s_i(k)$, should be **statistically
independent** - the joint probability distribution, $f(x,y)$, can be
fully described by  $f(x^p,y^q)=f(x^p)f(y^q)$

 The rationale for ICA is that the
sum of any two independent random variables will always be more
Gaussian than either of the individual random variables (i.e., from the
central limit theorem). If we identify an unmixing matrix, $W$, that
maximizes the non-Gaussianity of the distributions, then we would be
identifying the input signals.

```python
# Author: Jake VanderPlas
# License: BSD
#   The figure produced by this code is published in the textbook
#   "Statistics, Data Mining, and Machine Learning in Astronomy" (2013)
#   For more information, see http://astroML.github.com
#   To report a bug or issue, use the following forum:
#    https://groups.google.com/forum/#!forum/astroml-general
import numpy as np
from matplotlib import pyplot as plt

from sklearn.decomposition import FastICA

from astroML.datasets import sdss_corrected_spectra
from astroML.decorators import pickle_results

#----------------------------------------------------------------------
# This function adjusts matplotlib settings for a uniform feel in the textbook.
# Note that with usetex=True, fonts are rendered with LaTeX.  This may
# result in an error if LaTeX is not installed on your system.  In that case,
# you can set usetex to False.
from astroML.plotting import setup_text_plots
setup_text_plots(fontsize=8, usetex=True)

#------------------------------------------------------------
# Download data
data = sdss_corrected_spectra.fetch_sdss_corrected_spectra()
spectra = sdss_corrected_spectra.reconstruct_spectra(data)
wavelengths = sdss_corrected_spectra.compute_wavelengths(data)

#----------------------------------------------------------------------
# Compute ICA components

def compute_ICA(n_components=5):
    spec_mean = spectra.mean(0)

    # ICA treats sequential observations as related.  Because of this, we need
    # to fit with the transpose of the spectra
    ica = FastICA(n_components - 1, algorithm='deflation')
    ica.fit(spectra.T)
    ica_comp = np.vstack([spec_mean,
                          ica.transform(spectra.T).T])

    return ica_comp

n_components = 5
decompositions = compute_ICA(n_components)

#----------------------------------------------------------------------
# Plot the results
fig = plt.figure(figsize=(10, 8))
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.05,
                    bottom=0.1, top=0.95, hspace=0.05)

titles = ['ICA components',]

for j in range(n_components):
    ax = fig.add_subplot(n_components, 1, j + 1)

    ax.yaxis.set_major_formatter(plt.NullFormatter())
    ax.xaxis.set_major_locator(plt.MultipleLocator(1000))
    if j < n_components - 1:
        ax.xaxis.set_major_formatter(plt.NullFormatter())
    else:
        ax.set_xlabel(r'wavelength ${\rm (\AA)}$')

    ax.plot(wavelengths, comp[j], '-k', lw=1)

    # plot zero line
    xlim = [3000, 7999]
    ax.plot(xlim, [0, 0], '-', c='gray', lw=1)
    ax.set_xlim(xlim)

    if j == 0:
        ax.set_title(titles[0])
        label = 'mean'
    else:
        label = 'component %i' % j

    ax.text(0.03, 0.94, label, transform=ax.transAxes, ha='left', va='top')

    for l in ax.get_xticklines() + ax.get_yticklines():
        l.set_markersize(2)

    # adjust y limits
    ylim = plt.ylim()
    dy = 0.05 * (ylim[1] - ylim[0])
    ax.set_ylim(ylim[0] - dy, ylim[1] + 4 * dy)

plt.show()
```

## Non negative factorization##

Eigenvectors are defined relative to  mean data vector. Principal components that can be positive or negative but for many physical systems we know data are a linear sum of positive components (e.g. galaxy spectrum is a linear sum of stellar components

Nonnegative matrix factorization (NMF) applies positivity constraint

Assumes  any data matrix can be factored into two matrices, $W$ and $Y$, with
>$\begin{equation}
X=W Y,
\end{equation}
$

and both $W$ and $Y$ are nonnegative

$WY$ is an approximation of $X$. Minimizing the reconstruction error $|| (X - W Y)^2 ||$
nonnegative bases can be derived using a simple update rule (Lee and Seung),
>$
\begin{eqnarray}
W_{ki} &=& W_{ki}\frac{[XY^T]_{ki}}{[WYY^T]_{ki}},\\
Y_{in} &=& Y_{in}\frac{[W^TX]_{in}}{[W^TWY]_{in}},
\end{eqnarray}
$

Iterative process does not guarantee nonlocal minima (like $K$-means and EM).

Use random initialization and cross-validation

```python
# Author: Jake VanderPlas
# License: BSD
%matplotlib inline

import numpy as np
from matplotlib import pyplot as plt

from sklearn.decomposition import NMF
from sklearn.decomposition import RandomizedPCA

from astroML.datasets import sdss_corrected_spectra
from astroML.decorators import pickle_results

#----------------------------------------------------------------------
# This function adjusts matplotlib settings for a uniform feel in the textbook.
# Note that with usetex=True, fonts are rendered with LaTeX.  This may
# result in an error if LaTeX is not installed on your system.  In that case,
# you can set usetex to False.
from astroML.plotting import setup_text_plots
setup_text_plots(fontsize=8, usetex=True)

#------------------------------------------------------------
# Download data
data = sdss_corrected_spectra.fetch_sdss_corrected_spectra()
spectra = sdss_corrected_spectra.reconstruct_spectra(data)
wavelengths = sdss_corrected_spectra.compute_wavelengths(data)

#----------------------------------------------------------------------
# Compute PCA, and NMF components
#  we'll save the results so that they can be re-used
@pickle_results('spec_decompositions.pkl')
def compute_PCA_NMF(n_components=5):
    spec_mean = spectra.mean(0)

    # PCA: use randomized PCA for speed
    pca = RandomizedPCA(n_components - 1)
    pca.fit(spectra)
    pca_comp = np.vstack([spec_mean,
                          pca.components_])

    # NMF requires all elements of the input to be greater than zero
    spectra[spectra < 0] = 0
    nmf = NMF(n_components)
    nmf.fit(spectra)
    nmf_comp = nmf.components_

    return pca_comp, nmf_comp

n_components = 5
decompositions = compute_PCA_NMF(n_components)

#----------------------------------------------------------------------
# Plot the results
fig = plt.figure(figsize=(10, 8))
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.05,
                    bottom=0.1, top=0.95, hspace=0.05)

titles = ['PCA components', 'NMF components']

for i, comp in enumerate(decompositions):
    for j in range(n_components):
        ax = fig.add_subplot(n_components, 3, 3 * j + 1 + i)

        ax.yaxis.set_major_formatter(plt.NullFormatter())
        ax.xaxis.set_major_locator(plt.MultipleLocator(1000))
        if j < n_components - 1:
            ax.xaxis.set_major_formatter(plt.NullFormatter())
        else:
            ax.set_xlabel(r'wavelength ${\rm (\AA)}$')

        ax.plot(wavelengths, comp[j], '-k', lw=1)

        # plot zero line
        xlim = [3000, 7999]
        ax.plot(xlim, [0, 0], '-', c='gray', lw=1)
        ax.set_xlim(xlim)

        if j == 0:
            ax.set_title(titles[i])

        if titles[i].startswith('PCA') or titles[i].startswith('ICA'):
            if j == 0:
                label = 'mean'
            else:
                label = 'component %i' % j
        else:
            label = 'component %i' % (j + 1)

        ax.text(0.03, 0.94, label, transform=ax.transAxes,
                ha='left', va='top')

        for l in ax.get_xticklines() + ax.get_yticklines():
            l.set_markersize(2)

        # adjust y limits
        ylim = plt.ylim()
        dy = 0.05 * (ylim[1] - ylim[0])
        ax.set_ylim(ylim[0] - dy, ylim[1] + 4 * dy)

plt.show()

```

## Manifold Learning##

Real-world data sets often have very nonlinear features (e.g. QSOs with broad lines) which are hard to describe compactly using linear eigenvectors. Manifold learning techniques search for a representation of these data within a lower dimensional space

</img>

**Local Linear Embedding:**

Step 1: define local geometry
- local neighborhoods determined from $k$ nearest neighbors.
- for each point calculate weights that reconstruct a point from its $k$ nearest
neighbors
>$
\begin{equation}
  \mathcal{E}_1(W) = \left|X - WX\right|^2
\end{equation}
$
- essentially this is finding the hyperplane that describes the local surface at each point within the data set.

Step 2: embed within a lower dimensional space
- set all $W_{ij}=0$ except when point $j$ is one of the $k$ nearest neighbors of point $i$.
- $W$ becomes very sparse for $k \ll N$.
- minimize
>$\begin{equation}
  \mathcal{E}_2(Y) = \left|Y - W Y\right|^2,
\end{equation}
$

with $W$ fixed to find an $N$ by $d$ matrix ($d$ is the new dimensionality)

Step 1 requires a nearest-neighbor search.

Step 2 requires an
eigenvalue decomposition of the matrix $C_W \equiv (I-W)^T(I-W)$,

LLE has been applied to data as diverse as galaxy spectra, stellar spectra, and photometric light curves.

```python
# Author: Jake VanderPlas
# License: BSD
import numpy as np
from matplotlib import pyplot as plt
from sklearn import manifold, neighbors

from astroML.datasets import sdss_corrected_spectra
from astroML.datasets import fetch_sdss_corrected_spectra
from astroML.plotting.tools import discretize_cmap
from astroML.decorators import pickle_results

#----------------------------------------------------------------------
# This function adjusts matplotlib settings for a uniform feel in the textbook.
# Note that with usetex=True, fonts are rendered with LaTeX.  This may
# result in an error if LaTeX is not installed on your system.  In that case,
# you can set usetex to False.
from astroML.plotting import setup_text_plots
setup_text_plots(fontsize=8, usetex=True)

#------------------------------------------------------------
# Set up color-map properties
clim = (1.5, 6.5)
cmap = discretize_cmap(plt.cm.jet, 5)
cdict = ['unknown', 'star', 'absorption galaxy',
         'galaxy', 'emission galaxy',
         'narrow-line QSO', 'broad-line QSO']
cticks = [2, 3, 4, 5, 6]
formatter = plt.FuncFormatter(lambda t, *args: cdict[int(np.round(t))])

#------------------------------------------------------------
# Fetch the data; PCA coefficients have been pre-computed
data = fetch_sdss_corrected_spectra()
coeffs_PCA = data['coeffs']
c_PCA = data['lineindex_cln']
spec = sdss_corrected_spectra.reconstruct_spectra(data)
color = data['lineindex_cln']

#------------------------------------------------------------
# Compute the LLE projection; save the results
@pickle_results("spec_LLE.pkl")
def compute_spec_LLE(n_neighbors=10, out_dim=3):
    # Compute the LLE projection
    LLE = manifold.LocallyLinearEmbedding(n_neighbors, out_dim,
                                          method='modified',
                                          eigen_solver='dense')
    Y_LLE = LLE.fit_transform(spec)
    print (" - finished LLE projection")

    # remove outliers for the plot
    BT = neighbors.BallTree(Y_LLE)
    dist, ind = BT.query(Y_LLE, n_neighbors)
    dist_to_n = dist[:, -1]
    dist_to_n -= dist_to_n.mean()
    std = np.std(dist_to_n)
    flag = (dist_to_n > 0.25 * std)
    print (" - removing %i outliers for plot" % flag.sum())

    return Y_LLE[~flag], color[~flag]

coeffs_LLE, c_LLE = compute_spec_LLE(10, 3)

#----------------------------------------------------------------------
# Plot the results:
for (c, coeffs, xlim) in zip([c_PCA, c_LLE],
                             [coeffs_PCA, coeffs_LLE],
                             [(-1.2, 1.0), (-0.01, 0.014)]):
    fig = plt.figure(figsize=(5, 3.75))
    fig.subplots_adjust(hspace=0.05, wspace=0.05)

    # axes for colorbar
    cax = plt.axes([0.525, 0.525, 0.02, 0.35])

    # Create scatter-plots
    scatter_kwargs = dict(s=4, lw=0, edgecolors='none', c=c, cmap=cmap)

    ax1 = plt.subplot(221)
    im1 = ax1.scatter(coeffs[:, 0], coeffs[:, 1], **scatter_kwargs)
    im1.set_clim(clim)
    ax1.set_ylabel('$c_2$')

    ax2 = plt.subplot(223)
    im2 = ax2.scatter(coeffs[:, 0], coeffs[:, 2], **scatter_kwargs)
    im2.set_clim(clim)
    ax2.set_xlabel('$c_1$')
    ax2.set_ylabel('$c_3$')

    ax3 = plt.subplot(224)
    im3 = ax3.scatter(coeffs[:, 1], coeffs[:, 2], **scatter_kwargs)
    im3.set_clim(clim)
    ax3.set_xlabel('$c_2$')

    fig.colorbar(im3, ax=ax3, cax=cax,
                 ticks=cticks,
                 format=formatter)

    ax1.xaxis.set_major_formatter(plt.NullFormatter())
    ax3.yaxis.set_major_formatter(plt.NullFormatter())

    ax1.set_xlim(xlim)
    ax2.set_xlim(xlim)

    for ax in (ax1, ax2, ax3):
        ax.xaxis.set_major_locator(plt.MaxNLocator(5))
        ax.yaxis.set_major_locator(plt.MaxNLocator(5))

plt.show()

```


# Tutorial of Implementing Discontinuous Galerkin Methods
***

This notebook by Xiaozhou Li is licensed under a Creative Commons Attribution 4.0 International License.
All code examples are also licensed under the [MIT license](http://opensource.org/licenses/MIT).
***

**Content**

- [x] mesh
- [x] basis
- [x] DG formulation
- [x] Runge-Kutta DG
- [x] Further notes
    - [x] couple with the accuracy enhance filtering
    - [x] implicit time integrator
***

## Introduction

This is a short tutorial for implementation Discontinuous Galerkin (DG) methods for the advection equation,
\begin{align}
    u_t + u_x & = 0,\quad x \in [0,1] \\
    u(x,0) & = u_0(x)
\end{align}
with periodic boundary conditions.
***

## The Step by Step Guide

### The first step
First of all, we need make a partition of the domain $[0,1]$,
$$0 = x_{\frac{1}{2}} < x_{\frac{3}{2}} < \cdots < x_{N+\frac{1}{2}},$$
where
- the element/cell $I_j = [x_{j-\frac{1}{2}}, x_{j+\frac{1}{2}}]$
- the middle point of the $j$th element, $x_j = \frac{1}{2}(x_{j-\frac{1}{2}} + x_{j+\frac{1}{2}})$
- the size of the $j$th element, $\Delta x_j = x_{j+\frac{1}{2}} - x_{j-\frac{1}{2}}$

To store the mesh, one way is to store the middle point $x_j$ and the size $\Delta x_j$

```python
import numpy as np
import numpy.polynomial.legendre as npleg
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import matplotlib.pyplot as plt
from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets
from IPython.display import clear_output, display
```

```python
def generate_mesh(left, right, N):
    # uniform mesh
    x = np.zeros(N+1)
    h = (right - left)/N
    for j in range(1,N+1):
        x[j] = (j - 0.5)*h

    dx = np.zeros(N+1) + h
    return  x, dx
x, dx = generate_mesh(0, 1, 10)
```

### The DG (finite element) space
Now, we need to define the finite element space
$$ V_h^k = \left\{v: v|_{I_j} \in \mathbb{P}^k(I_j),\, 1 \leq j \leq N\right\},$$
where $\mathbb{P}^k(I_j)$ denotes the set of polynomials of degree up to $k$ defined on the element $I_j$.

For $v \in V_h^k$,
- piecewise polynomials
- **discontinous** across the element interface (bi-values)

To implement the finite element space $V_h^k$, we just need to choose a basis of $\mathbb{P}^k$, for example,
- monomial basis, $1, x, x^2, \ldots, x^k$,
- Legendre polynomial on $[-1,1]$,
$$ P_n(x) = \frac{1}{2^n n!}\frac{d^n}{dx^n}\left((x^2-1)^n\right), \quad x \in [-1,1].$$
    - orthogonality
        $$\int_{-1}^1 P_\ell(x)P_m(x) = \frac{2}{2\ell+1}\delta_{\ell,m}$$
    - recursive relation
        $$ (n+1)P_{n+1}(x) = (2n+1)xP_n(x) - n P_{n-1}(x)$$
    - end points
        $$ P_\ell(\pm1) = (\pm)^\ell $$
    - first several polynomials:
        $$P_0(x) = 1,\,\, P_1(x) = x,\,\, P_2(x) = \frac{1}{2}(3x^2-1), P_3(x)= \frac{1}{2}(5x^2-3),\,\,\ldots$$

In this tutorial, we are going to use the Legendre polynomial to construct the basis of the DG space, for element $I_j$,
$$ \varphi_j^\ell(x) = P_\ell\left(\frac{2(x - x_j)}{\Delta x_j}\right),\quad x \in I_j$$

```python
def LegendreBasis(x, n):
    if (n == 0):
        return 1.
    elif (n == 1):
        return x
    elif (n == 2):
        return 1.5*x*x - 0.5
    elif (n == 3):
        return 0.5*x*(5*x*x - 3)
    else:
        raise Exception("not yet")
```

### The DG formulation

Now, we are ready to implement the DG formulation.  That is find the numerical solution $u_h \in V_h^k$, such that, on element $I_j$,
$$ \int_{I_j}(u_h)_t v_h \,dx - \int_{I_j}u_h(v_h)_x\,dx + \hat{u}_h(x_{j+\frac{1}{2}})v_h(x_{j+\frac{1}{2}}^{-}) - \hat{u}_h(x_{j-\frac{1}{2}})v_h(x_{j-\frac{1}{2}}^{+}) = 0,\quad\, \text{for}\,\forall v_h \in V_h^k,$$
where the numerical flux $\hat{u}_h(x_{j+\frac{1}{2}})$ is chosen as the upwind flux $u_h(x_{j+\frac{1}{2}}^{-})$.

There are three terms need to be taken care:
1. $\int_{I_j}(u_h)_t v_h \,dx$
2. $\int_{I_j}u_h(v_h)_x\,dx$
3. $\hat{u}_h(x_{j+\frac{1}{2}})v_h(x_{j+\frac{1}{2}}^{-}) - \hat{u}_h(x_{j-\frac{1}{2}})v_h(x_{j-\frac{1}{2}}^{+})$

To be continue, let us write the numerical solution $u_h$ with its modal form
$$ u_h(x, t) = \sum\limits_{\ell = 0}^k u_j^\ell(t)\varphi_j^\ell(x),\quad x \in I_j,$$
and the test function
$$ v_h = \varphi_j^0, \varphi_j^1, \ldots, \varphi_j^k.$$

***
**Term 1**

\begin{align}
    \int_{I_j} (u_h)_t v_h\,dx &= \int_{I_j} \sum\limits_{\ell=0}^k\dot{u}_j^\ell(t)\varphi_j^\ell(x)\varphi_j^m(x)\, dx\qquad m = 0,\ldots,k \\
                               &= \sum\limits_{\ell=0}^k\dot{u}_j^\ell(t)\int_{I_j}\varphi_j^\ell(x)\varphi_j^m(x)\, dx \\
                               &= \sum\limits_{\ell=0}^k\dot{u}_j^\ell(t)\left(\varphi_j^\ell(x)\varphi_j^m(x)\right)_j
\end{align}

Denote the unknowns
$$U_j(t) = \begin{bmatrix}u_j^0(t) \\ u_j^1(t) \\ \vdots \\ u_j^k(t)\end{bmatrix},$$
and the mass matrix
$$
    M_j = \begin{bmatrix}\left(\varphi_j^0, \varphi_j^0\right)_j & \cdots & \left(\varphi_j^k, \varphi_j^0\right) \\
    \vdots &\ddots & \vdots \\
    \left(\varphi_j^0, \varphi_j^k\right)_j &\cdots & \left(\varphi_j^k, \varphi_j^k\right)\end{bmatrix}.
$$

In fact, since we define the basis $\varphi_j^\ell$ based on the Legendre polynomial $P_\ell$ on the reference element $[-1,1]$, the entry of the mass matrix
\begin{align}
    \left(\varphi_j^\ell,\varphi_j^m\right) & = \int_{I_j}\varphi_j^\ell(x)\varphi_j^m(x)\, dx = \int_{I_j}P_\ell\left(\frac{2(x - x_j)}{\Delta x_j}\right)P_m\left(\frac{2(x - x_j)}{\Delta x_j}\right)\, dx \\
    & = \int_{-1}^{1}P_\ell(\xi)P_m(\xi)\frac{\Delta x_j}{2}\,d\xi \\
    & = \frac{\Delta x_j}{2}\frac{2}{2\ell+1}\delta_{\ell,m}.
\end{align}
This leads the mass matrix
$$
    M_j = \frac{\Delta x_j}{2}M = \frac{\Delta x_j}{2}\begin{bmatrix}2 & & &  \\
     & \frac{2}{3} &  & \\
    & &\ddots &   \\
    &  &  & \frac{2}{2k+1}\end{bmatrix}.
$$
That is
$$
\int_{I_j} (u_h)_t v_h\,dx = \frac{\Delta x_j}{2}M \dot{U}_j(t)
$$
*Note:* in fact, we only need the inverse of the mass matrix,
$$
M^{-1} = \begin{bmatrix}\frac{1}{2} & & &  \\
     & \frac{3}{2} &  & \\
    & &\ddots &   \\
    &  &  & \frac{2k+1}{2}\end{bmatrix}.
$$

```python
def generate_mass_matrix(k):
    M = np.diag([2/(2*i+1) for i in range(k+1)])
    M_inv = np.diag([(2*i+1)/2. for i in range(k+1)])
    return M_inv,  M
```

***
**Term 2**

\begin{align}
    \int_{I_j}u_h(v_h)_x\,dx &= \int_{I_j} \sum\limits_{\ell=0}^k u_j^\ell(t)\varphi_j^\ell(x)\left(\varphi_j^m(x)\right)_x\, dx\qquad m = 0,\ldots,k \\
                               &= \sum\limits_{\ell=0}^k u_j^\ell(t)\int_{I_j}\varphi_j^\ell(x)\dot{\varphi}_j^m(x)\, dx \\
                               &= \sum\limits_{\ell=0}^k u_j^\ell(t)\left(\varphi_j^\ell(x)\dot{\varphi}_j^m(x)\right)_j
\end{align}

Here, we denote the matrix
$$
    S_j = \begin{bmatrix}\left(\varphi_j^0, \dot{\varphi}_j^0\right)_j & \cdots & \left(\varphi_j^k, \dot{\varphi}_j^0\right) \\
    \vdots &\ddots & \vdots \\
    \left(\varphi_j^0, \dot{\varphi}_j^k\right)_j &\cdots & \left(\varphi_j^k, \dot{\varphi}_j^k\right)\end{bmatrix}.
$$
as the stiff matrix.

Similar as the mass matrix, we have
\begin{align}
    \left(\varphi_j^\ell,\dot{\varphi}_j^m\right) & = \int_{I_j}\varphi_j^\ell(x)\dot{\varphi}_j^m(x)\, dx = \int_{I_j}P_\ell\left(\frac{2(x - x_j)}{\Delta x_j}\right)\dot{P}_m\left(\frac{2(x - x_j)}{\Delta x_j}\right)\frac{2}{\Delta x_j}\, dx \\
    & = \int_{-1}^{1}P_\ell(\xi)\dot{P}_m(\xi)\,d\xi. \\
\end{align}
We note that the stiff matrix is not dependent on $j$, namely $S_j = S$. Usually, one can calculate it by using Gaussian quadrature.  However, we are using the Legendre polynomials, for $m > \ell$
$$
\int_{-1}^{1}P_\ell(x)\dot{P}_m(x)\,dx = P_\ell(x)P_m(x)|_{-1}^1 - \int_{-1}^{1}\dot{P}_\ell(x) P_m(x)\,dx = 1 - (-1)^{\ell+m}.
$$
This leads
$$
\int_{-1}^{1}P_\ell(\xi)\dot{P}_m(\xi)\,d\xi = \left\{\begin{array}{cc}0 & m\leq \ell \\ 1-(-1)^{\ell+m} & m > \ell \end{array}\right.
$$
That is
$$
\int_{I_j} u_h \dot{v}_h\,dx = S U_j(t)
$$

```python
def generate_stiff_matrix(k):
    S = np.zeros([k+1,k+1])
    for i in range(k+1):
        for j in range(i+1, k+1, 2):
            S[j,i] = 2
    return S
```

***
**Term 3**

Now, let us have a look at the final term, the flux term, which is always the most confussion part of the DG methods.
\begin{align}
 & \,\hat{u}_h(x_{j+\frac{1}{2}})v_h(x_{j+\frac{1}{2}}^{-}) - \hat{u}_h(x_{j-\frac{1}{2}})v_h(x_{j-\frac{1}{2}}^{+}) \qquad m = 0,\ldots, k \\
 = &\, \sum\limits_{\ell=0}^k u_j^\ell(t)\varphi_j^\ell(x_{j+1}^{-})\varphi_j^m(x_{j+1}^{-}) - \sum\limits_{\ell=0}^k u_{j-1}^\ell(t)\varphi_{j-1}^\ell(x_{j-1}^{-})\varphi_j^m(x_{j-1}^{+}) \\
 = &\, \sum\limits_{\ell=0}^k u_j^\ell(t) - \sum\limits_{\ell=0}^k u_{j-1}^\ell(t)(-1)^m
\end{align}

For convenience, let us define the flux matrice,
$$
F_r = \begin{bmatrix}1 & 1 & \cdots & 1 \\ 1 & 1 & \cdots & 1 \\ \vdots & \vdots & \ddots & \vdots \\ 1 & 1 & \cdots & 1\end{bmatrix},\qquad
F_l = \begin{bmatrix}1 & 1 & \cdots & 1 \\ -1 & -1 & \cdots & -1 \\ \vdots & \vdots & \ddots & \vdots \\ (-1)^k & (-1)^k & \cdots & (-1)^k\end{bmatrix}.
$$
That is
$$
\hat{u}_h(x_{j+\frac{1}{2}})v_h(x_{j+\frac{1}{2}}^{-}) - \hat{u}_h(x_{j-\frac{1}{2}})v_h(x_{j-\frac{1}{2}}^{+}) = F_r U_j(t) - F_l U_{j-1}(t).
$$

```python
def generate_flux_matrix(k):
    Fr = np.ones([k+1,k+1])
    Fl = np.ones([k+1,k+1])
    for i in range (1, k+1, 2):
        Fl[i,:] = -Fl[i,:]
    return Fr, Fl
```

***
**Semi-Discrete DG formulation**

Finally, by putting all three terms together, we have the semi-discrete DG formulation,
$$
\frac{\Delta x_j}{2}M \dot{U}_j(t) = S U_j(t) - Fr U_j(t) + F_l U_{j-1}(t), \qquad j = 1,\ldots, N.
$$
For the boundary flux $U_{0}$, it is decided by the boundary condition:
- for periodic boundary condition, $U_0(t) = U_N(t)$,
- for Dirichlet boundary condition, $U_0(t) = u_l(t)$.

### Time Discretization

**The Third Order TVD Runge-Kutta Method**

The most popular scheme used for semi-discrete DG scheme is the following third order Runge-Kutta method for solving
$$ u_t = \text{Res}(u, t),$$
where $\text{Res}(u,t)$ is the spatial discretization operator
\begin{align}
    u^{(1)} & = u^n + \Delta t \text{Res}(u^n, t^n) \\
    u^{(2)} & = \frac{3}{4}u^n + \frac{1}{4}u^{(1)} + \frac{1}{4}\Delta t \text{Res}(u^{(1)}, t^n + \Delta t) \\
    u^{n+1} & = \frac{1}{3}u^n +\frac{2}{3}u^{(2)} + \frac{2}{3}\Delta t \text{Res}(u^{(2)}, t^n+\frac{1}{2}\Delta t).
\end{align}

***
Now, we couple the DG spatial discretization with the third ordre Runge-Kutta method (or other explicit time discretizations).  When coupled with explicit time discretization, we can solve the entire system element by element (or say block by block).

First of all, let us introduce the residue notation to simplify the writing
$$
    \dot{U}_j(t) = \text{Res}(U_j(t), U_{j-1}(t)), \qquad j = 1,\ldots, N.
$$
where $\text{Res}(U_j(t), U_{j-1}(t)) = \frac{2}{\Delta x_j}M^{-1}\left(S U_j(t) - Fr U_j(t) + F_l U_{j-1}(t)\right)$.

```python
def Residue(u, dx, N, M_inv, S, Fr, Fl):
    Res = np.zeros(np.shape(u))
    k = np.size(u, 1)-1

    # periodic boundary
    u[0,:] = u[N,:]

    for j in range(1, N+1):
        Res[j,:] = 2/dx[j]*np.dot(M_inv, np.dot(S,u[j,:]) - np.dot(Fr, u[j,:]) + np.dot(Fl, u[j-1,:]))

    return Res
```

***
For the first stage in the above Runge-Kutta method (or say the forward Euler method),
- for $j = 1$ to $N$:
$$
    U_j^{(1)} = U_j^n + \Delta t \text{Res}(U_j^n, U_{j-1}^{n})
$$
    - where $U_j^n = U_j(t^n)$
    - note for periodic boundary conditions, $U_0^{n} = U_N^n$.

For the second stage,
- for $j = 1$ to $N$:
$$
    U_j^{(2)} = \frac{3}{4}U_j^n + \frac{1}{4}U_j^{(1)} + \frac{1}{4}\Delta t \text{Res}(U_j^{(1)}, U_{j-1}^{(1)}).
$$

For the third stage,
- for $j = 1$ to $N$:
$$
    U_j^{n+1} = \frac{1}{3}U_j^n + \frac{2}{3}U_j^{(2)} + \frac{2}{3}\Delta t \text{Res}(U_j^{(2)}, U_{j-1}^{(2)}).
$$

*Note:*
- for some equation, $\text{Res}$ may also depend on time $t$.

```python
def Runge_Kutta(t, dt, u, dx, N, M_inv, S, Fr, Fl):
    # The thrid order Runge-Kutta method

    # save the u^n data
    u_n = np.copy(u)

    # first stage
    Res = Residue(u, dx, N, M_inv, S, Fr, Fl)
    u = u_n + dt*Res

    # second stage
    Res = Residue(u, dx, N, M_inv, S, Fr, Fl)
    u = 0.75*u_n + 0.25*u + 0.25*dt*Res

    # third stage
    Res = Residue(u, dx, N, M_inv, S, Fr, Fl)
    u = (u_n + 2*u + 2*dt*Res)/3.

    return u
```

```python
def DG_time(u, x, dx, cfl, time, N, M_inv, S, Fr, Fl):
    dt = cfl*np.max(dx)
    t =  dt
    while (t < time):
        u = Runge_Kutta(t, dt, u, dx, N, M_inv, S, Fr, Fl)
        t = t + dt
    dt = time + dt - t
    u = Runge_Kutta(time, dt, u, dx, N, M_inv, S, Fr, Fl)
    return u
```

***
**Initial Time**

From the above section, we know that how to march in time through the Runge-Kutta method.  However, we still need to know the initial data $U_j^0 = U_j(0)$.  In this tutorial, we will do a simple $L^2$ projection.  For element $I_j$, we have
\begin{align}
\int_{I_j} u_h v_h \,dx & = \int_{I_j} u_0 v_h\, dx \\
\int_{I_j}\sum\limits_{\ell=0}^k u_j^\ell(0)\varphi_j^\ell(x) \varphi_j^m(x)\, dx &= \int_{I_j}u_0(x) \varphi_j^m(x)\, dx \\
\sum\limits_{\ell=0}^k u_j^\ell(0)\frac{\Delta x_j}{2}\int_{-1}^1 P_\ell(\xi) P_m(\xi)\,d\xi & = \frac{\Delta x_j}{2}\int_{-1}^1 u_0\left(x_j + \frac{\Delta x_j}{2}\xi\right)P_m(\xi)\,d\xi \\
\sum\limits_{\ell=0}^k u_j^\ell(0)\int_{-1}^1 P_\ell(\xi) P_m(\xi)\,d\xi & = \int_{-1}^1 u_0\left(x_j + \frac{\Delta x_j}{2}\xi\right)P_m(\xi)\,d\xi
\end{align}
That is,
$$
    U_j(0) = M^{-1}F_j,\qquad j = 1,\ldots,N
$$
where
$$
    F_j =  \begin{bmatrix}\int_{-1}^1 u_0\left(x_j + \frac{\Delta x_j}{2}\xi\right)P_0(\xi)\,d\xi \\ \int_{-1}^1 u_0\left(x_j + \frac{\Delta x_j}{2}\xi\right)P_1(\xi)\,d\xi \\ \vdots \\ \int_{-1}^1 u_0\left(x_j + \frac{\Delta x_j}{2}\xi\right)P_k(\xi)\,d\xi\end{bmatrix},\qquad j = 1,\ldots,N
$$
can be calculated by using numerical quadrature, for example, the Gaussian quadrature.

```python
def initial_projection(fun0, k, N, x, dx, M_inv):
    # Gaussian quadrature
    #Gpn = int(np.ceil(0.5*(num+order)))
    Gpn = 6
    xg, wg = npleg.leggauss(Gpn)
    u =  np.zeros([N+1,k+1])

    F = np.zeros(k+1)
    for j in range(1, N+1):
        for l in range(k+1):
            F[l] = 0.
            for m in range(Gpn):
                F[l] += wg[m]*fun0(x[j] + 0.5*dx[j]*xg[m])*LegendreBasis(xg[m], l)
        u[j,:] = np.dot(M_inv, F)
    return u
```

***
### The DG solver for Advection Equation.

Now, let us put every thing together,
1. we set a function to genearte the mesh and all the matrice.

```python
def DG_init(k, N, left, right):
    # generate the mesh
    x,  dx = generate_mesh(left, right, N)

    # generate all the matrice
    M_inv, _ = generate_mass_matrix(k)
    S = generate_stiff_matrix(k)
    Fr, Fl = generate_flux_matrix(k)

    return x, dx, M_inv, S, Fr, Fl
```

2. we define the initial function
$$
    u_0(x) = \sin(2\pi x),
$$
and the exact solution is just
$$
    u(x,t) = u_0(x-t).
$$

```python
def fun0(x):
    return np.sin(2.*np.pi*x)
```

3. we set the DG solver for the advection equation $u_t + u_x = 0$.

```python
def DG_solver(k, N, left, right, fun0, time):
    # get the mesh, matrice
    x, dx, M_inv, S, Fr, Fl = DG_init(k, N, left, right)

    # set the cfl number
    cfl = 0.1**k

    # initial condition
    u = initial_projection(fun0, k, N, x, dx, M_inv)

    # time forward
    u = DG_time(u, x, dx, cfl, time, N, M_inv, S, Fr, Fl)

    return u, x, dx
```

```python
u, x, dx = DG_solver(1, 20, 0, 1, fun0, 1)
```

***
### Verification

Now, we algready have a working DG code.  However, for mathematicians, we want to make sure the code does the expected job.

* **check error:** $\|u_h - u\|$

```python
def DG_error(u, x, dx, fun0, time):
    # set Gaussian quadrature to calculate the L2 norm
    Gpn = 6
    xg, wg = npleg.leggauss(Gpn)
    err2 = 0.
    erri = 0.

    N = np.size(u, 0) - 1
    k = np.size(u, 1) - 1

    ## plotting
    plot_x = np.empty(N*Gpn)
    plot_y = np.empty(N*Gpn)

    for j in range(1,N+1):
        for m in range(Gpn):
            uh = 0.
            xj = x[j] + dx[j]*xg[m]/2
            for l in range(k+1):
                uh += u[j,l]*LegendreBasis(xg[m], l)
            err = np.abs(fun0(xj-time) - uh)
            err2 += 0.5*dx[j]*wg[m]*err**2
            erri = max(erri, err)

            plot_x[(j-1)*Gpn+m] = xj
            # plot_y[(j-1)*Gpn+m] = uh
            plot_y[(j-1)*Gpn+m] = err
    err2 = np.sqrt(err2/1.)
    plt.plot(plot_x, plot_y, linewidth=2.0)
    print ("N = ", N, "  k = ", k)
    print ("L2 error:   %7.2e" % err2)
    print ("Linf error: %7.2e" % erri)
    plt.yscale('log')
    plt.xlim(0, 1)
    #plt.ylim(1.e-6,1.e0)
    plt.xlabel('x',fontsize=18)
    plt.ylabel('|error|',fontsize=18)
    plt.show()
```

Now, let us have a look at the error of our numerical solution in the $L^2$ and $L^\infty$ norm, also we plot the point-wise error.

```python
DG_error(u, x, dx, fun0, 1)
```

From the above error and error, we can see that our numerical solution is a good approximation to the exact solution (well, in some sense).  However, it is not sufficient to show that the implementation is correct.  To further verify the code, we need to check the accuracy order.  That is, how the error changes along when the element number $N$ (the polynomial degree $k$).  In theory, the DG solution will have the optimal convergence rate of order $k+1$.
* **check the accuracy order:** $\|u_h - u\| = \mathcal{O}(h^{k+1})$

First, in order to output a result in a good table form, we define a print table function to control the output format of of the results.
- one can also design a function to output results in a form that can be used directly in the Latex.

```python
def print_OrderTable(number_coarse, erri, err2):
    print ("  N  Linf norm order  L2 norm  order")
    for i in range(np.size(erri)):
        N = number_coarse*2**i
        if (i == 0):
            print ("%3d  %7.2e   --   %7.2e   --" % \
                   (N, erri[i], err2[i]))
        else:
            print ("%3d  %7.2e  %4.2f  %7.2e  %4.2f" % \
                       (N, \
                        erri[i], np.log(erri[i-1]/erri[i])/np.log(2),\
                        err2[i], np.log(err2[i-1]/err2[i])/np.log(2)))
```

Now, let us check the accuracy order
- here, we fix the polynomial $k$, and change the number of element $N$.  We double $N$ in each level, then the error should be just $\frac{1}{2^{k+1}}$ (or taking $\log_2$ gives the $k+1$) of the previous level.

```python
def DG_order(number_coarse, degree):
    Gpn = 6
    xg, wg = npleg.leggauss(Gpn)
    err2 = np.zeros(4)
    erri = np.zeros(4)
    style = ['b.','g-.','r--','c-']

    for i in range(4):
        N = number_coarse*2**(i)

        time = 1.
        left = 0.
        right = 1.

        u, x, dx = DG_solver(degree, N, left, right, fun0, time)

        ## plotting
        plot_x = np.empty(N*Gpn)
        plot_y = np.empty(N*Gpn)

        for j in range(1,N+1):
            for m in range(Gpn):
                uh = 0.
                xj = x[j] + dx[j]*xg[m]/2
                for l in range(degree+1):
                    uh += u[j,l]*LegendreBasis(xg[m], l)
                err = np.abs(fun0(xj-time) - uh)
                err2[i] += 0.5*dx[j]*wg[m]*err**2
                erri[i] = max(erri[i], err)

                plot_x[(j-1)*Gpn+m] = xj
                plot_y[(j-1)*Gpn+m] = uh
                plot_y[(j-1)*Gpn+m] = err
        err2[i] = np.sqrt(err2[i]/1.)
        plt.plot(plot_x, plot_y, style[i],linewidth=2.0, label="N = " + str(N))
    print_OrderTable(number_coarse, erri, err2)
    plt.yscale('log')
    ax = plt.legend(fontsize=14)
    ax.draw_frame(False)
    plt.xlim(0, 1)
    #plt.ylim(1.e-6,1.e0)
    plt.xlabel('x',fontsize=18)
    plt.ylabel('|error|',fontsize=18)
    plt.show()
DG_order(20, 2)
```

**Some notes:**
- be aware that this code may take a while to run (depends on the mesh size, cfl number, final time, degree, etc.), one can use python build-in function, such as tqdm to visualize the progress.
- in this tutorial, the data array, such as $\text{x[j]}$ and $\text{u[j,:]}$,  the index $0$ is not the part of the final solution

This is the end of the common part of implementation of the DG method for advection equation.
***

***
## Further Notes

***
### Couple with Accuracy Enhance Filtering

From the above tutorial, we can see that the DG solution has accuracy order of $k+1$ in the $L^2$ norm, that is,
$$
\| u - u_h\|_0 \leq C h^{k+1}.
$$
This is also the optimal convergence rate for using approximation polynomial of degree $k$ in the $L^2$ norm.

In addition, one can use an accuracy enhace filtering (or, so-called Smoothness-Increasing Accuracy-Conserving (SIAC) filtering) technique.  By applying the filter technique, one can reconstruct the DG solution $u_h$ into a new filtered solution $u_h^*$ and improve the accuracy order from $k+1$ to $2k+1$,
$$    \|u - u_h^*\|_0 \leq C h^{2k+1}.  $$
For example, one can refer the following plot to see the improvment aftering applying the filtering technique.  Both the accuracy and the smoothness of the DG solution are greately improved.

***
**The Jupyter-notebook of the Filtering**

For the details of the above accuracy enhance technique, one can refer to the linked tutorial [nbviewer link](https://nbviewer.jupyter.org/github/xiaozhouli/Jupyter/blob/master/Tutorial_of_Filtering/SIAC_Filtering.ipynb).
To run the filtering tutorial, one can use the data (Fortran, quadruple-precision) provided within the tutorial.  Or one can also use the DG solution in this tutorial by using the follow routine to save the DG data (python, double-precision).  However, we note that the python version data has larger round-off error compare to the Fortran version data.

```python
def Save_DG(EleNum, degree, u):
    filename = './DGsolution_k'+str(degree)+'_N'+str(EleNum)+'.dat'
    np.savetxt(filename, u[1:,:].flatten())
```

***
### Implicit Integrator

Of course, for the advection equation, one donot need (or say may not) to use the implicit time integrator.  The purpose of this section is to simply demonstrate how to implement the implicit integrator (or how to assemble the global matrix).  For this purpose, we simply consider the backward Euler method for the time discretiztion,
$$
    u^{n+1}  = u^n + \Delta t \text{Res}(u^{n+1}, t^{n+1}).
$$

***
**Semi-Discrete DG formulation**

First, let us write down the semi-discrete DG formular,
$$
\frac{\Delta x_j}{2}M \dot{U}_j(t) = S U_j(t) - Fr U_j(t) + F_l U_{j-1}(t), \qquad j = 1,\ldots, N.
$$
For the boundary flux $U_{0}$, it is decided by the boundary condition:
- for periodic boundary condition, $U_0(t) = U_N(t)$,
- for Dirichlet boundary condition, $U_0(t) = u_l(t)$.

***
**Fully-Discrete formulation**

Now, we couple the DG spatial discretization with the backward Euler method
$$
    U^{n+1}_j(t) = U_j^n + 2\frac{\Delta t}{\Delta x_j}M^{-1}\left(S U^{n+1}_j(t) - Fr U^{n+1}_j(t) + F_l U^{n+1}_{j-1}(t)\right).
$$
Unlike coupling with explicit time discretization, the above system can not be solved element by element (or say block by block), one has to solve a global system.  Therefore, we need to rewrite the unknowns from
$$U_j(t) = \begin{bmatrix}u_j^0(t) \\ u_j^1(t) \\ \vdots \\ u_j^k(t)\end{bmatrix},\qquad j = 1,\ldots, N,$$
as
$$ U(t) = \begin{bmatrix}u_1^0(t) \\ u_1^1(t) \\ \vdots \\ u_1^k(t) \\ \vdots \\ u_N^0(t) \\ u_N^1(t) \\ \vdots \\ u_N^k(t)\end{bmatrix}. $$

***
**Global Matrix**
Then, we need to write the local formular
$$
    2\frac{\Delta t}{\Delta x_j}M^{-1}\left(S U_j(t) - Fr U_j(t) + F_l U_{j-1}(t)\right)
$$
as global matrix form $\text{Res}(U(t)) = AU(t)$, where the matrix
$$
    A = 2\Delta t\begin{bmatrix} \Delta x_1(S - F_r) &  &&& \Delta x_1 F_l\\
                                 \Delta x_2 F_l & \Delta x_2(S - F_r) & \\
                                                & \Delta x_3 F_l & \Delta x_3(S - F_r) & \\
                                                &                & \ddots & \ddots &  \\
                                                &                &        & \Delta x_N F_l & \Delta x_N(S - F_r)\end{bmatrix},
$$
or assume the uniform mesh $\Delta x = \Delta x_j$,
$$
    A = 2\frac{\Delta t}{\Delta x}\begin{bmatrix} S - F_r &  &&& F_l\\
                                 F_l & S - F_r & \\
                                                & F_l & S - F_r & \\
                                                &                & \ddots & \ddots &  \\
                                                &                &        & F_l & S - F_r\end{bmatrix}.
$$

**Note:**
- In this example, oe can see that the global matrix $A$ is independent with the unknown $U$.  Therefore, in practice, there are many methods to efficiently solve the linear system.

```python
def Residue_global(dx, N, M_inv, S, Fr, Fl):
    k = np.size(S, 1)-1
    A = np.zeros([(N+1)*(k+1), (N+1)*(k+1)])

    for j in range(1, N+1):
        A[j*(k+1):(j+1)*(k+1), j*(k+1):(j+1)*(k+1)] = 1./dx[j]*np.dot(M_inv, S - Fr)
        if j > 1:
            A[j*(k+1):(j+1)*(k+1), (j-1)*(k+1):j*(k+1)] = 1./dx[j]*np.dot(M_inv, Fl)
        else:
            A[j*(k+1):(j+1)*(k+1), N*(k+1):(N+1)*(k+1)] = 1./dx[j]*np.dot(M_inv, Fl)
    return A
```

***
Now, we solve the ODEs $\dot{U}(t) = A U(t)$ by the backward Euler method.

```python
def Implicit_Euler(t, dt, u, dx, N, A):
    # The thrid order Runge-Kutta method

    # save the u^n data
    #u_n = np.copy(u)

    u = np.linalg.solve(np.eye(np.size(u))-2*dt*A, u)

    return u
```

```python
def DG_time_imp(u, x, dx, cfl, time, N, A):
    dt = cfl*np.max(dx)
    t =  dt
    while (t < time):
        u = Implicit_Euler(t, dt, u, dx, N, A)
        t = t + dt
    dt = time + dt - t
    u = Implicit_Euler(time, dt, u, dx, N, A)
    return u
```

```python
def DG_solver(k, N, left, right, fun0, time):
    # get the mesh, matrice
    x, dx, M_inv, S, Fr, Fl = DG_init(k, N, left, right)

    # set the cfl number
    cfl = 0.001

    # initial condition
    u = initial_projection(fun0, k, N, x, dx, M_inv)
    # flatten the 2d arrary to 1d
    u = u.flatten()

    # global matrix
    A = Residue_global(dx, N, M_inv, S, Fr, Fl)

    # time forward
    u = DG_time_imp(u, x, dx, cfl, time, N, A)

    # let reshape the unknown back to 2d array, so we can use the verification functions
    u = np.reshape(u, [N+1,k+1])
    return u, x, dx
```

```python
u, x, dx = DG_solver(1, 20, 0, 1, fun0, 1)
```

```python
DG_error(u, x, dx, fun0, 1)
```