卒研

```python
import numpy as np
import math as ma
import sympy as sym

# import scipy as sp
# from scipy.integrate import solve_ivp
# from scipy.integrate import odeint
# import matplotlib.pyplot as plt
# from numpy.typing import _128Bit, _16Bit

"""パラメータの定義"""
LAMBDA,CONTACT_E,TOTAL,CONTACT_I,CONTACT_S1,ALPHA,MU,RATE_W,GAMMA ,THETA1,THETA2,SPAN1,SPAN2,SPAN12,S,E,I,R,S1 = sym.symbols(
    "LAMBDA,CONTACT_E,TOTAL,CONTACT_I,CONTACT_S1,ALPHA,MU,RATE_W,GAMMA ,THETA1,THETA2,SPAN1,SPAN2,SPAN12,S,E,I,R,S1"
)

#S,E,I,R,S1 =sym.symbols("S,E,I,R,S1")
"""
LAMBDA = 16600 # per day
#CONTACT = 0.03
CONTACT_E  = 0.08
CONTACT_I  = 0.12
CONTACT_S1  = 0.5 #(1/2)*(CONTACT_E + CONTACT_I )*(1-(85/100))
THETA1 = 4.45*10**(-5)
THETA2 = 4.88*10**(-5)
#SEVERE = 0.016
MU = 2.92*10**(-5)
ALPHA = 1/7
RATE_W = 0.01 # per day
GAMMA = 1/10 # per day
SPAN1 = 1/14
SPAN2 = 1/10
SPAN12 = 1/11
TOTAL = 8800000
"""

# 方程式の定義
eq1 = LAMBDA - (CONTACT_E*S*E/TOTAL+ CONTACT_I*S*I/TOTAL) - S*(SPAN1*THETA1+MU)
eq2 =  (CONTACT_E*S*E/TOTAL+ CONTACT_I*S*I/TOTAL+CONTACT_S1*S1*I/TOTAL+CONTACT_S1*S1*E/TOTAL) - E*(MU+ALPHA)
eq3 =  ALPHA*E - I*(RATE_W + MU + GAMMA)
eq4 =  S1*SPAN2*THETA2*SPAN12 + GAMMA*I - R*MU
eq5 =  S*SPAN1*THETA1- S1*(SPAN2*THETA2*SPAN12 + MU)-(CONTACT_S1*S1*I/TOTAL)-(CONTACT_S1*S1*E/TOTAL)
```

```python
eq1
```

解を探す

```python
# その１sym.solveの利用
Sol1 = sym.solve([eq1,eq2,eq3,eq4,eq5],[S,E,I,R,S1])
print(Sol1)
```

```python
# その2 scipyの利用
"""
X[0] = S
X[1] = E
X[2] = I
X[3] = R
X[4] = S1
"""
"""
def epidemic_model(X):
    return [LAMBDA - (CONTACT_E*X[0]*X[1]/TOTAL+ CONTACT_I*X[0]*X[2]/TOTAL) - X[0]*(SPAN1*THETA1+MU),
            (CONTACT_E*X[0]*X[1]/TOTAL+ CONTACT_I*X[0]*X[2]/TOTAL+CONTACT_S1*X[0]*X[2]/TOTAL+CONTACT_S1*X[4]*X[1]/TOTAL) - X[1]*(MU+ALPHA),
            ALPHA*X[1] - X[2]*(RATE_W + MU + GAMMA),
            X[4]*SPAN2*THETA2*SPAN12 + GAMMA*X[2] - X[3]*MU,
            X[0]*SPAN1*THETA1 - X[4]*(SPAN2*THETA2*SPAN12 + MU)-(CONTACT_S1*X[4]*X[2]/TOTAL)-(CONTACT_S1*X[4]*X[1]/TOTAL)]
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="hybr")
result = sp.optimize.root( epidemic_model,  [3500000,2368,1856,5259708,36068], method="lm")
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="broyden1")
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="krylov")
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="anderson")
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="excitingmixing")
#result = sp.optimize.root( epidemic_model, [3500000,2368,1856,5259708,36068], method="diagbroyden")
print(result)
"""
```


```python
import numpy as np
import sympy as sy
sy.init_printing()

q, p, gamma, sigma, h, kBT = sy.symbols("q, p, gamma, sigma, h, k_{B}T")
U = q**4
rho_conf = sy.exp(-U)
rho_vel = sy.exp(-p**2)
rho = rho_conf * rho_vel
sigma = sy.sqrt(2 * gamma * kBT)

def commutator(X, Y):
    """Return the commutator [X,Y] = XY - YX"""
    return lambda rho : X(Y(rho)) - Y(X(rho))

def commutator_3(X, Y, Z):
    """[X, Y, Z] = [X, [Y, Z]]"""
    return lambda rho : commutator(X, commutator(Y, Z))(rho)

def BCH_series(A, B):
    """First couple terms of BCH expansion"""
    return lambda rho : A(rho) + B(rho) + 0.5 * commutator(A, B)(rho) + (1.0/12) * (commutator(A, commutator(A, B))(rho) - commutator(B, commutator(A, B))(rho))

def first_term_BCH_series(A, B, C):
    """"""
    return lambda rho : A(rho) + B(rho) + C(rho) + (h/2) * (commutator(A,B)(rho) + commutator(A, C)(rho) + commutator(B, C)(rho))

def BCH_series_symmetric_scheme(X, Y, Z):
    """Symmetric schemes of the form XYZYX.
    X + Y + Z - h^2 (1/12 ([Z, Z, Y] + [Y+Z, Y+Z, X]) - 1/24 ([Y, Y, Z] + X, X, Y+Z)))

    """
    Y_Z = lambda rho : Y(rho) + Z(rho)
    L_0 = lambda rho : X(rho) + Y(rho) + Z(rho)
    L_2_a = lambda rho : (1.0 / 12) * (commutator_3(Z, Z, Y)(rho) +
                                     commutator_3(Y_Z, Y_Z, X)(rho))
    L_2_b = lambda rho : (- 1.0 / 24) * (commutator_3(Y, Y, Z)(rho) + commutator_3(X, X, Y_Z)(rho))
    L_2 = lambda rho : L_2_a(rho) + L_2_b(rho)
    return lambda rho : L_0(rho) + h**2 * L_2(rho)

# the next three definitions are from page 288 of Leimkuhler and Matthews, valid in 1D
def R_operator(rho):
    """Kolmogorov operator of R step"""
    return - p * sy.diff(rho, q)

A = R_operator

def V_operator(rho):
    """Kolmogorov operator of V step"""
    return sy.diff(U, q) * sy.diff(rho, p)

B = V_operator

def O_operator(rho):
    """Kolmogorov operator of O step"""
    return gamma * (rho + p * sy.diff(rho, p)) + (sigma**2 / 2) * sy.diff(rho, p, 2)

O = O_operator
```

```python
sy.simplify(BCH_series_symmetric_scheme(B, A, O)(rho))
```

```python
sy.simplify(commutator(B, O)(rho)), sy.simplify(commutator(A, B)(rho)), sy.simplify(commutator(A, O)(rho))
```

```python
sy.simplify(commutator(O, B)(rho)), sy.simplify(commutator(B, A)(rho)), sy.simplify(commutator(O, A)(rho))
```

```python
bch_expr = sy.simplify(first_term_BCH_series(A, B, O)(rho))
```

```python
bch_fxn = sy.lambdify((gamma, h, p, q, kBT), bch_expr)
```

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

ps,qs,cs = [], [], []
for i in range(10000):
    p,q = np.random.randn(2)
    c = bch_fxn(0.5, 0.1, p, q, 1.0)
    ps.append(p)
    qs.append(q)
    cs.append(c)
plt.scatter(ps, qs, c=cs, linewidths=0)
plt.colorbar()
```

```python

```

notes/Plotting BCH expansion for quartic system.ipynb


# ***Introduction to Radar Using Python and MATLAB***
## Andy Harrison - Copyright (C) 2019 Artech House
<br/>

# Hertzian Dipole
***

Referring to Section 4.1, the electric field radiated by a Hertzian dipole is written as (Equation 4.8)

\begin{align}
\vec{E} = 2\eta\,I_o\, \Delta{z} \, \frac{e^{-jkr}}{4 \pi r}\,\cos{\theta}\left[\frac{1}{r} - \frac{j}{kr^2} \right]\,\hat{r} + j\eta k\, I_o\, \Delta{z}\,\frac{e^{-jkr}}{4 \pi r}\,\sin{\theta}\,\left[1+\frac{1}{jkr} - \frac{1}{(kr)^2} \right]\hat{\theta}\hspace{0.5in} \text{(V/m)}
\end{align}
***

Begin by getting the library path

```python
import lib_path
```

Import the routines for the Hertzian dipole

```python
from Libs.radar_range.hertzian_dipole import *
```

For plotting the antenna patterns, the angle will be set from 0 to 2 $\pi$ using the `linspace` routine from `scipy`

```python
from numpy import linspace
```

```python
theta = linspace(0, 2.0 * pi, 256)
```

Set the relative permittivity, relative permeability, frequency (Hz), current (A), length (m), and distance (m)

```python
relative_permittivity = 1.0

relative_permeability = 1.0

frequency = 1e6

current = 1

length = 0.1

r = 1e6
```

Set up the keyword args

```python
kwargs = {'relative_permittivity': relative_permittivity,

          'relative_permeability': relative_permeability,

          'frequency': frequency,

          'current': current,

          'length': length,

          'r': r,

          'theta': theta}
```

Calculate the electric field (V/m)

```python
electric_field = electric_field(**kwargs)
```

Import the `matplotlib` routines

```python
from matplotlib import pyplot as plt
```

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.polar(theta, abs(electric_field))

# Set the plot title and labels

plt.title('Hertzian Dipole Electric Field (V/m)', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```

The magnetic field radiated by a Hertzian dipole is written as (Equation 4.6)

\begin{equation}
    \vec{H} =  I_o \,\Delta z \, \sin{\theta}\, \frac{e^{-jkr}}{4 \pi r} \left[ jk + \frac{1}{r} \right] \, \hat{\phi}\hspace{0.5in} \text{(A/m)}
\end{equation}

Keyword args are slightly different for the magnetic field

```python
kwargsH = {'frequency': frequency,

           'current': current,

           'length': length,

           'r': r,

           'theta': theta}
```

Calculate the magnetic field (A/m)

```python
magnetic_field = magnetic_field(**kwargsH)
```

`matplotlib` is imported so just create the plot

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

 # Display the results

plt.polar(theta, abs(magnetic_field))

# Set the plot title and labels

plt.title('Hertzian Dipole Magnetic Field (A/m)', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```

The power density radiated by the Hertzian dipole is written as (Equation 4.12)

\begin{equation}\label{eq:hertzian_poynting_vector2}
    \vec{P}_d = \frac{\eta}{2}\, \Bigg[ \frac{k\, I_o\, \Delta{z}}{4\pi r} \sin{\theta}\Bigg]^2\, \hat{r} \hspace{0.5in} \text{(W/m\textsuperscript{2})}
\end{equation}

Calculate the power density (Watts/m^2)

```python
power_density = power_density(**kwargs)
```

Display the results using `matplotlib`

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.polar(theta, power_density)

# Set the plot title and labels

plt.title('Hertzian Dipole Power Density (W/m$^2$)', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```

Set up the keyword args

```python
kwargs = {'relative_permittivity': relative_permittivity,

          'relative_permeability': relative_permeability,

          'frequency': frequency,

          'current': current,

          'length': length,

          'theta': theta}
```

The radiation intensity is written as (Equation 4.17)

\begin{equation}
    U(\theta,\phi) = \frac{\eta}{2}\,\left[ \frac{k\,I_o\,\Delta{z}}{4\pi}\,\sin{\theta}\right]^2  \hspace{0.5in} \text{(W/sr)}
\end{equation}

Calculate the radiation intensity (W/sr)

```python
radiation_intensity = radiation_intensity(**kwargs)
```

Display the results using `matplotlib`

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.polar(theta, radiation_intensity, '')

# Set the plot title and labels

plt.title('Hertzian Dipole Radiation Intensity', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```

The directivity of a Hertzian dipole is written as (Equation 4.18)

\begin{equation}
    D(\theta,\phi) = \frac{4\pi\,U(\theta,\phi)}{P_{rad}} = \frac{3}{2}\,\sin^2{\theta}
\end{equation}

Calculate the directivity

```python
directivity = directivity(**kwargs)
```

Display the results using `matplotlib`

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.polar(theta, directivity, '')

# Set the plot title and labels

plt.title('Hertzian Dipole Directivity', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```

The total power radiated by the Hertzian dipole is written as (Equation 4.13)

\begin{equation}\label{eq:hertzian_power_density}
    P_{rad} = \int\limits_{0}^{\pi}\int\limits_{0}^{2\pi} \vec{P}_d\cdot r^2 \sin{\theta}\,d\theta\,d\phi\,\hat{r}\hspace{0.5in} \text{(W)}
\end{equation}

Calculate and display the total radiated power (W)

Set up the keyword args

```python
kwargs = {'relative_permittivity': relative_permittivity,

          'relative_permeability': relative_permeability,

          'frequency': frequency,

          'current': current,

          'length': length}
```

```python
total_radiated_power = total_radiated_power(**kwargs)

print('{:.3e}'.format(total_radiated_power))
```

    4.390e-05


# Chapter 4
`Original content created by Cam Davidson-Pilon`

`Ported to Python 3 and PyMC3 by Max Margenot (@clean_utensils) and Thomas Wiecki (@twiecki) at Quantopian (@quantopian)`

______

## The greatest theorem never told

This chapter focuses on an idea that is always bouncing around our minds, but is rarely made explicit outside books devoted to statistics. In fact, we've been using this simple idea in every example thus far.

```python
!python
```

    Python 3.5.2 |Anaconda custom (x86_64)| (default, Jul  2 2016, 17:52:12)
    [GCC 4.2.1 Compatible Apple LLVM 4.2 (clang-425.0.28)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> ^C
    KeyboardInterrupt
    >>>
    >>>

### The Law of Large Numbers

Let $Z_i$ be $N$ independent samples from some probability distribution. According to *the Law of Large numbers*,  so long as the expected value $E[Z]$ is finite, the following holds,

$$\frac{1}{N} \sum_{i=1}^N Z_i \rightarrow E[ Z ],  \;\;\; N \rightarrow \infty.$$

In words:

>   The average of a sequence of random variables from the same distribution converges to the expected value of that distribution.

This may seem like a boring result, but it will be the most useful tool you use.

### Intuition

If the above Law is somewhat surprising,  it can be made more clear by examining a simple example.

Consider a random variable $Z$ that can take only two values, $c_1$ and $c_2$. Suppose we have a large number of samples of $Z$, denoting a specific sample $Z_i$. The Law says that we can approximate the expected value of $Z$ by averaging over all samples. Consider the average:

$$ \frac{1}{N} \sum_{i=1}^N \;Z_i $$

By construction, $Z_i$ can only take on $c_1$ or $c_2$, hence we can partition the sum over these two values:

\begin{align}
\frac{1}{N} \sum_{i=1}^N \;Z_i
& =\frac{1}{N} \big(  \sum_{ Z_i = c_1}c_1 + \sum_{Z_i=c_2}c_2 \big) \\\\[5pt]
& = c_1 \sum_{ Z_i = c_1}\frac{1}{N} + c_2 \sum_{ Z_i = c_2}\frac{1}{N} \\\\[5pt]
& = c_1 \times \text{ (approximate frequency of $c_1$) } \\\\
& \;\;\;\;\;\;\;\;\; + c_2 \times \text{ (approximate frequency of $c_2$) } \\\\[5pt]
& \approx c_1 \times P(Z = c_1) + c_2 \times P(Z = c_2 ) \\\\[5pt]
& = E[Z]
\end{align}

Equality holds in the limit, but we can get closer and closer by using more and more samples in the average. This Law holds for almost *any distribution*, minus some important cases we will encounter later.

##### Example
____

Below is a diagram of the Law of Large numbers in action for three different sequences of Poisson random variables.

 We sample `sample_size = 100000` Poisson random variables with parameter $\lambda = 4.5$. (Recall the expected value of a Poisson random variable is equal to it's parameter.) We calculate the average for the first $n$ samples, for $n=1$ to `sample_size`.

```python
%matplotlib inline
import numpy as np
from IPython.core.pylabtools import figsize
import matplotlib.pyplot as plt

figsize( 12.5, 5 )

sample_size = 100000
expected_value = lambda_ = 4.5
poi = np.random.poisson
N_samples = range(1,sample_size,100)

for k in range(3):

    samples = poi( lambda_, sample_size )

    partial_average = [ samples[:i].mean() for i in N_samples ]

    plt.plot( N_samples, partial_average, lw=1.5,label="average \
of  $n$ samples; seq. %d"%k)

plt.plot( N_samples, expected_value*np.ones_like( partial_average),
    ls = "--", label = "true expected value", c = "k" )

plt.ylim( 4.35, 4.65)
plt.title( "Convergence of the average of \n random variables to its \
expected value" )
plt.ylabel( "average of $n$ samples" )
plt.xlabel( "# of samples, $n$")
plt.legend();
```

Looking at the above plot, it is clear that when the sample size is small, there is greater variation in the average (compare how *jagged and jumpy* the average is initially, then *smooths* out). All three paths *approach* the value 4.5, but just flirt with it as $N$ gets large. Mathematicians and statistician have another name for *flirting*: convergence.

Another very relevant question we can ask is *how quickly am I converging to the expected value?* Let's plot something new. For a specific $N$, let's do the above trials thousands of times and compute how far away we are from the true expected value, on average. But wait — *compute on average*? This is simply the law of large numbers again! For example, we are interested in, for a specific $N$, the quantity:

$$D(N) = \sqrt{ \;E\left[\;\; \left( \frac{1}{N}\sum_{i=1}^NZ_i  - 4.5 \;\right)^2 \;\;\right] \;\;}$$

The above formulae is interpretable as a distance away from the true value (on average), for some $N$. (We take the square root so the dimensions of the above quantity and our random variables are the same). As the above is an expected value, it can be approximated using the law of large numbers: instead of averaging $Z_i$, we calculate the following multiple times and average them:

$$ Y_k = \left( \;\frac{1}{N}\sum_{i=1}^NZ_i  - 4.5 \; \right)^2 $$

By computing the above many, $N_y$, times (remember, it is random), and averaging them:

$$ \frac{1}{N_Y} \sum_{k=1}^{N_Y} Y_k \rightarrow E[ Y_k ] = E\;\left[\;\; \left( \frac{1}{N}\sum_{i=1}^NZ_i  - 4.5 \;\right)^2 \right]$$

Finally, taking the square root:

$$ \sqrt{\frac{1}{N_Y} \sum_{k=1}^{N_Y} Y_k} \approx D(N) $$

```python
figsize( 12.5, 4)

N_Y = 250 #use this many to approximate D(N)
N_array = np.arange( 1000, 50000, 2500 ) #use this many samples in the approx. to the variance.
D_N_results = np.zeros( len( N_array ) )

lambda_ = 4.5
expected_value = lambda_ #for X ~ Poi(lambda) , E[ X ] = lambda

def D_N( n ):
    """
    This function approx. D_n, the average variance of using n samples.
    """
    Z = poi( lambda_, (n, N_Y) )
    average_Z = Z.mean(axis=0)
    return np.sqrt( (  (average_Z - expected_value)**2  ).mean() )

for i,n in enumerate(N_array):
    D_N_results[i] =  D_N(n)

plt.xlabel( "$N$" )
plt.ylabel( "expected squared-distance from true value" )
plt.plot(N_array, D_N_results, lw = 3,
            label="expected distance between\n\
expected value and \naverage of $N$ random variables.")
plt.plot( N_array, np.sqrt(expected_value)/np.sqrt(N_array), lw = 2, ls = "--",
        label = r"$\frac{\sqrt{\lambda}}{\sqrt{N}}$" )
plt.legend()
plt.title( "How 'fast' is the sample average converging? " );
```

As expected, the expected distance between our sample average and the actual expected value shrinks as $N$ grows large. But also notice that the *rate* of convergence decreases, that is, we need only 10 000 additional samples to move from 0.020 to 0.015, a difference of 0.005, but *20 000* more samples to again decrease from 0.015  to 0.010, again only a 0.005 decrease.

It turns out we can measure this rate of convergence. Above I have plotted a second line, the function $\sqrt{\lambda}/\sqrt{N}$. This was not chosen arbitrarily. In most cases, given a sequence of random variable distributed like $Z$, the rate of convergence to $E[Z]$ of the Law of Large Numbers is

$$ \frac{ \sqrt{ \; Var(Z) \; } }{\sqrt{N} }$$

This is useful to know: for a given large $N$, we know (on average) how far away we are from the estimate. On the other hand, in a Bayesian setting, this can seem like a useless result: Bayesian analysis is OK with uncertainty so what's the *statistical* point of adding extra precise digits? Though drawing samples can be so computationally cheap that having a *larger* $N$ is fine too.

### How do we compute $Var(Z)$ though?

The variance is simply another expected value that can be approximated! Consider the following, once we have the expected value (by using the Law of Large Numbers to estimate it, denote it $\mu$), we can estimate the variance:

$$ \frac{1}{N}\sum_{i=1}^N \;(Z_i - \mu)^2 \rightarrow E[ \;( Z - \mu)^2 \;] = Var( Z )$$

### Expected values and probabilities
There is an even less explicit relationship between expected value and estimating probabilities. Define the *indicator function*

$$\mathbb{1}_A(x) =
\begin{cases} 1 &  x \in A \\\\
              0 &  else
\end{cases}
$$
Then, by the law of large numbers, if we have many samples $X_i$, we can estimate the probability of an event $A$, denoted $P(A)$, by:

$$ \frac{1}{N} \sum_{i=1}^N \mathbb{1}_A(X_i) \rightarrow E[\mathbb{1}_A(X)] =  P(A) $$

Again, this is fairly obvious after a moments thought: the indicator function is only 1 if the event occurs, so we are summing only the times the event occurs and dividing by the total number of trials  (consider how we usually approximate probabilities using frequencies). For example, suppose we wish to estimate the probability that a $Z \sim Exp(.5)$ is greater than 5, and we have many samples from a $Exp(.5)$ distribution.

$$ P( Z > 5 ) =  \sum_{i=1}^N \mathbb{1}_{z > 5 }(Z_i) $$

```python
N = 10000
print( np.mean( [ np.random.exponential( 0.5 ) > 5 for i in range(N) ] ) )
```

    0.0001

### What does this all have to do with Bayesian statistics?

*Point estimates*, to be introduced in the next chapter, in Bayesian inference are computed using expected values. In more analytical Bayesian inference, we would have been required to evaluate complicated expected values represented as multi-dimensional integrals. No longer. If we can sample from the posterior distribution directly, we simply need to evaluate averages. Much easier. If accuracy is a priority, plots like the ones above show how fast you are converging. And if further accuracy is  desired, just take more samples from the posterior.

When is enough enough? When can you stop drawing samples from the posterior? That is the practitioners decision, and also dependent on the variance of the samples (recall from above a high variance means the average will converge slower).

We also should understand when the Law of Large Numbers fails. As the name implies, and comparing the graphs above for small $N$, the Law is only true for large sample sizes. Without this, the asymptotic result is not reliable. Knowing in what situations the Law fails can give us *confidence in how unconfident we should be*. The next section deals with this issue.

## The Disorder of Small Numbers

The Law of Large Numbers is only valid as $N$ gets *infinitely* large: never truly attainable.  While the law is a powerful tool, it is foolhardy to apply it liberally. Our next example illustrates this.

##### Example: Aggregated geographic data

Often data comes in aggregated form. For instance, data may be grouped by state, county, or city level. Of course, the population numbers vary per geographic area. If the data is an average of some characteristic of each the geographic areas, we must be conscious of the Law of Large Numbers and how it can *fail* for areas with small populations.

We will observe this on a toy dataset. Suppose there are five thousand counties in our dataset. Furthermore,  population number in each state are uniformly distributed between 100 and 1500. The way the population numbers are generated is irrelevant to the discussion, so we do not justify this. We are interested in measuring the average height of individuals per county. Unbeknownst to us, height does **not** vary across county, and each individual, regardless of the county he or she is currently living in, has the same distribution of what their height may be:

$$ \text{height} \sim \text{Normal}(150, 15 ) $$

We aggregate the individuals at the county level, so we only have data for the *average in the county*. What might our dataset look like?

```python
figsize( 12.5, 4)
std_height = 15
mean_height = 150

n_counties = 5000
pop_generator = np.random.randint
norm = np.random.normal

#generate some artificial population numbers
population = pop_generator(100, 1500, n_counties )

average_across_county = np.zeros( n_counties )
for i in range( n_counties ):
    #generate some individuals and take the mean
    average_across_county[i] = norm(mean_height, 1./std_height,
                                        population[i] ).mean()

#located the counties with the apparently most extreme average heights.
i_min = np.argmin( average_across_county )
i_max = np.argmax( average_across_county )

#plot population size vs. recorded average
plt.scatter( population, average_across_county, alpha = 0.5, c="#7A68A6")
plt.scatter( [ population[i_min], population[i_max] ],
           [average_across_county[i_min], average_across_county[i_max] ],
           s = 60, marker = "o", facecolors = "none",
           edgecolors = "#A60628", linewidths = 1.5,
            label="extreme heights")

plt.xlim( 100, 1500 )
plt.title( "Average height vs. County Population")
plt.xlabel("County Population")
plt.ylabel("Average height in county")
plt.plot( [100, 1500], [150, 150], color = "k", label = "true expected \
height", ls="--" )
plt.legend(scatterpoints = 1);
```

What do we observe? *Without accounting for population sizes* we run the risk of making an enormous inference error: if we ignored population size, we would say that the county with the shortest and tallest individuals have been correctly circled. But this inference is wrong for the following reason. These two counties do *not* necessarily have the most extreme heights. The error results from the calculated average of smaller populations not being a good reflection of the true expected value of the population (which in truth should be $\mu =150$). The sample size/population size/$N$, whatever you wish to call it,  is simply too small to invoke the Law of Large Numbers effectively.

We provide more damning evidence against this inference. Recall the population numbers were uniformly distributed over 100 to 1500. Our intuition should tell us that the counties with the most extreme population heights should also be uniformly spread over 100 to 4000, and certainly independent of the county's population. Not so. Below are the population sizes of the counties with the most extreme heights.

```python
print("Population sizes of 10 'shortest' counties: ")
print(population[ np.argsort( average_across_county )[:10] ], '\n')
print("Population sizes of 10 'tallest' counties: ")
print(population[ np.argsort( -average_across_county )[:10] ])
```

    Population sizes of 10 'shortest' counties:
    [103 110 242 124 141 131 124 113 118 157]

    Population sizes of 10 'tallest' counties:
    [168 164 167 122 225 233 120 246 110 250]

Not at all uniform over 100 to 1500. This is an absolute failure of the Law of Large Numbers.

##### Example:  Kaggle's *U.S. Census Return Rate Challenge*

Below is data from the 2010 US census, which partitions populations beyond counties to the level of block groups (which are aggregates of city blocks or equivalents). The dataset is from a Kaggle machine learning competition some colleagues and I participated in. The objective was to predict the census letter mail-back rate of a group block, measured between 0 and 100, using census variables (median income, number of females in the block-group, number of trailer parks, average number of children etc.). Below we plot the census mail-back rate versus block group population:

```python
figsize( 12.5, 6.5 )
data = np.genfromtxt( "./data/census_data.csv", skip_header=1,
                        delimiter= ",")
plt.scatter( data[:,1], data[:,0], alpha = 0.5, c="#7A68A6")
plt.title("Census mail-back rate vs Population")
plt.ylabel("Mail-back rate")
plt.xlabel("population of block-group")
plt.xlim(-100, 15e3 )
plt.ylim( -5, 105)

i_min = np.argmin(  data[:,0] )
i_max = np.argmax(  data[:,0] )

plt.scatter( [ data[i_min,1], data[i_max, 1] ],
             [ data[i_min,0],  data[i_max,0] ],
             s = 60, marker = "o", facecolors = "none",
             edgecolors = "#A60628", linewidths = 1.5,
             label="most extreme points")

plt.legend(scatterpoints = 1);
```

The above is a classic phenomenon in statistics. I say *classic* referring to the "shape" of the scatter plot above. It follows a classic triangular form, that tightens as we increase the sample size (as the Law of Large Numbers becomes more exact).

I am perhaps overstressing the point and maybe I should have titled the book *"You don't have big data problems!"*, but here again is an example of the trouble with *small datasets*, not big ones. Simply, small datasets cannot be processed using the Law of Large Numbers. Compare with applying the Law without hassle to big datasets (ex. big data). I mentioned earlier that paradoxically big data prediction problems are solved by relatively simple algorithms. The paradox is partially resolved by understanding that the Law of Large Numbers creates solutions that are *stable*, i.e. adding or subtracting a few data points will not affect the solution much. On the other hand, adding or removing data points to a small dataset can create very different results.

For further reading on the hidden dangers of the Law of Large Numbers, I would highly recommend the excellent manuscript [The Most Dangerous Equation](http://nsm.uh.edu/~dgraur/niv/TheMostDangerousEquation.pdf).

##### Example: How to order Reddit submissions

You may have disagreed with the original statement that the Law of Large numbers is known to everyone, but only implicitly in our subconscious decision making. Consider ratings on online products: how often do you trust an average 5-star rating if there is only 1 reviewer? 2 reviewers? 3 reviewers? We implicitly understand that with such few reviewers that the average rating is **not** a good reflection of the true value of the product.

This has created flaws in how we sort items, and more generally, how we compare items. Many people have realized that sorting online search results by their rating, whether the objects be books, videos, or online comments, return poor results. Often the seemingly top videos or comments have perfect ratings only from a few enthusiastic fans, and truly more quality videos or comments are hidden in later pages with *falsely-substandard* ratings of around 4.8. How can we correct this?

Consider the popular site Reddit (I purposefully did not link to the website as you would never come back). The site hosts links to stories or images, called submissions, for people to comment on. Redditors can vote up or down on each submission (called upvotes and downvotes). Reddit, by default, will sort submissions to a given subreddit by Hot, that is, the submissions that have the most upvotes recently.

How would you determine which submissions are the best? There are a number of ways to achieve this:

1. *Popularity*: A submission is considered good if it has many upvotes. A problem with this model is that a submission with hundreds of upvotes, but thousands of downvotes. While being very *popular*, the submission is likely more controversial than best.
2. *Difference*: Using the *difference* of upvotes and downvotes. This solves the above problem, but fails when we consider the temporal nature of submission. Depending on when a submission is posted, the website may be experiencing high or low traffic. The difference method will bias the *Top* submissions to be the those made during high traffic periods, which have accumulated more upvotes than submissions that were not so graced, but are not necessarily the best.
3. *Time adjusted*:  Consider using Difference divided by the age of the submission. This creates a *rate*, something like *difference per second*, or *per minute*. An immediate counter-example is, if we use per second, a 1 second old submission with 1 upvote would be better than a 100 second old submission with 99 upvotes. One can avoid this by only considering at least t second old submission. But what is a good t value? Does this mean no submission younger than t is good? We end up comparing unstable quantities with stable quantities (young vs. old submissions).
3. *Ratio*: Rank submissions by the ratio of upvotes to total number of votes (upvotes plus downvotes). This solves the temporal issue, such that new submissions who score well can be considered Top just as likely as older submissions, provided they have many upvotes to total votes. The problem here is that a submission with a single upvote (ratio = 1.0) will beat a submission with 999 upvotes and 1 downvote (ratio = 0.999), but clearly the latter submission is *more likely* to be better.

I used the phrase *more likely* for good reason. It is possible that the former submission, with a single upvote, is in fact a better submission than the later with 999 upvotes. The hesitation to agree with this is because we have not seen the other 999 potential votes the former submission might get. Perhaps it will achieve an additional 999 upvotes and 0 downvotes and be considered better than the latter, though not likely.

What we really want is an estimate of the *true upvote ratio*. Note that the true upvote ratio is not the same as the observed upvote ratio: the true upvote ratio is hidden, and we only observe upvotes vs. downvotes (one can think of the true upvote ratio as "what is the underlying probability someone gives this submission a upvote, versus a downvote"). So the 999 upvote/1 downvote submission probably has a true upvote ratio close to 1, which we can assert with confidence thanks to the Law of Large Numbers, but on the other hand we are much less certain about the true upvote ratio of the submission with only a single upvote. Sounds like a Bayesian problem to me.

One way to determine a prior on the upvote ratio is to look at the historical distribution of upvote ratios. This can be accomplished by scraping Reddit's submissions and determining a distribution. There are a few problems with this technique though:

1. Skewed data:  The vast majority of submissions have very few votes, hence there will be many submissions with ratios near the extremes (see the "triangular plot" in the above Kaggle dataset), effectively skewing our distribution to the extremes. One could try to only use submissions with votes greater than some threshold. Again, problems are encountered. There is a tradeoff between number of submissions available to use and a higher threshold with associated ratio precision.
2. Biased data: Reddit is composed of different subpages, called subreddits. Two examples are *r/aww*, which posts pics of cute animals, and *r/politics*. It is very likely that the user behaviour towards submissions of these two subreddits are very different: visitors are likely friendly and affectionate in the former, and would therefore upvote submissions more, compared to the latter, where submissions are likely to be controversial and disagreed upon. Therefore not all submissions are the same.

In light of these, I think it is better to use a `Uniform` prior.

With our prior in place, we can find the posterior of the true upvote ratio. The Python script `top_showerthoughts_submissions.py` will scrape the best posts from the `showerthoughts` community on Reddit. This is a text-only community so the title of each post *is* the post. Below is the top post as well as some other sample posts:

```python
#adding a number to the end of the %run call with get the ith top post.
%run top_showerthoughts_submissions.py 2

print("Post contents: \n")
print(top_post)
```

```python
"""
contents: an array of the text from the last 100 top submissions to a subreddit
votes: a 2d numpy array of upvotes, downvotes for each submission.
"""
n_submissions = len(votes)
submissions = np.random.randint( n_submissions, size=4)
print("Some Submissions (out of %d total) \n-----------"%n_submissions)
for i in submissions:
    print('"' + contents[i] + '"')
    print("upvotes/downvotes: ",votes[i,:], "\n")
```

 For a given true upvote ratio $p$ and $N$ votes, the number of upvotes will look like a Binomial random variable with parameters $p$ and $N$. (This is because of the equivalence between upvote ratio and probability of upvoting versus downvoting, out of $N$ possible votes/trials). We create a function that performs Bayesian inference on $p$, for a particular submission's upvote/downvote pair.

```python
import pymc3 as pm

def posterior_upvote_ratio( upvotes, downvotes, samples = 20000):
    """
    This function accepts the number of upvotes and downvotes a particular submission recieved,
    and the number of posterior samples to return to the user. Assumes a uniform prior.
    """
    N = upvotes + downvotes
    with pm.Model() as model:
        upvote_ratio = pm.Uniform("upvote_ratio", 0, 1)
        observations = pm.Binomial( "obs",  N, upvote_ratio, observed=upvotes)

        trace = pm.sample(samples, step=pm.Metropolis())

    burned_trace = trace[int(samples/4):]
    return burned_trace["upvote_ratio"]

```

Below are the resulting posterior distributions.

```python
figsize( 11., 8)
posteriors = []
colours = ["#348ABD", "#A60628", "#7A68A6", "#467821", "#CF4457"]
for i in range(len(submissions)):
    j = submissions[i]
    posteriors.append( posterior_upvote_ratio( votes[j, 0], votes[j,1] ) )
    plt.hist( posteriors[i], bins = 10, normed = True, alpha = .9,
            histtype="step",color = colours[i%5], lw = 3,
            label = '(%d up:%d down)\n%s...'%(votes[j, 0], votes[j,1], contents[j][:50]) )
    plt.hist( posteriors[i], bins = 10, normed = True, alpha = .2,
            histtype="stepfilled",color = colours[i], lw = 3, )

plt.legend(loc="upper left")
plt.xlim( 0, 1)
plt.title("Posterior distributions of upvote ratios on different submissions");
```

Some distributions are very tight, others have very long tails (relatively speaking), expressing our uncertainty with what the true upvote ratio might be.

### Sorting!

We have been ignoring the goal of this exercise: how do we sort the submissions from *best to worst*? Of course, we cannot sort distributions, we must sort scalar numbers. There are many ways to distill a distribution down to a scalar: expressing the distribution through its expected value, or mean, is one way. Choosing the mean is a bad choice though. This is because the mean does not take into account the uncertainty of distributions.

I  suggest using the *95% least plausible value*, defined as the value such that there is only a 5% chance the true parameter is lower (think of the lower bound on the 95% credible region). Below are the posterior distributions with the 95% least-plausible value plotted:

```python
N = posteriors[0].shape[0]
lower_limits = []

for i in range(len(submissions)):
    j = submissions[i]
    plt.hist( posteriors[i], bins = 20, normed = True, alpha = .9,
            histtype="step",color = colours[i], lw = 3,
            label = '(%d up:%d down)\n%s...'%(votes[j, 0], votes[j,1], contents[j][:50]) )
    plt.hist( posteriors[i], bins = 20, normed = True, alpha = .2,
            histtype="stepfilled",color = colours[i], lw = 3, )
    v = np.sort( posteriors[i] )[ int(0.05*N) ]
    #plt.vlines( v, 0, 15 , color = "k", alpha = 1, linewidths=3 )
    plt.vlines( v, 0, 10 , color = colours[i], linestyles = "--",  linewidths=3  )
    lower_limits.append(v)
    plt.legend(loc="upper left")

plt.legend(loc="upper left")
plt.title("Posterior distributions of upvote ratios on different submissions");
order = np.argsort( -np.array( lower_limits ) )
print(order, lower_limits)
```

The best submissions, according to our procedure, are the submissions that are *most-likely* to score a high percentage of upvotes. Visually those are the submissions with the 95% least plausible value close to 1.

Why is sorting based on this quantity a good idea? By ordering by the 95% least plausible value, we are being the most conservative with what we think is best. That is, even in the worst case scenario, when we have severely overestimated the upvote ratio, we can be sure the best submissions are still on top. Under this ordering, we impose the following very natural properties:

1. given two submissions with the same observed upvote ratio, we will assign the submission with more votes as better (since we are more confident it has a higher ratio).
2. given two submissions with the same number of votes, we still assign the submission with more upvotes as *better*.

### But this is too slow for real-time!

I agree, computing the posterior of every submission takes a long time, and by the time you have computed it, likely the data has changed. I delay the mathematics to the appendix, but I suggest using the following formula to compute the lower bound very fast.

$$ \frac{a}{a + b} - 1.65\sqrt{ \frac{ab}{ (a+b)^2(a + b +1 ) } }$$

where
\begin{align}
& a = 1 + u \\\\
& b = 1 + d \\\\
\end{align}

$u$ is the number of upvotes, and $d$ is the number of downvotes. The formula is a shortcut in Bayesian inference, which will be further explained in Chapter 6 when we discuss priors in more detail.

```python
def intervals(u,d):
    a = 1. + u
    b = 1. + d
    mu = a/(a+b)
    std_err = 1.65*np.sqrt( (a*b)/( (a+b)**2*(a+b+1.) ) )
    return ( mu, std_err )

print("Approximate lower bounds:")
posterior_mean, std_err  = intervals(votes[:,0],votes[:,1])
lb = posterior_mean - std_err
print(lb)
print("\n")
print("Top 40 Sorted according to approximate lower bounds:")
print("\n")
order = np.argsort( -lb )
ordered_contents = []
for i in order[:40]:
    ordered_contents.append( contents[i] )
    print(votes[i,0], votes[i,1], contents[i])
    print("-------------")
```

    Approximate lower bounds:
    [ 0.93349005  0.9532194   0.94149718  0.90859764  0.88705356  0.8558795
      0.85644927  0.93752679  0.95767101  0.91131012  0.910073    0.915999
      0.9140058   0.83276025  0.87593961  0.87436674  0.92830849  0.90642832
      0.89187973  0.89950891  0.91295322  0.78607629  0.90250203  0.79950031
      0.85219422  0.83703439  0.7619808   0.81301134  0.7313114   0.79137561
      0.82701445  0.85542404  0.82309334  0.75211374  0.82934814  0.82674958
      0.80933194  0.87448152  0.85350205  0.75460106  0.82934814  0.74417233
      0.79924258  0.8189683   0.75460106  0.90744016  0.83838023  0.78802791
      0.78400654  0.64638659  0.62047936  0.76137738  0.81365241  0.83838023
      0.78457533  0.84980627  0.79249393  0.69020315  0.69593922  0.70758151
      0.70268831  0.91620627  0.73346864  0.86382644  0.80877728  0.72708753
      0.79822085  0.68333632  0.81699014  0.65100453  0.79809005  0.74702492
      0.77318569  0.83221179  0.66500492  0.68134548  0.7249286   0.59412132
      0.58191312  0.73142963  0.73142963  0.66251028  0.87152685  0.74107856
      0.60935684  0.87152685  0.77484517  0.88783675  0.81814153  0.54569789
      0.6122496   0.75613569  0.53511973  0.74556767  0.81814153  0.85773646
      0.6122496   0.64814153]

    Top 40 Sorted according to approximate lower bounds:

    596 18 Someone should develop an AI specifically for reading Terms & Conditions and flagging dubious parts.
    -------------
    2360 98 Porn is the only industry where it is not only acceptable but standard to separate people based on race, sex and sexual preference.
    -------------
    1918 101 All polls are biased towards people who are willing to take polls
    -------------
    948 50 They should charge less for drinks in the drive-thru because you can't refill them.
    -------------
    3740 239 When I was in elementary school and going through the DARE program, I was positive a gang of older kids was going to corner me and force me to smoke pot. Then I became an adult and realized nobody is giving free drugs to somebody that doesn't want them.
    -------------
    166 7 "Noted" is the professional way of saying "K".
    -------------
    29 0 Rewatching Mr. Bean, I've realised that the character is an eccentric genius and not a blithering idiot.
    -------------
    289 18 You've been doing weird cameos in your friends' dreams since kindergarten.
    -------------
    269 17 At some point every parent has stopped wiping their child's butt and hoped for the best.
    -------------
    121 6 Is it really fair to say a person over 85 has heart failure? Technically, that heart has done exceptionally well.
    -------------
    535 40 It's surreal to think that the sun and moon and stars we gaze up at are the same objects that have been observed for millenia, by everyone in the history of humanity from cavemen to Aristotle to Jesus to George Washington.
    -------------
    527 40 I wonder if America's internet is censored in a similar way that North Korea's is, but we have no idea of it happening.
    -------------
    1510 131 Kenny's family is poor because they're always paying for his funeral.
    -------------
    43 1 If I was as careful with my whole paycheck as I am with my last $20 I'd be a whole lot better off
    -------------
    162 10 Black hair ties are probably the most popular bracelets in the world.
    -------------
    107 6 The best answer to the interview question "What is your greatest weakness?" is "interviews".
    -------------
    127 8 Surfing the internet without ads feels like a summer evening without mosquitoes
    -------------
    159 12 I wonder if Superman ever put a pair of glasses on Lois Lane's dog, and she was like "what's this Clark? Did you get me a new dog?"
    -------------
    21 0 Sitting on a cold toilet seat or a warm toilet seat both suck for different reasons.
    -------------
    1414 157 My life is really like Rihanna's song, "just work work work work work" and the rest of it I can't really understand.
    -------------
    222 22 I'm honestly slightly concerned how often Reddit commenters make me laugh compared to my real life friends.
    -------------
    52 3 The world must have been a spookier place altogether when candles and gas lamps were the only sources of light at night besides the moon and the stars.
    -------------
    194 19 I have not been thankful enough in the last few years that the Black Eyed Peas are no longer ever on the radio
    -------------
    18 0 Living on the coast is having the window seat of the land you live on.
    -------------
    18 0 Binoculars are like walkie talkies for the deaf.
    -------------
    28 1 Now that I am a parent of multiple children I have realized that my parents were lying through their teeth when they said they didn't have a favorite.
    -------------
    16 0 I sneer at people who read tabloids, but every time I look someone up on Wikipedia the first thing I look for is what controversies they've been involved in.
    -------------
    1559 233 Kid's menus at restaurants should be smaller portions of the same adult dishes at lower prices and not the junk food that they usually offer.
    -------------
    1426 213 Eventually once all phones are waterproof we'll be able to push people into pools again
    -------------
    61 5 Myspace is so outdated that jokes about it being outdated has become outdated
    -------------
    52 4 As a kid, seeing someone step on a banana peel and not slip was a disappointment.
    -------------
    90 9 Yahoo!® is the RadioShack® of the Internet.
    -------------
    34 2 People who "tell it like it is" rarely do so to say something nice
    -------------
    39 3 Closing your eyes after turning off your alarm is a very dangerous game.
    -------------
    39 3 Your known 'first word' is the first word your parents heard you speak. In reality, it may have been a completely different word you said when you were alone.
    -------------
    87 10 "Smells Like Teen Spirit" is as old to listeners of today as "Yellow Submarine" was to listeners of 1991.
    -------------
    239 36 if an ocean didnt stop immigrants from coming to America what makes us think a wall will?
    -------------
    22 1 The phonebook was the biggest invasion of privacy that everyone was oddly ok with.
    -------------
    57 6 I'm actually the most productive when I procrastinate because I'm doing everything I possibly can to avoid the main task at hand.
    -------------
    57 6 You will never feel how long time is until you have allergies and snot slowly dripping out of your nostrils, while sitting in a classroom with no tissues.
    -------------

We can view the ordering visually by plotting the posterior mean and bounds, and sorting by the lower bound. In the plot below, notice that the left error-bar is sorted (as we suggested this is the best way to determine an ordering), so the means, indicated by dots, do not follow any strong pattern.

```python
r_order = order[::-1][-40:]
plt.errorbar( posterior_mean[r_order], np.arange( len(r_order) ),
               xerr=std_err[r_order], capsize=0, fmt="o",
                color = "#7A68A6")
plt.xlim( 0.3, 1)
plt.yticks( np.arange( len(r_order)-1,-1,-1 ), map( lambda x: x[:30].replace("\n",""), ordered_contents) );
```

In the graphic above, you can see why sorting by mean would be sub-optimal.

### Extension to Starred rating systems

The above procedure works well for upvote-downvotes schemes, but what about systems that use star ratings, e.g. 5 star rating systems. Similar problems apply with simply taking the average: an item with two perfect ratings would beat an item with thousands of perfect ratings, but a single sub-perfect rating.

We can consider the upvote-downvote problem above as binary: 0 is a downvote, 1 if an upvote. A $N$-star rating system can be seen as a more continuous version of above, and we can set $n$ stars rewarded is equivalent to rewarding $\frac{n}{N}$. For example, in a 5-star system, a 2 star rating corresponds to 0.4. A perfect rating is a 1. We can use the same formula as before, but with $a,b$ defined differently:

$$ \frac{a}{a + b} - 1.65\sqrt{ \frac{ab}{ (a+b)^2(a + b +1 ) } }$$

where

\begin{align}
& a = 1 + S \\\\
& b = 1 + N - S \\\\
\end{align}

where $N$ is the number of users who rated, and $S$ is the sum of all the ratings, under the equivalence scheme mentioned above.

##### Example: Counting Github stars

What is the average number of stars a Github repository has? How would you calculate this? There are over 6 million respositories, so there is more than enough data to invoke the Law of Large numbers. Let's start pulling some data. TODO

### Conclusion

While the Law of Large Numbers is cool, it is only true so much as its name implies: with large sample sizes only. We have seen how our inference can be affected by not considering *how the data is shaped*.

1. By (cheaply) drawing many samples from the posterior distributions, we can ensure that the Law of Large Number applies as we approximate expected values (which we will do in the next chapter).

2. Bayesian inference understands that with small sample sizes, we can observe wild randomness. Our posterior distribution will reflect this by being more spread rather than tightly concentrated. Thus, our inference should be correctable.

3. There are major implications of not considering the sample size, and trying to sort objects that are unstable leads to pathological orderings. The method provided above solves this problem.

### Appendix

##### Derivation of sorting submissions formula

Basically what we are doing is using a Beta prior (with parameters $a=1, b=1$, which is a uniform distribution), and using a Binomial likelihood with observations $u, N = u+d$. This means our posterior is a Beta distribution with parameters $a' = 1 + u, b' = 1 + (N - u) = 1+d$. We then need to find the value, $x$, such that 0.05 probability is less than $x$. This is usually done by inverting the CDF ([Cumulative Distribution Function](http://en.wikipedia.org/wiki/Cumulative_Distribution_Function)), but the CDF of the beta, for integer parameters, is known but is a large sum [3].

We instead use a Normal approximation. The mean of the Beta is $\mu = a'/(a'+b')$ and the variance is

$$\sigma^2 = \frac{a'b'}{ (a' + b')^2(a'+b'+1) }$$

Hence we solve the following equation for $x$ and have an approximate lower bound.

$$ 0.05 = \Phi\left( \frac{(x - \mu)}{\sigma}\right) $$

$\Phi$ being the [cumulative distribution for the normal distribution](http://en.wikipedia.org/wiki/Normal_distribution#Cumulative_distribution)

##### Exercises

1\. How would you estimate the quantity $E\left[ \cos{X} \right]$, where $X \sim \text{Exp}(4)$? What about $E\left[ \cos{X} | X \lt 1\right]$, i.e. the expected value *given* we know $X$ is less than 1? Would you need more samples than the original samples size to be equally accurate?

```python
## Enter code here
import scipy.stats as stats
exp = stats.expon( scale=4 )
N = 1e5
X = exp.rvs( int(N) )
X
```

    array([ 4.15916329,  0.47426831,  1.82708441, ...,  3.31282443,
            1.48344699,  2.26235793])

```python
len(X)
```

    100000

```python
np.mean(X)
```

    3.9996017031340521

```python
## CI 95%
from math import sqrt

z= 1.98
se = sqrt(np.var(X)/N)
print('Confidence Interval for N= {}'.format(N),np.mean(X)-z*se,np.mean(X)+z*se)
print('Distance',(np.mean(X)+z*se) - (np.mean(X)-z*se))
```

    Confidence Interval for N= 100000.0 3.97455904507 4.0246443612
    Distance 0.0500853161295

```python
### know X is less than 1
se_2 = sqrt(np.var(X[X<1])/N)
print('Confidence Interval for N= {}'.format(N),np.mean(X[X<1])-z*se_2,np.mean(X[X<1])+z*se_2)
print('Distance',(np.mean(X[X<1])+z*se_2) - (np.mean(X[X<1])-z*se_2))
```

    Confidence Interval for N= 100000.0 0.476438056389 0.480046838729
    Distance 0.00360878234006

```python
### We are more confident in the estimates that only look at X when X<1
```

```python

```

2\. The following table was located in the paper "Going for Three: Predicting the Likelihood of Field Goal Success with Logistic Regression" [2]. The table ranks football field-goal kickers by their percent of non-misses. What mistake have the researchers made?

-----

####  Kicker Careers Ranked by Make Percentage
<table><tbody><tr><th>Rank </th><th>Kicker </th><th>Make % </th><th>Number  of Kicks</th></tr><tr><td>1 </td><td>Garrett Hartley </td><td>87.7 </td><td>57</td></tr><tr><td>2</td><td> Matt Stover </td><td>86.8 </td><td>335</td></tr><tr><td>3 </td><td>Robbie Gould </td><td>86.2 </td><td>224</td></tr><tr><td>4 </td><td>Rob Bironas </td><td>86.1 </td><td>223</td></tr><tr><td>5</td><td> Shayne Graham </td><td>85.4 </td><td>254</td></tr><tr><td>… </td><td>… </td><td>…</td><td> </td></tr><tr><td>51</td><td> Dave Rayner </td><td>72.2 </td><td>90</td></tr><tr><td>52</td><td> Nick Novak </td><td>71.9 </td><td>64</td></tr><tr><td>53 </td><td>Tim Seder </td><td>71.0 </td><td>62</td></tr><tr><td>54 </td><td>Jose Cortez </td><td>70.7</td><td> 75</td></tr><tr><td>55 </td><td>Wade Richey </td><td>66.1</td><td> 56</td></tr></tbody></table>

```python
### These researches have not taken into account the number of kicks that each kicker has made/attempted.
```

```python

```

In August 2013, [a popular post](http://bpodgursky.wordpress.com/2013/08/21/average-income-per-programming-language/) on the average income per programmer of different languages was trending. Here's the summary chart: (reproduced without permission, cause when you lie with stats, you gunna get the hammer). What do you notice about the extremes?

------

#### Average household income by programming language

<table >
 <tr><td>Language</td><td>Average Household Income ($)</td><td>Data Points</td></tr>
 <tr><td>Puppet</td><td>87,589.29</td><td>112</td></tr>
 <tr><td>Haskell</td><td>89,973.82</td><td>191</td></tr>
 <tr><td>PHP</td><td>94,031.19</td><td>978</td></tr>
 <tr><td>CoffeeScript</td><td>94,890.80</td><td>435</td></tr>
 <tr><td>VimL</td><td>94,967.11</td><td>532</td></tr>
 <tr><td>Shell</td><td>96,930.54</td><td>979</td></tr>
 <tr><td>Lua</td><td>96,930.69</td><td>101</td></tr>
 <tr><td>Erlang</td><td>97,306.55</td><td>168</td></tr>
 <tr><td>Clojure</td><td>97,500.00</td><td>269</td></tr>
 <tr><td>Python</td><td>97,578.87</td><td>2314</td></tr>
 <tr><td>JavaScript</td><td>97,598.75</td><td>3443</td></tr>
 <tr><td>Emacs Lisp</td><td>97,774.65</td><td>355</td></tr>
 <tr><td>C#</td><td>97,823.31</td><td>665</td></tr>
 <tr><td>Ruby</td><td>98,238.74</td><td>3242</td></tr>
 <tr><td>C++</td><td>99,147.93</td><td>845</td></tr>
 <tr><td>CSS</td><td>99,881.40</td><td>527</td></tr>
 <tr><td>Perl</td><td>100,295.45</td><td>990</td></tr>
 <tr><td>C</td><td>100,766.51</td><td>2120</td></tr>
 <tr><td>Go</td><td>101,158.01</td><td>231</td></tr>
 <tr><td>Scala</td><td>101,460.91</td><td>243</td></tr>
 <tr><td>ColdFusion</td><td>101,536.70</td><td>109</td></tr>
 <tr><td>Objective-C</td><td>101,801.60</td><td>562</td></tr>
 <tr><td>Groovy</td><td>102,650.86</td><td>116</td></tr>
 <tr><td>Java</td><td>103,179.39</td><td>1402</td></tr>
 <tr><td>XSLT</td><td>106,199.19</td><td>123</td></tr>
 <tr><td>ActionScript</td><td>108,119.47</td><td>113</td></tr>
</table>

### References

1. Wainer, Howard. *The Most Dangerous Equation*. American Scientist, Volume 95.
2. Clarck, Torin K., Aaron W. Johnson, and Alexander J. Stimpson. "Going for Three: Predicting the Likelihood of Field Goal Success with Logistic Regression." (2013): n. page. [Web](http://www.sloansportsconference.com/wp-content/uploads/2013/Going%20for%20Three%20Predicting%20the%20Likelihood%20of%20Field%20Goal%20Success%20with%20Logistic%20Regression.pdf). 20 Feb. 2013.
3. http://en.wikipedia.org/wiki/Beta_function#Incomplete_beta_function

```python
from IPython.core.display import HTML
def css_styling():
    styles = open("../styles/custom.css", "r").read()
    return HTML(styles)
css_styling()
```

<style>
    @font-face {
        font-family: "Computer Modern";
        src: url('http://9dbb143991406a7c655e-aa5fcb0a5a4ec34cff238a2d56ca4144.r56.cf5.rackcdn.com/cmunss.otf');
    }
    @font-face {
        font-family: "Computer Modern";
        font-weight: bold;
        src: url('http://9dbb143991406a7c655e-aa5fcb0a5a4ec34cff238a2d56ca4144.r56.cf5.rackcdn.com/cmunsx.otf');
    }
    @font-face {
        font-family: "Computer Modern";
        font-style: oblique;
        src: url('http://9dbb143991406a7c655e-aa5fcb0a5a4ec34cff238a2d56ca4144.r56.cf5.rackcdn.com/cmunsi.otf');
    }
    @font-face {
        font-family: "Computer Modern";
        font-weight: bold;
        font-style: oblique;
        src: url('http://9dbb143991406a7c655e-aa5fcb0a5a4ec34cff238a2d56ca4144.r56.cf5.rackcdn.com/cmunso.otf');
    }
    div.cell{
        width:800px;
        margin-left:16% !important;
        margin-right:auto;
    }
    h1 {
        font-family: Helvetica, serif;
    }
    h4{
        margin-top:12px;
        margin-bottom: 3px;
       }
    div.text_cell_render{
        font-family: Computer Modern, "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
        line-height: 145%;
        font-size: 130%;
        width:800px;
        margin-left:auto;
        margin-right:auto;
    }
    .CodeMirror{
            font-family: "Source Code Pro", source-code-pro,Consolas, monospace;
    }
    .prompt{
        display: None;
    }
    .text_cell_render h5 {
        font-weight: 300;
        font-size: 22pt;
        color: #4057A1;
        font-style: italic;
        margin-bottom: .5em;
        margin-top: 0.5em;
        display: block;
    }

    .warning{
        color: rgb( 240, 20, 20 )
        }
</style>

<style>
    img{
        max-width:800px}
</style>


\tableofcontents
% These TeX commands run at the start to remove section numbering
\renewcommand{\thesection}{\hspace*{-1.0em}}
\renewcommand{\thesubsection}{\hspace*{-1.0em}}
\renewcommand{\thesubsubsection}{\hspace*{-1.0em}}

```python
%pylab inline
#%matplotlib qt
import sk_dsp_comm.sigsys as ss
import scipy.signal as signal
from IPython.display import Audio, display
from IPython.display import Image, SVG
```

```python
pylab.rcParams['savefig.dpi'] = 100 # default 72
#pylab.rcParams['figure.figsize'] = (6.0, 4.0) # default (6,4)
#%config InlineBackend.figure_formats=['png'] # default for inline viewing
%config InlineBackend.figure_formats=['svg'] # SVG inline viewing
#%config InlineBackend.figure_formats=['pdf'] # render pdf figs for LaTeX
```

```python
import scipy.special as special
import sk_dsp_comm.digitalcom as dc
import sk_dsp_comm.fec_block as block
```

# Block Codes

Block codes take serial source symbols and group them into k-symbol blocks. They then take n-k check symbols to make code
words of length n > k. The code is denoted (n,k). The following shows a general block diagram of block encoder.

The block encoder takes k source bits and encodes it into a length n codeword. A block decoder then works in reverse. The length n channel symbol codewords are decoded into the original length k source bits.

## Single Error Correction Block Codes

Several block codes are able to correct only one error per block. Two common single error correction codes are cyclic codes and hamming codes. In `scikit-dsp-comm` there is a module called `fec_block.py`. This module contains two classes so far: `fec_cyclic` for cyclic codes and `fec_hamming` for hamming codes. Each class has methods for encoding, decoding, and plotting theoretical bit error probability bounds.

### Cyclic Codes

A (n,k) cyclic code can easily be generated with an n-k stage shift register with appropriate feedback according to Ziemer and Tranter pgs 646 and 647. The following shows a block diagram for a cyclic encoder.

This block diagram can be expanded to larger codes as well. A generator polynomial can be used to determine the position of the binary adders. The previous example uses a generator polynomial of '1011'. This means that there is a binary adder after the input, after second shift register, and after the third shift register.

The source symbol length and the channel symbol length can be determined from the number of shift registers $j$. The length of the generator polynomial is always $1+j$. In this case we have 3 shift registers, so $j=3$. We have $k=4$ source bits and $n=7$ channel bits. For other shift register lengths, we can use the following equations. $n=j^2-1$ and $k = n-j$. The following table (from Ziemer and Peterson pg 429) shows the source symbol length, channel symbol length, and the code rate for various shift register lengths for single error correction codes.

| j |  k  |  n  | R=k/n |
|---|-----|-----|-------|
|3  |4    |7    |0.57   |
|4  |11   |15   |0.73   |
|5  |26   |31   |0.84   |
|6  |57   |63   |0.90   |
|7  |120  |127  |0.94   |
|8  |247  |255  |0.97   |
|9  |502  |511  |0.98   |
|10 |1013 |1023 |0.99   |

The following block diagram shows a block decoder (from Ziemer and Tranter page 647). The block decoder takes in a codeword of channel symbol length n and decodes it to the original source bits of length k.

The `fec_cyclic` class can be used to generate a cyclic code object. The cyclic code object can be initialized by a generator polynomial. The length of the generator determines the source symbol length, the channel symbol length, and the rate. The following shows the generator polynomial '1011' considered in the two example block diagrams.

```python
cc1 = block.FECCyclic('1011')
```

After the cyclic code object `cc1` is created, the `cc1.cyclic_encoder` method can be used to encode source data bits. In the following example, we generate 16 distinct source symbols to get 16 distinct channel symbol codewords using the `cyclic_encoder` method. The `cyclic_encoder` method takes an array of source bits as a paramter. The array of source bits must be a length of a multiple of $k$. Otherwise, the method will throw an error.

```python
# Generate 16 distinct codewords
codewords = zeros((16,7),dtype=int)
x = zeros((16,4))
for i in range(0,16):
    xbin = block.binary(i,4)
    xbin = array(list(xbin)).astype(int)
    x[i,:] = xbin
x = reshape(x,size(x)).astype(int)
codewords = cc1.cyclic_encoder(x)
print(reshape(codewords,(16,7)))
```

Now, a bit error is introduced into each of the codewords. Then, the codwords with the error are decoded using the `cyclic_decoder` method. The `cyclic_decoder` method takes an array of codewords of length $n$ as a parameter and returns an array of source bits. Even with 1 error introduced into each codeword, All of the original source bits are still decoded properly.

```python
# introduce 1 bit error into each code word and decode
codewords = reshape(codewords,(16,7))
for i in range(16):
    error_pos = i % 6
    codewords[i,error_pos] = (codewords[i,error_pos] +1) % 2
codewords = reshape(codewords,size(codewords))
decoded_blocks = cc1.cyclic_decoder(codewords)
print(reshape(decoded_blocks,(16,4)))
```

The following example generates many random source symbols. It then encodes the symbols using the cyclic encoder. It then simulates a channel by adding noise. It then implements hard decisions on each of the incoming bits and puts the received noisy bits into the cyclic decoder. Source bits are then returned and errors are counted until 100 bit errors are received. Once 100 bit errors are received, the bit error probability is calculated. This code can be run at a variety of SNRs and with various code rates.

```python
cc1 = block.FECCyclic('101001')
N_blocks_per_frame = 2000
N_bits_per_frame = N_blocks_per_frame*cc1.k
EbN0 = 6
total_bit_errors = 0
total_bit_count = 0

while total_bit_errors < 100:
    # Create random 0/1 bits
    x = randint(0,2,N_bits_per_frame)
    y = cc1.cyclic_encoder(x)
    # Add channel noise to bits and scale to +/- 1
    yn = dc.cpx_awgn(2*y-1,EbN0-10*log10(cc1.n/cc1.k),1) # Channel SNR is dB less
    # Scale back to 0 and 1
    yn = ((sign(yn.real)+1)/2).astype(int)
    z = cc1.cyclic_decoder(yn)
    # Count bit errors
    bit_count, bit_errors = dc.bit_errors(x,z)
    total_bit_errors += bit_errors
    total_bit_count += bit_count
    print('Bits Received = %d, Bit errors = %d, BEP = %1.2e' %\
          (total_bit_count, total_bit_errors,\
           total_bit_errors/total_bit_count))
print('*****************************************************')
print('Bits Received = %d, Bit errors = %d, BEP = %1.2e' %\
      (total_bit_count, total_bit_errors,\
       total_bit_errors/total_bit_count))
```

There is a function in the `fec_block` module called `block_single_error_Pb_bound` that can be used to generate the theoretical bit error probability bounds for single error correction block codes. Measured bit error probabilities from the previous example were recorded to compare to the bounds.

```python
SNRdB = arange(0,12,.1)
#SNRdB = arange(9.4,9.6,0.1)
Pb_uc = block.block_single_error_Pb_bound(3,SNRdB,False)
Pb_c_3 = block.block_single_error_Pb_bound(3,SNRdB)
Pb_c_4 = block.block_single_error_Pb_bound(4,SNRdB)
Pb_c_5 = block.block_single_error_Pb_bound(5,SNRdB)
figure(figsize=(5,5))
semilogy(SNRdB,Pb_uc,'k-')
semilogy(SNRdB,Pb_c_3,'c--')
semilogy(SNRdB,Pb_c_4,'m--')
semilogy(SNRdB,Pb_c_5,'g--')
semilogy([4,5,6,7,8,9],[1.44e-2,5.45e-3,2.37e-3,6.63e-4,1.33e-4,1.31e-5],'cs')
semilogy([5,6,7,8],[4.86e-3,1.16e-3,2.32e-4,2.73e-5],'ms')
semilogy([5,6,7,8],[4.31e-3,9.42e-4,1.38e-4,1.15e-5],'gs')
axis([0,12,1e-10,1e0])
title('Cyclic code BEP')
xlabel(r'$E_b/N_0$ (dB)')
ylabel(r'Bit Error Probability')
legend(('Uncoded BPSK','(7,4), hard',\
        '(15,11), hard', '(31,26), hard',\
        '(7,4) sim', '(15,11) sim', \
        '(31,26) sim'),loc='lower left')
grid();
```

These plots show that the simulated bit error probability is very close to the theoretical bit error probabilites.

### Hamming Code

Hamming codes are another form of single error correction block codes. Hamming codes use parity-checks in order to generate and decode block codes. The code rates of Hamming codes are generated the same way as cyclic codes. In this case a parity-check length of length $j$ is chosen, and n and k are calculated by $n=2^j-1$ and $k=n-j$. Hamming codes are generated first by defining a parity-check matrix $H$. The parity-check matrix is a j x n matrix containing binary numbers from 1 to n as the columns. For a $j=3$ ($k=4$, $n=7$) Hamming code. The parity-check matrix starts out as the following:

\begin{equation}
\mathbf{H} = \left[\begin{array}
{rrr}
0 & 0 & 0 & 1 & 1 & 1 & 1\\
0 & 1 & 1 & 0 & 0 & 1 & 1\\
1 & 0 & 1 & 0 & 1 & 0 & 1
\end{array}\right]
\end{equation}

The parity-chekc matrix can be reordered to provice a systematic code by interchanging the columns to create an identity matrix on the right side of the matrix. In this case, this is done by interchangeing columsn 1 and 7, columns 2 and 6, and columsn 4 and 5. The resulting parity-check matrix is the following.

\begin{equation}
\mathbf{H} = \left[\begin{array}
{rrr}
1 & 1 & 0 & 1 & 1 & 0 & 0\\
1 & 1 & 1 & 0 & 0 & 1 & 0\\
1 & 0 & 1 & 1 & 0 & 0 & 1
\end{array}\right]
\end{equation}

Next, a generator matrix $G$ is created by restructuring the parity-check matrix. The $G$ matrix is gathered from the $H$ matrix through the following relationship.

\begin{equation}
\mathbf{G} = \left[\begin{array}
{rrr}
I_k & ... & H_p
\end{array}\right]
\end{equation}

where $H_p$ is defined as the transpose of the first k columns of H. For this example we arrive at the following $G$ matrix. G always ends up being a k x n matrix.

\begin{equation}
\mathbf{G} = \left[\begin{array}
{rrr}
1 & 0 & 0 & 0 & 1 & 1 & 1\\
0 & 1 & 0 & 0 & 1 & 1 & 0\\
0 & 0 & 1 & 0 & 0 & 1 & 1\\
0 & 0 & 0 & 1 & 1 & 0 & 1
\end{array}\right]
\end{equation}

Codewords can be generated by multiplying a source symbol matrix by the generator matrix.

\begin{equation}
codeword = xG
\end{equation}

Where the codeword is a column vector of length $n$ and x is a row vector of length $n$. This is the basic operation of the encoder. The decoder is slightly more complicated. The decoder starts by taking the parity-check matrix $H$ and multiplying it by the codeword column vector. This gives the "syndrome" of the block. The syndrome tells us whether or not there is an error in the codeword. If no errors are present, the syndrome will be 0. If there is an error in the codeword, the syndrome will tell us which bit has the error.

\begin{equation}
S = H \cdot codeword
\end{equation}

If the syndrome is nonzero, then it can be used to correct the error bit in the codeword. After that, the original source blocks can be decoded from the codewords by the following equation.

\begin{equation}
source = R\cdot codeword
\end{equation}

Where $R$ is a k x n matrix where R is made up of a k x k identity matrix and a k x n-k matrix of zeros. Again, the Hamming code is only capable of correcting one error per block, so if more than one error is present in the block, then the syndrome cannot be used to correct the error.

The hamming code class can be found in the `fec_block` module as `fec_hamming`. Hamming codes are sometimes generated using generator polynomials just like with cyclic codes. This is not completely necessary, however, if the previously described process is used. This process simply relies on choosing a number of parity bits and then systematic single-error correction hamming codes are automatically generated. The following will go through an example of a $j=3$ ($k=4$, $n=7$) hamming code.

Hamming Block Code Class Definition:

```python
hh1 = block.FECHamming(3)
```

$k$ and $n$ are calculated form the number of parity checks $j$ and can be accessed by `hh1.k` and `hh1.n`. The $j$ x $n$ parity-check matrix $H$ and the $k$ x $n$ generator matrix $G$ can be accessed by `hh1.H` and `hh1.G`. These are exactly as described previously.

```python
print('k = ' + str(hh1.k))
print('n = ' + str(hh1.n))
print('H = \n' + str(hh1.H))
print('G = \n' + str(hh1.G))
```

The `fec_hamming` class has an encoder method called `hamm_encoder`. This method works the same way as the cyclic encoder. It takes an array of source bits with a length that is a multiple of $k$ and returns an array of codewords. This class has another method called `hamm_decoder` which can decode an array of codewords. The array of codewords must have a length that is a multiple of $n$. The following example generates random source bits, encodes them using a hamming encoder, simulates transmitting them over a channel, uses hard decisions after the receiver to get a received array of codewords, and decodes the codewords using the hamming decoder. It runs until it counds 100 bit errors and then calculates the bit error probability. This can be used to simulate hamming codes with different rates (different numbers of parity checks) at different SNRs.

```python
hh1 = block.FECHamming(5)
N_blocks_per_frame = 20000
N_bits_per_frame = N_blocks_per_frame*hh1.k
EbN0 = 8
total_bit_errors = 0
total_bit_count = 0

while total_bit_errors < 100:
    # Create random 0/1 bits
    x = randint(0,2,N_bits_per_frame)
    y = hh1.hamm_encoder(x)
    # Add channel noise to bits and scale to +/- 1
    yn = dc.cpx_awgn(2*y-1,EbN0-10*log10(hh1.n/hh1.k),1) # Channel SNR is dB less
    # Scale back to 0 and 1
    yn = ((sign(yn.real)+1)/2).astype(int)
    z = hh1.hamm_decoder(yn)
    # Count bit errors
    bit_count, bit_errors = dc.bit_errors(x,z)
    total_bit_errors += bit_errors
    total_bit_count += bit_count
    print('Bits Received = %d, Bit errors = %d, BEP = %1.2e' %\
          (total_bit_count, total_bit_errors,\
           total_bit_errors/total_bit_count))
print('*****************************************************')
print('Bits Received = %d, Bit errors = %d, BEP = %1.2e' %\
      (total_bit_count, total_bit_errors,\
       total_bit_errors/total_bit_count))
```

The `fec_block.block_single_error_Pb_bound` function can also be used to generate the bit error probability bounds for hamming codes. The following example generates theoretical bit error probability bounds for hamming codes and compares it with simulated bit error probabilities from the previous examples.

```python
SNRdB = arange(0,12,.1)
Pb_uc = block.block_single_error_Pb_bound(3,SNRdB,False)
Pb_c_3 = block.block_single_error_Pb_bound(3,SNRdB)
Pb_c_4 = block.block_single_error_Pb_bound(4,SNRdB)
Pb_c_5 = block.block_single_error_Pb_bound(5,SNRdB)
figure(figsize=(5,5))
semilogy(SNRdB,Pb_uc,'k-')
semilogy(SNRdB,Pb_c_3,'c--')
semilogy(SNRdB,Pb_c_4,'m--')
semilogy(SNRdB,Pb_c_5,'g--')
semilogy([5,6,7,8,9,10],[6.64e-3,2.32e-3,5.25e-4,1.16e-4,1.46e-5,1.19e-6],'cs')
semilogy([5,6,7,8,9],[4.68e-3,1.19e-3,2.48e-4,3.6e-5,1.76e-6],'ms')
semilogy([5,6,7,8,9],[4.42e-3,1.11e-3,1.41e-4,1.43e-5,6.73e-7],'gs')
axis([0,12,1e-10,1e0])
title('Hamming code BEP')
xlabel(r'$E_b/N_0$ (dB)')
ylabel(r'Bit Error Probability')
legend(('Uncoded BPSK','(7,4), hard',\
        '(15,11), hard', '(31,26), hard',\
        '(7,4) sim', '(15,11) sim', \
        '(31,26) sim'),loc='lower left')
grid();
```

## Multiple Error Correction Block Codes

Other block codes are capable of correcting multiple errors in blocks. Golay Codes, Bose_Chaudhuri-Hocquenghem (BCH) Codes, and Reed-Solomon Codes are all capable of correcting multiple errors. These codes have not been developed yet, but they will be the next codes to be added to the `fec_block` module.

### Golay Code

Golay codes are capable of correcting three errors in a block of 23 symbols. Golay codes are one of the few known "perfect" codes where all error patterns with hamming weight $t$ or less and no error patters with weight greater than $t$ are correctable using a minimum-distance maximum-likelihood decoder. Golay codes are discussed in detail in Ziemer and Peterson pgs 448-450.

### Bose-Chaudhuri-Hocquenghem (BCH) Codes

BCH codes are very important because they exist for a wide range of rates, can achieve significant coding gain, and decoders can be implemented even at high speeds. BCH codes are described in detail in Ziemer and Peterson pgs 436-444.

### Reed-Solomon Codes

RS codes are nonbinary BCH codes that use input and output alphabets having $2^m$ symbols, {$0,1,2,...,2^m-1$}. Block length is $n=2^m-1$ and can be extended to $n=2^m$ or $n=2^m+1$. Reed-Solomon codes are useful in burst communications Reed-Solomon Codes are discussed in detail in Ziemer and Peterson pgs 444-447.


# Random Signals

*This jupyter notebook is part of a [collection of notebooks](../index.ipynb) on various topics of Digital Signal Processing. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## Stationary Random Processes

### Definition

When the average statistical properties of a random process do not depend on the time index $k$, this process is termed as [*stationary random process*](https://en.wikipedia.org/wiki/Stationary_process). This can be expressed formally as

\begin{equation}
E\{ f(x[k_1], x[k_2], \dots) \} = E\{ f(x[k_1 + \Delta], x[k_2 + \Delta], \dots) \}
\end{equation}

where $\Delta \in \mathbb{Z}$ denotes an arbitrary shift of the sample index $k$. Above condition has to hold for all possible mapping functions $f(\cdot)$. From this definition it becomes clear that

* random signals of finite length and
* deterministic signals $s[k] \neq C$ with $C \in \mathbb{R}$

cannot be stationary random processes in a strict sense. However, in practice it is often assumed to be sufficient if above condition holds within a finite interval of sample indexes of a random signal.

### Cumulative Distribution Functions and Probability Density Functions

It follows from above definition of a stationary process, that the univariate cumulative distribution function (CDF) of a stationary random process does not depend on the time index $k$

\begin{equation}
P_x(\theta, k) = P_x(\theta)
\end{equation}

the same holds for the univariate probability density function (PDF)

\begin{equation}
p_x(\theta, k) = p_x(\theta)
\end{equation}

The bivariate CDF of two stationary random signals $x[k]$ and $y[k]$ depends only on the difference $\kappa = k_x - k_y$

\begin{equation}
P_{xy}(\theta_x, \theta_y, k_x, k_y) = P_{xy}(\theta_x, \theta_y, \kappa)
\end{equation}

The same holds for the bivariate PDF

\begin{equation}
p_{xy}(\theta_x, \theta_y, k_x, k_y) = p_{xy}(\theta_x, \theta_y, \kappa)
\end{equation}

### First-Order Ensemble Averages

According to above definition of stationarity, for the first-order ensemble average of a stationary process the following relation must hold

\begin{equation}
E\{ f(x[k]) \} = E\{ f(x[k + \Delta]) \}
\end{equation}

hence it does not depend on the time index $k$. For the linear mean we consequently yield

\begin{equation}
\mu_x[k] = \mu_x
\end{equation}

and for the variance

\begin{equation}
\sigma_x^2[k] = \sigma_x^2
\end{equation}

### Cross- and Auto-Correlation Function

It follows by introducing the PDF's properties of a stationary process into the definition of the cross-correlation function (CCF) and auto-correlation function (ACF) that

\begin{equation}
\varphi_{xy}[k_x, k_y] = \varphi_{xy}[\kappa] = E\{ x[k] \cdot y[k-\kappa]\} = E\{ x[k+\kappa] \cdot y[k]\}
\end{equation}

and

\begin{equation}
\varphi_{xx}[k_1, k_2] = \varphi_{xx}[\kappa] = E\{ x[k] \cdot x[k-\kappa]\} = E\{ x[k+\kappa] \cdot x[k]\}
\end{equation}

where for the latter $\kappa = k_1 - k_2$.

## Wide-Sense Stationary Random Processes

### Definition

The definition of a stationary random process in the previous section must hold for any mapping function $f(\cdot)$. In practice this cannot be checked in a strict sense. For a wide-sense stationary (WSS) random process the conditions for stationarity must hold only for linear mappings. This leads to the following two conditions a WSS random process has to fulfill

\begin{equation}
E\{ x[k_1] \cdot x[k_2] \} = E\{ x[k_1 + \Delta] \cdot x[k_2 + \Delta] \}
\end{equation}

and

\begin{equation}
E\{ x[k] \} = E\{ x[k + \Delta] \}
\end{equation}

A random signal of finite length cannot be WSS in a strict sense.

### Example - Evaluation of Wide-Sense Stationarity

From above definition of a WSS process it is evident that it is sufficient to check the time dependence of the linear mean $x_\mu[k]$ and the auto-correlation function $\varphi_{xx}[k_1, k_2]$. Both quantities are estimated and plotted for two different random processes.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

L = 64  # number of random samples
N = 1000  # number of sample functions

# generate sample functions
np.random.seed(1)
x = np.random.normal(size=(N, L))
x1 = x + np.tile(np.cos(2*np.pi/L*np.arange(L)), [N,1])
h = 2*np.fft.irfft([1,1,1,0,0,0])
x2 = np.asarray([np.convolve(x[n,:], h, mode='same') for n in range(N)])

# compute and plot results
def compute_plot_results(x):

    # estimate linear mean by ensemble average
    mu = 1/N * np.sum(x, 0)
    # estimate the auto-correlation function
    acf = np.zeros((L, L))
    for n in range(L):
        for m in range(L):
            acf[n, m] = 1/N * np.sum(x[:, n]*x[:, m], 0)

    plt.subplot(121)
    plt.stem(mu)
    plt.title(r'Estimate of linear mean $\hat{\mu}_x[k]$')
    plt.xlabel(r'$k$')
    plt.ylabel(r'$\hat{\mu}[k]$')
    plt.axis([0, L, -1.5, 1.5])

    plt.subplot(122)
    plt.pcolor(np.arange(L), np.arange(L), acf, vmin=-2, vmax=2)
    plt.title(r'Estimate of ACF $\hat{\varphi}_{xx}[k_1, k_2]$')
    plt.xlabel(r'$k_1$')
    plt.ylabel(r'$k_2$')
    plt.colorbar()
    plt.autoscale(tight=True)

plt.figure(figsize = (10, 4))
plt.gcf().suptitle('Random Process 1', fontsize=12, y=1.05)
compute_plot_results(x1)

plt.figure(figsize = (10, 4))
plt.gcf().suptitle('Random Process 2', fontsize=12, y=1.05)
compute_plot_results(x2)
```

**Exercise**

* Which process can be assumed to be WSS? Why?
* Increase the number `N` of sample functions. Do the results support your initial assumption?

Solution: Inspection of the estimated linear mean $\hat{\mu}_x[k]$ for the first random process reveals that it is highly dependent on the sample index $k$. The same holds for the ACF which depends on both sample indexes $k_1$ and $k_2$. For the second random process, the estimated linear mean can be assumed to be approximately zero. Its ACF depends approximately only on the difference $\kappa = k_1 - k_2$, as can be concluded from the diagonal structures in the plot. Increasing the number of sample functions $N$ increases the certainty of the estimates in the statistical sense.

## Higher-Order Temporal Averages

Ensemble averages are defined as the average across all sample functions $x_n[k]$ for a particular time index $k$. So far, averaging over the sample index $k$ of one sample function was not considered. Such averaging is termed commonly as temporal averaging since the sample index $k$ represents time in many cases. For a stationary process, the higher-order temporal average along the $n$-th sample function is defined as

\begin{equation}
\overline{ f(x_n[k], x_n[k-\kappa_1], x_n[k-\kappa_2], \dots) } = \lim_{K \to \infty} \frac{1}{2K + 1} \sum_{k = -K}^{K} f(x_n[k], x_n[k-\kappa_1], x_n[k-\kappa_2], \dots)
\end{equation}

where $\kappa_i \in \mathbb{Z} \; \forall i$. The introduced statistical measures, like the linear mean, variance and correlation functions, can be computed for each sample function by temporal averaging by using the respective mapping functions $f(\cdot)$ introduced above. The linear mean as temporal average of the $n$-th sample function $x_n[k]$ is for instance given by

\begin{equation}
\overline{x_n[k]} = \lim_{K \to \infty} \frac{1}{2K + 1} \sum_{k = -K}^{K} x_n[k]
\end{equation}

## Ergodic Random Processes

An [ergodic process](https://en.wikipedia.org/wiki/Ergodic_process) is a stationary random process whose higher-order temporal averages of all sample functions are equal to the ensemble averages

\begin{equation}
\overline{ f(x_n[k], x_n[k-\kappa_1], x_n[k-\kappa_2], \dots) } = E\{  f(x[k], x[k-\kappa_1], x[k-\kappa_2], \dots)  \} \;\; \forall n
\end{equation}

This implies that all higher-order temporal averages are equal. Any sample function from the process represents the average statistical properties of the entire process. The ensemble averages for a stationary and ergodic random process are given by the temporal averages of one sample function. This result is very important for the practical computation of statistical properties of random signals.

## Wide-Sense Ergodic Random Processes

### Definition

As for a WSS process, the conditions for ergodicity have to hold only for linear mappings $f(\cdot)$. Under the assumption of a WSS process, the following two conditions have to be met by a wide-sense ergodic random process

\begin{equation}
\overline{ x_n[k] \cdot x_n[k-\kappa] } = E\{  x[k] \cdot x[k-\kappa]  \} \;\; \forall n
\end{equation}

and

\begin{equation}
\overline{ x_n[k] } = E\{  x[k] \} \;\; \forall n
\end{equation}

### Example - Evaluation of Wide-Sense Ergodicity

In the following example, the linear mean and autocorrelation function are computed as temporal and ensemble averages for three random processes. The plots show the estimated temporal averages $\overline{ x_n[k] }$ and $\overline{ x_n[k] \cdot x_n[k-\kappa] }$ on the right-hand side of the sample functions $x_n[k]$. Note, the linear mean as temporal average is a scalar value whose value is illustrated by a bar plot. The estimated ensemble averages $E\{  x[k] \}$ and $E\{  x[k_1] \cdot x[k_2]  \}$ are shown below the sample functions to indicate the averaging across sample functions.

```python
L = 64  # number of random samples
N = 10000  # number of sample functions

# generate sample functions
np.random.seed(11)
x = np.random.normal(size=(N, L))
k = np.arange(L)
x1 = x + np.tile(np.cos(2*np.pi/L*k), [N, 1])
x2 = x + np.tile([np.ones(L), -np.ones(L)], [N//2, 1])
x3 = x + np.ones([N, L])

# function to compute and plot results
def compute_plot_results(x):

    # estimate linear mean by ensemble average
    mu = 1/N * np.sum(x, 0)
    # estimate the auto-correlation function by ensemble average
    acf = np.zeros((L, L))
    for n in range(L):
        for m in range(L):
            acf[n, m] = 1/N * np.sum(x[:, n]*x[:, m], 0)
    # estimate linear mean as temporal average
    mut = 1/L * np.sum(x, 1)
    # estimate the auto-correlation function as temporal average
    acft = np.zeros((N, L))
    for n in range(N):
        acft[n, :] = np.correlate(x[n, :], x[n, :], mode='same')
    kappa = np.arange(L) - L//2

    for n in range(2):
        plt.figure(figsize = (10, 5))
        plt.subplot(131)
        plt.stem(x[n, :])
        plt.title(r'Sample function $x_%d[k]$'%n)
        plt.xlabel(r'$k$')
        plt.ylabel(r'$x_%d[k]$'%n)
        plt.axis([0, L, -4, 4])

        plt.subplot(132)
        plt.bar(-0.4, mut[n])
        plt.title(r'Linear mean $\overline{ x_%d[k] }$'%n)
        plt.ylabel(r'$\hat{\mu}_{x,%d}$'%n)
        plt.axis([-.5, .5, -1.5, 1.5])

        plt.subplot(133)
        plt.stem(kappa, acft[n, :])
        plt.title(r'Autocorrelation $\overline{ x_%d[k] \cdot x_%d[k-\kappa] }$'%(n,n))
        plt.xlabel(r'$\kappa$')
        plt.ylabel(r'$\hat{\varphi}_{xx,%d}[\kappa]$'%n)
        plt.axis([-L//2, L//2, -30, 150])
        plt.tight_layout()

    plt.figure(figsize = (10, 5))
    plt.subplot(131)
    plt.stem(mu)
    plt.title(r'Linear mean $E\{  x[k] \}$')
    plt.xlabel(r'$k$')
    plt.ylabel(r'$\hat{\mu}[k]$')
    plt.axis([0, L, -1.5, 1.5])

    plt.figure(figsize = (4, 4))
    plt.pcolor(k, k, acf, vmin=-2, vmax=2)
    plt.title(r'ACF $E\{  x[k_1] \cdot x[k_2]  \}$')
    plt.xlabel(r'$k_1$')
    plt.ylabel(r'$k_2$')
    plt.colorbar()
    plt.autoscale(tight=True)
```

**Random Process 1**

```python
compute_plot_results(x1)
```

**Random Process 2**

```python
compute_plot_results(x2)
```

**Random Process 3**

```python
compute_plot_results(x3)
```

**Exercise**

* Which process can be assumed to be stationary and/or ergodic? Why?

Solution: The linear mean $\mu_x[k]$ estimated by the ensemble average of the first process depends on the sample index $k$. Consequently, this process is not stationary and also not ergodic. Inspection of the linear mean and auto-correlation function estimated by the ensemble averages of the second random process reveals that this processed can be assumed to be WSS. The linear mean as temporal average of the sample functions is not constant with respect to the sample functions. Consequently, this process can be assumed to be WSS but not ergodic. The third process can be assumed to be wide-sense stationary and ergodic.

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Digital Signal Processing - Lecture notes featuring computational examples, 2016-2018*.


# Diagrammatic Differentiation

**Quantum Group Workshop**, _26th January 2021_

Alexis Toumi, joint work with Richie Yeung and Giovanni de Felice

## 1) Parametrised matrices

Fix a rig $(\mathbb{S}, +, \times, 0, 1)$.

**Definition:** A matrix $f \in \mathbf{Mat}_\mathbb{S}(m, n)$ is a function $f : [m] \times [n] \to \mathbb{S}$ where $[n] = \{0, \dots, n - 1\}$ for $n \in \mathbb{N}$.

**Definition:** A parametrised matrix is a function $f : \mathbb{S} \to \mathbf{Mat}_\mathbb{S}(m, n)$, or equivalently a function-valued matrix $f \in \mathbf{Mat}_{\mathbb{S} \to \mathbb{S}}(m, n)$.

**Example:**

```python
from discopy import *
from sympy.abc import phi

Rz(phi).array
```

    array([[exp(-1.0*I*pi*phi), 0],
           [0, exp(1.0*I*pi*phi)]], dtype=object)

```python
(lambda phi: Rz(phi).array)(0.25)
```

    array([[0.70710678-0.70710678j, 0.        +0.j        ],
           [0.        +0.j        , 0.70710678+0.70710678j]])

## 2) Parametrised diagrams

Fix a monoidal signature $(\Sigma_0, \Sigma_1, \text{dom}, \text{cod} : \Sigma_1 \to \Sigma_0^\star)$
for $X^\star = \coprod_{n \in \mathbb{N}} X^n$ the free monoid.

**Definition:** An _abstract_ diagram $d \in \mathbf{C}_\Sigma(s, t)$ is defined by a list of
$\text{layers}(d) = (\text{left}_0, \text{box}_0, \text{right}_0), \dots, (\text{left}_n, \text{box}_n, \text{right}_n) \in \Sigma_0^\star \times \Sigma_1 \times \Sigma_0^\star$.

**Definition:** A _concrete_ diagram is an abstract diagram with a monoidal functor $F : \mathbf{C}_\Sigma \to \mathbf{Mat}_\mathbb{S}$.

**Definition:** A _parametrised_ diagram is a function $d : \mathbb{S} \to \mathbf{C}_\Sigma(s, t)$, or equivalently, a diagram with a monoidal functor $F : \mathbf{C}_\Sigma \to \mathbf{Mat}_{\mathbb{S} \to \mathbb{S}}$.

**Example:**

```python
from discopy import *

x, y, z = Dim(1), Dim(2), Dim(2)
f = tensor.Box('f', x, y, [phi + 1, -phi * 2])
g = tensor.Box('g', y @ y, z, [1, 0, 0, 0, 0, 0, 0, phi ** 2 + 1])

d = f @ f >> g
print(d)
d.draw(figsize=(2, 2))

d.eval().array
```

```python
d.subs(phi, 0.25).eval().array
```

    array([1.56250000000000, 0.265625000000000], dtype=object)

## 3) Product rule

We define the gradient of a parametrised matrix $f \in \mathbf{Mat}_{\mathbb{S} \to \mathbb{S}}(m, n)$ elementwise:
$$
\frac{\partial f}{\partial x}(i, j)
= \frac{\partial}{\partial x} f(i, j)
$$

Given a parametrised diagram $d, F$ we want a new diagram $\frac{\partial d}{\partial x}$ such that:
$$
F\big(\frac{\partial d}{\partial x}\big)
= \frac{\partial F(d)}{\partial x}
$$

We can do this by defining gradients as _formal sums of diagrams_ and using the product rule:
$$
\frac{\partial}{\partial x} (d \otimes d')
= \frac{\partial d}{\partial x} \otimes d'
+ d \otimes \frac{\partial d'}{\partial x}
$$
$$
\frac{\partial}{\partial x} (d \circ d')
= \frac{\partial d}{\partial x} \circ d'
+ d \circ \frac{\partial d'}{\partial x}
$$

**Example:**

```python
d.grad(phi).draw(figsize=(8, 3))
```

## 4) Chain rule

Given an arbitrary function $f : \mathbb{S} \to \mathbb{S}$,
we lift it to matrices by applying it elementwise.
Diagrammatically, we represent this as a _bubble_ around a subdiagram.

Gradients of bubbles are then given by the chain rule:

$$
\frac{\partial}{\partial x} f(d)
= \frac{\partial f}{\partial x} (d)
\times \frac{\partial d}{\partial x}
$$

where the elementwise product can be encoded as pre- and post-composition with spiders.

```python
g = tensor.Box('g', Dim(2), Dim(2), [2 * phi, 0, 0, phi + 1])
f = lambda d: d.bubble(func=lambda x: x ** 2, drawing_name="f")
lhs, rhs = tensor.Box.grad(f(g), phi), f(g).grad(phi)
drawing.equation(lhs, rhs, draw_type_labels=False)
```

## 5) Applications

* Gradients of quantum circuits using the parameter shift rule.
* Gradients of neural nets and classical post-processing with bubbles.
* Gradients of circuit functors for QNLP.