# API demonstration for paper of v1.0

_the LSST-DESC CLMM team_

Here we demonstrate how to use `clmm` to estimate a WL halo mass from observations of a galaxy cluster when source galaxies follow a given distribution (The LSST DESC Science Requirements Document - arXiv:1809.01669,  implemented in `clmm`). It uses several functionalities of the support `mock_data` module to produce mock datasets.

- Setting things up, with the proper imports.
- Computing the binned reduced tangential shear profile, for the 2 datasets, using logarithmic binning.
- Setting up a model accounting for the redshift distribution.
- Perform a simple fit using `scipy.optimize.curve_fit` included in `clmm` and visualize the results.

## Setup

First, we import some standard packages.

```python
import matplotlib.pyplot as plt
import numpy as np
plt.rcParams['font.family'] = ['gothambook','gotham','gotham-book','serif']
```

## Generating mock data

`clmm` has a support code to generate a mock catalog given a input cosmology and cluster parameters. We will use this to generate a data sample to be used in this example:

```python
from clmm import Cosmology
import clmm.support.mock_data as mock
```

```python
np.random.seed(14) # For reproducibility

# Set cosmology of mock data
cosmo = Cosmology(H0=70.0, Omega_dm0=0.27-0.045, Omega_b0=0.045, Omega_k0=0.0)

# Cluster info
cluster_m = 1.e15 # Cluster mass - ($M200_m$) [Msun]
concentration = 4  # Cluster concentration
cluster_z = 0.3 # Cluster redshift
cluster_ra = 0. # Cluster Ra in deg
cluster_dec = 0. # Cluster Dec in deg

# Catalog info
field_size = 10 # i.e. 10 x 10 Mpc field at the cluster redshift, cluster in the center

# Make mock galaxies
mock_galaxies = mock.generate_galaxy_catalog(
    cluster_m=cluster_m, cluster_z=cluster_z, cluster_c=concentration, # Cluster data
    cosmo=cosmo, # Cosmology object
    zsrc='desc_srd', # Galaxy redshift distribution,
    zsrc_min=0.4, # Minimum redshift of the galaxies
    shapenoise=0.05, # Gaussian shape noise to the galaxy shapes
    photoz_sigma_unscaled=0.05, # Photo-z errors to source redshifts
    field_size=field_size,
    ngal_density=20 # number of gal/arcmin2 for z in [0, infty]
)['ra', 'dec', 'e1', 'e2', 'z', 'ztrue', 'pzbins', 'pzpdf', 'id']
print(f'Catalog table with the columns: {", ".join(mock_galaxies.colnames)}')

ngals_init = len(mock_galaxies)
print(f'Initial number of galaxies: {ngals_init:,}')

# Keeping only galaxies with "measured" redshift greater than cluster redshift
mock_galaxies = mock_galaxies[(mock_galaxies['z']>cluster_z)]
ngals_good = len(mock_galaxies)

if ngals_good < ngals_init:
    print(f'Number of excluded galaxies (with photoz < cluster_z): {ngals_init-ngals_good:,}')
    # reset galaxy id for later use
    mock_galaxies['id'] = np.arange(ngals_good)

# Check final density
from clmm.utils import convert_units
field_size_arcmin = convert_units(field_size, 'Mpc', 'arcmin', redshift=cluster_z, cosmo=cosmo)
print(f'Background galaxy density = {ngals_good/field_size_arcmin**2:.2f} gal/arcmin2\n')
```

We can extract the column of this mock catalog to show explicitely how the quantities can be used on `clmm` functionality and how to add them to a `GalaxyCluster` object:

```python
# Put galaxy values on arrays
gal_ra = mock_galaxies['ra'] # Galaxies Ra in deg
gal_dec = mock_galaxies['dec'] # Galaxies Dec in deg
gal_e1 = mock_galaxies['e1'] # Galaxies elipticipy 1
gal_e2 = mock_galaxies['e2'] # Galaxies elipticipy 2
gal_z = mock_galaxies['z'] # Galaxies observed redshift
gal_ztrue = mock_galaxies['ztrue'] # Galaxies true redshift
gal_pzbins = mock_galaxies['pzbins'] # Galaxies P(z) bins
gal_pzpdf = mock_galaxies['pzpdf'] # Galaxies P(z)
gal_id = mock_galaxies['id'] # Galaxies ID
```

## Measuring shear profiles

From the source galaxy quantities, we can compute the elepticities and corresponding radial profile usimg `clmm.dataops` functions:

```python
import clmm.dataops as da
```

```python
# Convert elipticities into shears
gal_ang_dist, gal_gt, gal_gx = da.compute_tangential_and_cross_components(cluster_ra, cluster_dec,
                                                                          gal_ra, gal_dec,
                                                                          gal_e1, gal_e2,
                                                                          geometry="flat")

# Measure profile
profile = da.make_radial_profile([gal_gt, gal_gx, gal_z],
                                 gal_ang_dist, "radians", "Mpc",
                                 bins=da.make_bins(0.01, field_size/2., 50),
                                 cosmo=cosmo,
                                 z_lens=cluster_z,
                                 include_empty_bins=False)
print(f'Profile table has columns: {", ".join(profile.colnames)},')
print('where p_(0, 1, 2) = (gt, gx, z)')
```

The other possibility is to use the `GalaxyCluster` object. This is the main approach to handle data with `clmm`, and also the simpler way. For that you just have to provide the following information of the cluster:

* Ra, Dec [deg]
* Mass - ($M200_m$) [Msun]
* Concentration
* Redshift

and the source galaxies:

* Ra, Dec [deg]
* 2 axis of eliptticities
* Redshift

```python
import clmm
```

```python
# Create a GCData with the galaxies
galaxies = clmm.GCData([gal_ra, gal_dec, gal_e1, gal_e2, gal_z,
                        gal_ztrue, gal_pzbins, gal_pzpdf, gal_id],
                      names=['ra', 'dec', 'e1', 'e2', 'z',
                             'ztrue', 'pzbins', 'pzpdf', 'id'])

# Create a GalaxyCluster
cluster = clmm.GalaxyCluster("Name of cluster", cluster_ra, cluster_dec,
                                   cluster_z, mock_galaxies)

# Convert elipticities into shears for the members
cluster.compute_tangential_and_cross_components(geometry="flat")
print(cluster.galcat.colnames)

# Measure profile and add profile table to the cluster
seps = convert_units(cluster.galcat['theta'], 'radians', 'mpc',cluster.z, cosmo)

cluster.make_radial_profile(bins=da.make_bins(0.1, field_size/2., 25, method='evenlog10width'),
                            bin_units="Mpc",
                            cosmo=cosmo,
                            include_empty_bins=False,
                            gal_ids_in_bins=True,
                           )
print(cluster.profile.colnames)
```

This results in an attribute `table` added to the `cluster` object.

```python
from paper_formating import prep_plot
prep_plot(figsize=(9, 9))
errorbar_kwargs = dict(linestyle='', marker='o',
    markersize=1, elinewidth=.5, capthick=.5)
plt.errorbar(cluster.profile['radius'], cluster.profile['gt'],
             cluster.profile['gt_err'], c='k', **errorbar_kwargs)
plt.xlabel('r [Mpc]', fontsize = 10)
plt.ylabel(r'$g_t$', fontsize = 10)
plt.xscale('log')
plt.yscale('log')
```

## Theoretical predictions

We consider 3 models:
1. One model where all sources are considered at the same redshift
2. One model using the overall source redshift distribution to predict the reduced tangential shear
3. A more accurate model, relying on the fact that we have access to the individual redshifts of the sources, where the average reduced tangential shear is averaged independently in each bin, accounting for the acutal population of sources in each bin.

All models rely on `clmm.predict_reduced_tangential_shear` to make a prediction that accounts for the redshift distribution of the galaxies in each radial bin:

### Model considering all sources located at the average redshift
\begin{equation}
     g_{t,i}^{\rm{avg(z)}} = g_t(R_i, \langle z \rangle)\;,
 \label{eq:wrong_gt_model}
 \end{equation}

```python
def predict_reduced_tangential_shear_mean_z(profile, logm):
    return clmm.compute_reduced_tangential_shear(
            r_proj=profile['radius'], # Radial component of the profile
            mdelta=10**logm, # Mass of the cluster [M_sun]
            cdelta=4, # Concentration of the cluster
            z_cluster=cluster_z, # Redshift of the cluster
            z_source=np.mean(cluster.galcat['z']), # Mean value of source galaxies redshift
            cosmo=cosmo,
            delta_mdef=200,
            halo_profile_model='nfw'
        )
```

### Model relying on the overall redshift distribution of the sources N(z), not using individual redshift information (eq. (6) from Applegate et al. 2014, MNRAS, 439, 48)
\begin{equation}
     g_{t,i}^{N(z)} = \frac{\langle\beta_s\rangle \gamma_t(R_i, z\rightarrow\infty)}{1-\frac{\langle\beta_s^2\rangle}{\langle\beta_s\rangle}\kappa(R_i, z\rightarrow\infty)}
     \label{eq:approx_model}
 \end{equation}

```python
z_inf = 1000
dl_inf = cosmo.eval_da_z1z2(cluster_z, z_inf)
d_inf = cosmo.eval_da(z_inf)

def betas(z):
    dls = cosmo.eval_da_z1z2(cluster_z, z)
    ds = cosmo.eval_da(z)
    return dls * d_inf / (ds * dl_inf)

def predict_reduced_tangential_shear_approx(profile, logm):

    bs_mean = np.mean(betas(cluster.galcat['z']))
    bs2_mean = np.mean(betas(cluster.galcat['z'])**2)

    gamma_t_inf = clmm.compute_tangential_shear(
            r_proj=profile['radius'], # Radial component of the profile
            mdelta=10**logm, # Mass of the cluster [M_sun]
            cdelta=4, # Concentration of the cluster
            z_cluster=cluster_z, # Redshift of the cluster
            z_source=z_inf, # Redshift value at infinity
            cosmo=cosmo,
            delta_mdef=200,
            halo_profile_model='nfw')
    convergence_inf = clmm.compute_convergence(
            r_proj=profile['radius'], # Radial component of the profile
            mdelta=10**logm, # Mass of the cluster [M_sun]
            cdelta=4, # Concentration of the cluster
            z_cluster=cluster_z, # Redshift of the cluster
            z_source=z_inf, # Redshift value at infinity
            cosmo=cosmo,
            delta_mdef=200,
            halo_profile_model='nfw')

    return bs_mean*gamma_t_inf/(1-(bs2_mean/bs_mean)*convergence_inf)
```

### Model using individual redshift and radial information, to compute the averaged shear in each radial bin, based on the galaxies actually present in that bin.
\begin{equation}
    g_{t,i}^{z, R} = \frac{1}{N_i}\sum_{{\rm gal\,}j\in {\rm bin\,}i} g_t(R_j, z_j)
    \label{eq:exact_model}
 \end{equation}

```python
cluster.galcat['theta_mpc'] = convert_units(cluster.galcat['theta'], 'radians', 'mpc',cluster.z, cosmo)

def predict_reduced_tangential_shear_exact(profile, logm):
    return np.array([np.mean(
        clmm.compute_reduced_tangential_shear(
            # Radial component of each source galaxy inside the radial bin
            r_proj=cluster.galcat[radial_bin['gal_id']]['theta_mpc'],
            mdelta=10**logm, # Mass of the cluster [M_sun]
            cdelta=4, # Concentration of the cluster
            z_cluster=cluster_z, # Redshift of the cluster
            # Redshift value of each source galaxy inside the radial bin
            z_source=cluster.galcat[radial_bin['gal_id']]['z'],
            cosmo=cosmo,
            delta_mdef=200,
            halo_profile_model='nfw'
        )) for radial_bin in profile])
```

## Mass fitting

We estimate the best-fit mass using `scipy.optimize.curve_fit`. The choice of fitting $\log M$ instead of $M$ lowers the range of pre-defined fitting bounds from several order of magnitude for the mass to unity. From the associated error $\sigma_{\log M}$ we calculate the error to mass as $\sigma_M = M_{fit}\ln(10)\sigma_{\log M}$.

#### First, identify bins with sufficient galaxy statistics to be kept for the fit
For small samples, error bars should not be computed using the simple error on the mean approach available so far in CLMM)

```python
mask_for_fit = cluster.profile['n_src'] > 5
data_for_fit = cluster.profile[mask_for_fit]
```

#### Perform the fits

```python
from clmm.support.sampler import fitters
def fit_mass(predict_function):
    popt, pcov = fitters['curve_fit'](predict_function,
        data_for_fit,
        data_for_fit['gt'],
        data_for_fit['gt_err'], bounds=[10.,17.])
    logm, logm_err = popt[0], np.sqrt(pcov[0][0])
    return {'logm':logm, 'logm_err':logm_err,
            'm': 10**logm, 'm_err': (10**logm)*logm_err*np.log(10)}
```

```python
fit_mean_z = fit_mass(predict_reduced_tangential_shear_mean_z)
fit_approx = fit_mass(predict_reduced_tangential_shear_approx)
fit_exact = fit_mass(predict_reduced_tangential_shear_exact)
```

```python
print(f'Input mass = {cluster_m:.2e} Msun\n')

print(f'Best fit mass for average redshift               = {fit_mean_z["m"]:.3e} +/- {fit_mean_z["m_err"]:.3e} Msun')
print(f'Best fit mass for N(z) model                     = {fit_approx["m"]:.3e} +/- {fit_approx["m_err"]:.3e} Msun')
print(f'Best fit mass for individual redshift and radius = {fit_exact["m"]:.3e} +/- {fit_exact["m_err"]:.3e} Msun')
```

As expected, the reconstructed mass is biased when the redshift distribution is not accounted for in the model

## Visualization of the results

For visualization purpose, we calculate the reduced tangential shear predicted by the model with estimated masses for noisy and ideal data.

```python
def get_predicted_shear(predict_function, fit_values):
    gt_est = predict_function(data_for_fit, fit_values['logm'])
    gt_est_err = [predict_function(data_for_fit, fit_values['logm']+i*fit_values['logm_err'])
                          for i in (-3, 3)]
    return gt_est, gt_est_err
```

```python
gt_mean_z, gt_err_mean_z =  get_predicted_shear(predict_reduced_tangential_shear_mean_z, fit_mean_z)
gt_approx, gt_err_approx =  get_predicted_shear(predict_reduced_tangential_shear_approx, fit_approx)
gt_exact, gt_err_exact =  get_predicted_shear(predict_reduced_tangential_shear_exact, fit_exact)
```

Check reduced chi2 values of the best-fit model

```python
chi2_mean_z_dof = np.sum((gt_mean_z-data_for_fit['gt'])**2/(data_for_fit['gt_err'])**2)/(len(data_for_fit)-1)
chi2_approx_dof = np.sum((gt_approx-data_for_fit['gt'])**2/(data_for_fit['gt_err'])**2)/(len(data_for_fit)-1)
chi2_exact_dof = np.sum((gt_exact-data_for_fit['gt'])**2/(data_for_fit['gt_err'])**2)/(len(data_for_fit)-1)

print(f'Reduced chi2 (mean z model) = {chi2_mean_z_dof}')
print(f'Reduced chi2 (N(z) model) = {chi2_approx_dof}')
print(f'Reduced chi2 (individual (R,z) model) = {chi2_exact_dof}')
```

We compare to tangential shear obtained with theoretical mass. We plot the reduced tangential shear models first when redshift distribution is accounted for in the model then for the naive approach, with respective best-fit masses.

```python
from matplotlib.ticker import MultipleLocator
prep_plot(figsize=(9 , 9))
gt_ax = plt.axes([.25, .42, .7, .55])
gt_ax.errorbar(data_for_fit['radius'],data_for_fit['gt'], data_for_fit['gt_err'],
             c='k', label=rf'$M_{{input}} = {cluster_m*1e-15}\times10^{{{15}}} M_\odot$',
            **errorbar_kwargs)

# Points in grey have not been used for the fit
gt_ax.errorbar(cluster.profile['radius'][~mask_for_fit], cluster.profile['gt'][~mask_for_fit],
               cluster.profile['gt_err'][~mask_for_fit],
               c='grey',**errorbar_kwargs)

pow10 = 15
mlabel = lambda name, fits: fr'$M_{{fit}}^{{{name}}} = {fits["m"]/10**pow10:.3f}\pm{fits["m_err"]/10**pow10:.3f}\times 10^{{{pow10}}} M_\odot$'
# Avg z
gt_ax.loglog(data_for_fit['radius'], gt_mean_z,'-C0',
           label=mlabel('avg(z)', fit_mean_z),lw=.5)
gt_ax.fill_between(data_for_fit['radius'], *gt_err_mean_z, lw=0, color='C0', alpha=.2)
# Approx model
gt_ax.loglog(data_for_fit['radius'], gt_approx,'-C1',
           label=mlabel('N(z)', fit_approx),
             lw=.5)
gt_ax.fill_between(data_for_fit['radius'], *gt_err_approx, lw=0, color='C1', alpha=.2)
# Exact model
gt_ax.loglog(data_for_fit['radius'], gt_exact,'-C2',
           label=mlabel('z,R', fit_exact),
             lw=.5)
gt_ax.fill_between(data_for_fit['radius'], *gt_err_exact, lw=0, color='C2', alpha=.2)

gt_ax.set_ylabel(r'$g_t$', fontsize = 8)
gt_ax.legend(fontsize=6)
gt_ax.set_xticklabels([])
gt_ax.tick_params('x', labelsize=8)
gt_ax.tick_params('y', labelsize=8)

#gt_ax.set_yscale('log')
errorbar_kwargs2 = {k:v for k, v in errorbar_kwargs.items() if 'marker' not in k}
errorbar_kwargs2['markersize'] = 3
errorbar_kwargs2['markeredgewidth'] = .5
res_ax = plt.axes([.25, .2, .7, .2])
delta = (cluster.profile['radius'][1]/cluster.profile['radius'][0])**.25
res_err = data_for_fit['gt_err']/data_for_fit['gt']
res_ax.errorbar(data_for_fit['radius']/delta, gt_mean_z/data_for_fit['gt']-1,
                yerr=res_err, marker='.', c='C0', **errorbar_kwargs2)
errorbar_kwargs2['markersize'] = 1.5
res_ax.errorbar(data_for_fit['radius'], gt_approx/data_for_fit['gt']-1,
                yerr=res_err, marker='s', c='C1', **errorbar_kwargs2)
errorbar_kwargs2['markersize'] = 3
errorbar_kwargs2['markeredgewidth'] = .5
res_ax.errorbar(data_for_fit['radius']*delta, gt_exact/data_for_fit['gt']-1,
                yerr=res_err, marker='*', c='C2', **errorbar_kwargs2)
res_ax.set_xlabel(r'$R$ [Mpc]', fontsize = 8)

res_ax.set_ylabel(r'$g_t^{mod.}/g_t^{data}-1$', fontsize = 8)
res_ax.set_xscale('log')
res_ax.set_xlim(gt_ax.get_xlim())
res_ax.set_ylim(-0.65,0.65)
res_ax.yaxis.set_minor_locator(MultipleLocator(.1))

res_ax.tick_params('x', labelsize=8)
res_ax.tick_params('y', labelsize=8)

for p in (gt_ax, res_ax):
    p.xaxis.grid(True, which='major', lw=.5)
    p.yaxis.grid(True, which='major', lw=.5)
    p.xaxis.grid(True, which='minor', lw=.1)
    p.yaxis.grid(True, which='minor', lw=.1)

plt.savefig('r_gt.png')
```


# Statistical Data Modeling

Some or most of you have probably taken some undergraduate- or graduate-level statistics courses. Unfortunately, the curricula for most introductory statisics courses are mostly focused on conducting **statistical hypothesis tests** as the primary means for interest: t-tests, chi-squared tests, analysis of variance, etc. Such tests seek to esimate whether groups are "significantly different "or effects are "statistically significant", a concept that is poorly understood, and hence, often misused by practioners. Even when interpreted *correctly*, statistical significance (as characterized by a small p-value) is a questionable goal for statistical inference, as it is not a measure of evidence in any statistical sense.

A far more powerful approach to statistical analysis involves building flexible **models** with the overarching aim of *estimating* quantities of interest. This section of the tutorial illustrates how to use Python to build statistical models of low to moderate difficulty from scratch, and use them to extract estimates and associated measures of uncertainty. These estimates can then be passed on to individuals with domain expertise who can then appraise them for "real-world" significance.

```
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm

# Set some Pandas options
pd.set_option('display.notebook_repr_html', False)
pd.set_option('display.max_columns', 20)
pd.set_option('display.max_rows', 25)
```

Estimation
==========

An recurring statistical problem is finding estimates of the relevant parameters that correspond to the distribution that best represents our data.

In **parametric** inference, we specify *a priori* a suitable distribution, then choose the parameters that best fit the data.

* e.g. $\mu$ and $\sigma^2$ in the case of the normal distribution

```
x = np.array([ 1.00201077,  1.58251956,  0.94515919,  6.48778002,  1.47764604,
        5.18847071,  4.21988095,  2.85971522,  3.40044437,  3.74907745,
        1.18065796,  3.74748775,  3.27328568,  3.19374927,  8.0726155 ,
        0.90326139,  2.34460034,  2.14199217,  3.27446744,  3.58872357,
        1.20611533,  2.16594393,  5.56610242,  4.66479977,  2.3573932 ])
_ = plt.hist(x, bins=10)
```

### Fitting data to probability distributions

We start with the problem of finding values for the parameters that provide the best fit between the model and the data, called point estimates. First, we need to define what we mean by ‘best fit’. There are two commonly used criteria:

* **Method of moments** chooses the parameters so that the sample moments (typically the sample mean and variance) match the theoretical moments of our chosen distribution.
* **Maximum likelihood** chooses the parameters to maximize the likelihood, which measures how likely it is to observe our given sample.

### Discrete Random Variables

$$X = \{0,1\}$$

$$Y = \{\ldots,-2,-1,0,1,2,\ldots\}$$

**Probability Mass Function**:

For discrete $X$,

$$Pr(X=x) = f(x|\theta)$$

***e.g. Poisson distribution***

The Poisson distribution models unbounded counts:

$$Pr(X=x)=\frac{e^{-\lambda}\lambda^x}{x!}$$

* $X=\{0,1,2,\ldots\}$
* $\lambda > 0$

$$E(X) = \text{Var}(X) = \lambda$$

### Continuous Random Variables

$$X \in [0,1]$$

$$Y \in (-\infty, \infty)$$

**Probability Density Function**:

For continuous $X$,

$$Pr(x \le X \le x + dx) = f(x|\theta)dx \, \text{ as } \, dx \rightarrow 0$$

***e.g. normal distribution***

$$f(x) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left[-\frac{(x-\mu)^2}{2\sigma^2}\right]$$

* $X \in \mathbf{R}$
* $\mu \in \mathbf{R}$
* $\sigma>0$

$$\begin{align}E(X) &= \mu \cr
\text{Var}(X) &= \sigma^2 \end{align}$$

### Example: Nashville Precipitation

The dataset `nashville_precip.txt` contains [NOAA precipitation data for Nashville measured since 1871](http://bit.ly/nasvhville_precip_data).

```
precip = pd.read_table("data/nashville_precip.txt", index_col=0,
                       na_values='NA', delim_whitespace=True)
precip.head()
```

           Jan   Feb   Mar    Apr   May   Jun   Jul   Aug   Sep   Oct   Nov   Dec
    Year
    1871  2.76  4.58  5.01   4.13  3.30  2.98  1.58  2.36  0.95  1.31  2.13  1.65
    1872  2.32  2.11  3.14   5.91  3.09  5.17  6.10  1.65  4.50  1.58  2.25  2.38
    1873  2.96  7.14  4.11   3.59  6.31  4.20  4.63  2.36  1.81  4.28  4.36  5.94
    1874  5.22  9.23  5.36  11.84  1.49  2.87  2.65  3.52  3.12  2.63  6.12  4.19
    1875  6.15  3.06  8.14   4.22  1.73  5.63  8.12  1.60  3.79  1.25  5.46  4.30

    [5 rows x 12 columns]

```
_ = precip.hist(sharex=True, sharey=True, grid=False, figsize=(10,6))
plt.tight_layout()
```

The first step is recognixing what sort of distribution to fit our data to. A couple of observations:

1. The data are skewed, with a longer tail to the right than to the left
2. The data are positive-valued, since they are measuring rainfall
3. The data are continuous

The **gamma distribution** is often a good fit to aggregated rainfall data, and will be our candidate distribution in this case.

$$x \sim \text{Gamma}(\alpha, \beta) = \frac{\beta^{\alpha}x^{\alpha-1}e^{-\beta x}}{\Gamma(\alpha)}$$

The ***method of moments*** simply assigns the empirical mean and variance to their theoretical counterparts, so that we can solve for the parameters.

So, for the gamma distribution, the mean and variance are:

$$ \hat{\mu} = \bar{X} = \alpha \beta $$
$$ \hat{\sigma}^2 = S^2 = \alpha \beta^2 $$

```
from IPython.display import IFrame
IFrame('http://en.wikipedia.org/wiki/Gamma_distribution?useformat=mobile', width=600, height=350)
```

So, if we solve for these parameters, we can use a gamma distribution to describe our data:

$$ \alpha = \frac{\bar{X}^2}{S^2}, \, \beta = \frac{S^2}{\bar{X}} $$

Let's deal with the missing value in the October data.

```
precip.Oct.ix[1960:1970]
```

    Year
    1960    1.38
    1961    1.12
    1962    2.29
    1963     NaN
    1964    1.83
    1965    0.57
    1966    2.50
    1967    1.57
    1968    3.92
    1969    2.01
    1970    2.94
    Name: Oct, dtype: float64

Given what we are trying to do, it is most sensible to fill in the missing value with the average of the available values.

```
precip.fillna(value={'Oct': precip.Oct.mean()}, inplace=True)
```

            Jan    Feb    Mar    Apr   May   Jun   Jul   Aug   Sep   Oct   Nov  \
    Year
    1871   2.76   4.58   5.01   4.13  3.30  2.98  1.58  2.36  0.95  1.31  2.13
    1872   2.32   2.11   3.14   5.91  3.09  5.17  6.10  1.65  4.50  1.58  2.25
    1873   2.96   7.14   4.11   3.59  6.31  4.20  4.63  2.36  1.81  4.28  4.36
    1874   5.22   9.23   5.36  11.84  1.49  2.87  2.65  3.52  3.12  2.63  6.12
    1875   6.15   3.06   8.14   4.22  1.73  5.63  8.12  1.60  3.79  1.25  5.46
    1876   6.41   2.22   5.28   3.62  3.40  5.65  7.15  5.77  2.52  2.68  1.26
    1877   4.05   1.06   4.98   9.47  1.25  6.02  3.25  4.16  5.40  2.61  4.93
    1878   3.34   2.10   3.48   6.88  2.33  3.28  9.43  5.02  1.28  2.17  3.20
    1879   6.32   3.13   3.81   2.88  2.88  2.50  8.47  4.62  5.18  2.90  5.85
    1880   3.74  12.37   8.16   5.26  4.13  3.97  5.69  2.22  5.39  7.24  5.77
    1881   3.54   5.48   2.79   5.12  3.67  3.70  0.86  1.81  6.57  4.80  4.89
    1882  14.51   8.61   9.38   3.59  7.38  2.54  4.06  5.54  1.61  1.11  3.60
    1883   3.76   7.90   3.98   9.12  4.82  3.82  4.94  4.47  2.23  5.27  3.11
    1884   7.20   8.18   8.89   3.51  3.58  6.53  3.18  2.81  2.36  2.43  1.57
    1885   6.29   2.00   2.33   3.75  4.36  3.72  5.26  1.02  5.60  2.99  2.73
    1886   5.18   3.82   4.76   2.36  2.10  7.69  1.90  5.50  3.68  0.51  5.76
    1887   5.13   8.47   3.36   2.67  3.43  2.31  3.77  2.89  6.85  1.92  2.29
    1888   6.29   3.78   6.46   4.18  2.97  4.68  2.36  7.03  3.82  2.82  4.33
    1889   3.83   1.84   2.47   2.83  5.30  5.33  2.74  1.57  6.81  1.54  6.88
    1890   8.10  10.95   8.64   3.84  4.16  2.23  0.46  6.59  5.86  3.01  2.01
    1891   6.15   6.96  10.31   2.24  2.39  6.50  1.49  3.72  1.25  0.84  6.71
    1892   2.81   2.73   4.10   7.45  4.03  5.01  5.13  3.39  4.78  0.25  3.91
    1893   1.27   4.88   3.37   4.11  7.31  4.74  2.12  1.92  6.43  3.68  2.97
    1894   4.28   8.65   2.69   4.05  2.53  3.55  5.45  2.43  3.07  0.53  1.92
    1895   5.71   0.98   5.09   3.07  2.05  2.90  7.14  1.40  6.69  1.57  2.14
            ...    ...    ...    ...   ...   ...   ...   ...   ...   ...   ...

           Dec
    Year
    1871  1.65
    1872  2.38
    1873  5.94
    1874  4.19
    1875  4.30
    1876  0.95
    1877  2.49
    1878  6.04
    1879  9.15
    1880  3.32
    1881  4.85
    1882  1.52
    1883  4.97
    1884  3.78
    1885  2.90
    1886  1.48
    1887  5.31
    1888  1.77
    1889  1.17
    1890  4.12
    1891  4.26
    1892  6.43
    1893  3.50
    1894  2.81
    1895  4.09
           ...

    [141 rows x 12 columns]

Now, let's calculate the sample moments of interest, the means and variances by month:

```
precip_mean = precip.mean()
precip_mean
```

    Jan    4.523688
    Feb    4.097801
    Mar    4.977589
    Apr    4.204468
    May    4.325674
    Jun    3.873475
    Jul    3.895461
    Aug    3.367305
    Sep    3.377660
    Oct    2.610500
    Nov    3.685887
    Dec    4.176241
    dtype: float64

```
precip_var = precip.var()
precip_var
```

    The history saving thread hit an unexpected error (OperationalError('disk I/O error',)).History will not be written to the database.

    Jan    6.928862
    Feb    5.516660
    Mar    5.365444
    Apr    4.117096
    May    5.306409
    Jun    5.033206
    Jul    3.777012
    Aug    3.779876
    Sep    4.940099
    Oct    2.741659
    Nov    3.679274
    Dec    5.418022
    dtype: float64

We then use these moments to estimate $\alpha$ and $\beta$ for each month:

```
alpha_mom = precip_mean ** 2 / precip_var
beta_mom = precip_var / precip_mean
```

```
estimates = pd.DataFrame({'alpha_mom': alpha_mom, 'beta_mom': beta_mom})
estimates
```

         alpha_mom  beta_mom
    Jan   2.953407  1.531684
    Feb   3.043866  1.346249
    Mar   4.617770  1.077920
    Apr   4.293694  0.979219
    May   3.526199  1.226724
    Jun   2.980965  1.299403
    Jul   4.017624  0.969593
    Aug   2.999766  1.122522
    Sep   2.309383  1.462581
    Oct   2.485616  1.050243
    Nov   3.692511  0.998206
    Dec   3.219070  1.297344

    [12 rows x 2 columns]

We can use the `gamma.pdf` function in `scipy.stats.distributions` to plot the ditribtuions implied by the calculated alphas and betas. For example, here is January:

```
from scipy.stats import gamma

xvals = np.linspace(0, 10)
yvals = gamma.pdf(xvals, alpha_mom[0], beta_mom[0])
precip.Jan.hist(normed=True, bins=20)
plt.plot(xvals, yvals)
```

Looping over all months, we can create a grid of plots for the distribution of rainfall, using the gamma distribution:

```
axs = precip.hist(normed=True, figsize=(12, 8), sharex=True, sharey=True, bins=15, grid=False)

for ax in axs.ravel():

    # Get month
    m = ax.get_title()

    # Plot fitted distribution
    x = np.linspace(*ax.get_xlim())
    ax.plot(x, gamma.pdf(x, alpha_mom[m], beta_mom[m]))

    # Annotate with parameter estimates
    label = 'alpha = {0:.2f}\nbeta = {1:.2f}'.format(alpha_mom[m], beta_mom[m])
    ax.annotate(label, xy=(10, 0.2))

plt.tight_layout()
```

Maximum Likelihood
==================

**Maximum likelihood** (ML) fitting is usually more work than the method of moments, but it is preferred as the resulting estimator is known to have good theoretical properties.

There is a ton of theory regarding ML. We will restrict ourselves to the mechanics here.

Say we have some data $y = y_1,y_2,\ldots,y_n$ that is distributed according to some distribution:

$$Pr(Y_i=y_i | \theta)$$

Here, for example, is a **Poisson distribution** that describes the distribution of some discrete variables, typically *counts*:

```
y = np.random.poisson(5, size=100)
plt.hist(y, bins=np.sqrt(len(y)), normed=True)
plt.xlabel('y'); plt.ylabel('Pr(y)')
```

The product $\prod_{i=1}^n Pr(y_i | \theta)$ gives us a measure of how **likely** it is to observe the set of values $y_1,\ldots,y_n$ given the parameters $\theta$. Maximum likelihood fitting consists of choosing the appropriate function $l= Pr(Y|\theta)$ to maximize for a given set of observations. We call this function the *likelihood function*, because it is a measure of how likely the observations are if the model is true.

> Given these data, how likely is this model?

In the above model, the data were drawn from a Poisson distribution with parameter $\lambda =5$.

$$L(y|\lambda=5) = \frac{e^{-5} 5^y}{y!}$$

So, for any given value of $y$, we can calculate its likelihood:

```
poisson_like = lambda x, lam: np.exp(-lam) * (lam**x) / (np.arange(x)+1).prod()

lam = 6
value = 10
poisson_like(value, lam)
```

    0.041303093412337726

```
np.sum(poisson_like(yi, lam) for yi in y)
```

    12.681664407577044

```
lam = 8
np.sum(poisson_like(yi, lam) for yi in y)
```

    8.058131431782682

We can plot the likelihood function for any value of the parameter(s):

```
lambdas = np.linspace(0,15)
x = 5
plt.plot(lambdas, [poisson_like(x, l) for l in lambdas])
plt.xlabel('$\lambda$')
plt.ylabel('L($\lambda$|x={0})'.format(x));
```

How is the likelihood function different than the probability distribution (or mass) function? The likelihood is a function of the parameter(s) *given the data*, whereas the PDF returns the probability of data given a particular parameter value. Here is the PMF of the Poisson for $\lambda=5$.

```
lam = 5
xvals = np.arange(15)
yvals = [poisson_like(x, lam) for x in xvals]
plt.bar(xvals, yvals)
plt.xlabel('x')
plt.ylabel('Pr(X|$\lambda$=5)');
```

Why are we interested in the likelihood function?

A reasonable estimate of the true, unknown value for the parameter is one which **maximizes the likelihood function**. So, inference is reduced to an optimization problem.

Going back to the rainfall data, if we are using a gamma distribution we need to maximize:

$$\begin{align}l(\alpha,\beta) &= \sum_{i=1}^n \log[\beta^{\alpha} x^{\alpha-1} e^{-x/\beta}\Gamma(\alpha)^{-1}] \cr
&= n[(\alpha-1)\overline{\log(x)} - \bar{x}\beta + \alpha\log(\beta) - \log\Gamma(\alpha)]\end{align}$$

(*Its usually easier to work in the log scale*)

where $n = 2012 − 1871 = 141$ and the bar indicates an average over all *i*. We choose $\alpha$ and $\beta$ to maximize $l(\alpha,\beta)$.

Notice $l$ is infinite if any $x$ is zero. We do not have any zeros, but we do have an NA value for one of the October data, which we dealt with above.

### Finding the MLE

To find the maximum of any function, we typically take the *derivative* with respect to the variable to be maximized, set it to zero and solve for that variable.

$$\frac{\partial l(\alpha,\beta)}{\partial \beta} = n\left(\frac{\alpha}{\beta} - \bar{x}\right) = 0$$

Which can be solved as $\beta = \alpha/\bar{x}$. However, plugging this into the derivative with respect to $\alpha$ yields:

$$\frac{\partial l(\alpha,\beta)}{\partial \alpha} = \log(\alpha) + \overline{\log(x)} - \log(\bar{x}) - \frac{\Gamma(\alpha)'}{\Gamma(\alpha)} = 0$$

This has no closed form solution. We must use ***numerical optimization***!

Numerical optimization alogarithms take an initial "guess" at the solution, and iteratively improve the guess until it gets "close enough" to the answer.

Here, we will use Newton-Raphson algorithm:

$$x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)}$$

Which is available to us via SciPy:

```
from scipy.optimize import newton
```

Here is a graphical example of how Newton-Raphson converges on a solution, using an arbitrary function:

```
# some function
func = lambda x: 3./(1 + 400*np.exp(-2*x)) - 1
xvals = np.linspace(0, 6)
plt.plot(xvals, func(xvals))
plt.text(5.3, 2.1, '$f(x)$', fontsize=16)

# zero line
plt.plot([0,6], [0,0], 'k-')

# value at step n
plt.plot([4,4], [0,func(4)], 'k:')
plt.text(4, -.2, '$x_n$', fontsize=16)

# tangent line
tanline = lambda x: -0.858 + 0.626*x
plt.plot(xvals, tanline(xvals), 'r--')

# point at step n+1
xprime = 0.858/0.626
plt.plot([xprime, xprime], [tanline(xprime), func(xprime)], 'k:')
plt.text(xprime+.1, -.2, '$x_{n+1}$', fontsize=16)
```

To apply the Newton-Raphson algorithm, we need a function that returns a vector containing the **first and second derivatives** of the function with respect to the variable of interest. In our case, this is:

```
from scipy.special import psi, polygamma

dlgamma = lambda m, log_mean, mean_log: np.log(m) - psi(m) - log_mean + mean_log
dl2gamma = lambda m, *args: 1./m - polygamma(1, m)
```

where `log_mean` and `mean_log` are $\log{\bar{x}}$ and $\overline{\log(x)}$, respectively. `psi` and `polygamma` are complex functions of the Gamma function that result when you take first and second derivatives of that function.

```
# Calculate statistics
log_mean = precip.mean().apply(np.log)
mean_log = precip.apply(np.log).mean()
```

Time to optimize!

```
# Alpha MLE for December
alpha_mle = newton(dlgamma, 2, dl2gamma, args=(log_mean[-1], mean_log[-1]))
alpha_mle
```

    3.5189679152399647

And now plug this back into the solution for beta:

$$ \beta  = \frac{\alpha}{\bar{X}} $$

```
beta_mle = alpha_mle/precip.mean()[-1]
beta_mle
```

    0.84261607548413797

We can compare the fit of the estimates derived from MLE to those from the method of moments:

```
dec = precip.Dec
dec.hist(normed=True, bins=10, grid=False)
x = np.linspace(0, dec.max())
plt.plot(x, gamma.pdf(x, alpha_mom[-1], beta_mom[-1]), 'm-')
plt.plot(x, gamma.pdf(x, alpha_mle, beta_mle), 'r--')
```

For some common distributions, SciPy includes methods for fitting via MLE:

```
from scipy.stats import gamma

ahat, _, bhat = gamma.fit(precip.Dec, floc=0)
ahat, 1./bhat
```

    (3.5189679152399753, 0.84261607548414119)

Note that SciPy's `gamma.fit` method fits a slightly different version of the gamma distribution, which has to be transformed to match ours.

### Kernel density estimates

In some instances, we may not be interested in the parameters of a particular distribution of data, but just a smoothed representation of the data at hand. In this case, we can estimate the disribution *non-parametrically* (i.e. making no assumptions about the form of the underlying distribution) using kernel density estimation.

```
# Some random data
y = np.random.random(15) * 10
y
```

    array([ 3.72458625,  8.02504424,  2.5152199 ,  1.15249687,  3.69592197,
            4.81974995,  7.21932052,  4.78104845,  2.67680421,  9.90455954,
            8.162251  ,  7.12880674,  6.79736262,  5.98943549,  7.99250998])

```
from scipy.stats import norm

# Create an array of x-valuese``````````````````````````````
x = np.linspace(y.min()-1, y.max()+1, 100)

# Smoothing parameter
s = 0.4

# Calculate the kernels
kernels = np.transpose([norm.pdf(x, yi, s) for yi in y])
plt.plot(x, kernels, 'k:')
plt.plot(x, kernels.sum(1))
plt.plot(y, np.zeros(len(y)), 'ro', ms=10)
```

SciPy implements a Gaussian KDE that automatically chooses an appropriate bandwidth. Let's create a bi-modal distribution of data that is not easily summarized by a parametric distribution:

```
# Create a bi-modal distribution with a mixture of Normals.
x1 = np.random.normal(0, 2, 50)
x2 = np.random.normal(4, 1, 50)

# Append by row
x = np.r_[x1, x2]
```

```
plt.hist(x, bins=10, normed=True)
```

```
from scipy.stats import kde

density = kde.gaussian_kde(x)
xgrid = np.linspace(x.min(), x.max(), 100)
plt.hist(x, bins=10, normed=True)
plt.plot(xgrid, density(xgrid), 'r-')
```

## Regression models

A general, primary goal of many statistical data analysis tasks is to relate the influence of one variable on another. For example, we may wish to know how different medical interventions influence the incidence or duration of disease, or perhaps a how baseball player's performance varies as a function of age.

```
x = np.array([2.2, 4.3, 5.1, 5.8, 6.4, 8.0])
y = np.array([0.4, 10.1, 14.0, 10.9, 15.4, 18.5])
plt.plot(x,y,'ro')
```

We can build a model to characterize the relationship between $X$ and $Y$, recognizing that additional factors other than $X$ (the ones we have measured or are interested in) may influence the response variable $Y$.

$y_i = f(x_i) + \epsilon_i$

where $f$ is some function, for example a linear function:

$y_i = \beta_0 + \beta_1 x_i + \epsilon_i$

and $\epsilon_i$ accounts for the difference between the observed response $y_i$ and its prediction from the model $\hat{y_i} = \beta_0 + \beta_1 x_i$. This is sometimes referred to as **process uncertainty**.

We would like to select $\beta_0, \beta_1$ so that the difference between the predictions and the observations is zero, but this is not usually possible. Instead, we choose a reasonable criterion: ***the smallest sum of the squared differences between $\hat{y}$ and $y$***.

$$R^2 = \sum_i (y_i - [\beta_0 + \beta_1 x_i])^2 = \sum_i \epsilon_i^2 $$

Squaring serves two purposes: (1) to prevent positive and negative values from cancelling each other out and (2) to strongly penalize large deviations. Whether the latter is a good thing or not depends on the goals of the analysis.

In other words, we will select the parameters that minimize the squared error of the model.

```
ss = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x) ** 2)
```

```
ss([0,1],x,y)
```

    333.35000000000002

```
from scipy.optimize import fmin

b0,b1 = fmin(ss, [0,1], args=(x,y))
b0,b1
```

    Optimization terminated successfully.
             Current function value: 21.375000
             Iterations: 79
             Function evaluations: 153

    (-4.3500136038870876, 3.0000002915386412)

```
plt.plot(x, y, 'ro')
plt.plot([0,10], [b0, b0+b1*10])
```

```
plt.plot(x, y, 'ro')
plt.plot([0,10], [b0, b0+b1*10])
for xi, yi in zip(x,y):
    plt.plot([xi]*2, [yi, b0+b1*xi], 'k:')
plt.xlim(2, 9); plt.ylim(0, 20)
```

Minimizing the sum of squares is not the only criterion we can use; it is just a very popular (and successful) one. For example, we can try to minimize the sum of absolute differences:

```
sabs = lambda theta, x, y: np.sum(np.abs(y - theta[0] - theta[1]*x))
b0,b1 = fmin(sabs, [0,1], args=(x,y))
print b0,b1
plt.plot(x, y, 'ro')
plt.plot([0,10], [b0, b0+b1*10])
```

We are not restricted to a straight-line regression model; we can represent a curved relationship between our variables by introducing **polynomial** terms. For example, a cubic model:

$y_i = \beta_0 + \beta_1 x_i + \beta_2 x_i^2 + \epsilon_i$

```
ss2 = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x - theta[2]*(x**2)) ** 2)
b0,b1,b2 = fmin(ss2, [1,1,-1], args=(x,y))
print b0,b1,b2
plt.plot(x, y, 'ro')
xvals = np.linspace(0, 10, 100)
plt.plot(xvals, b0 + b1*xvals + b2*(xvals**2))
```

Although polynomial model characterizes a nonlinear relationship, it is a linear problem in terms of estimation. That is, the regression model $f(y | x)$ is linear in the parameters.

For some data, it may be reasonable to consider polynomials of order>2. For example, consider the relationship between the number of home runs a baseball player hits and the number of runs batted in (RBI) they accumulate; clearly, the relationship is positive, but we may not expect a linear relationship.

```
ss3 = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x - theta[2]*(x**2) - theta[3]*(x**3)) ** 2)

bb = pd.read_csv("data/baseball.csv", index_col=0)
plt.plot(bb.hr, bb.rbi, 'r.')
b0,b1,b2,b3 = fmin(ss3, [0,1,-1,0], args=(bb.hr, bb.rbi))
xvals = np.arange(40)
plt.plot(xvals, b0 + b1*xvals + b2*(xvals**2) + b3*(xvals**3))
```

Of course, we need not fit least squares models by hand. The `statsmodels` package implements least squares models that allow for model fitting in a single line:

```
import statsmodels.api as sm

straight_line = sm.OLS(y, sm.add_constant(x)).fit()
straight_line.summary()
```

OLS Regression Results

  Dep. Variable:	            y	          R-squared:         	    0.891

  Model:	                   OLS	         Adj. R-squared:    	    0.864

  Method:	             Least Squares	    F-statistic:       	    32.67

  Date:	             Sun, 23 Feb 2014	   Prob (F-statistic):	  0.00463

  Time:	                 10:40:26	       Log-Likelihood:    	  -12.325

  No. Observations:	           6	        AIC:               	    28.65

  Df Residuals:	               4	        BIC:               	    28.23

  Df Model:	                   1

  Covariance Type:	      nonrobust

    	       coef	     std err	      t	      P>|t|	 [95.0% Conf. Int.]

  const	    -4.3500	     2.937	    -1.481	  0.213	   -12.505     3.805

  x1	        3.0000	     0.525	     5.716	  0.005	     1.543     4.457

  Omnibus:	          nan	   Durbin-Watson:     	    2.387

  Prob(Omnibus):	    nan	   Jarque-Bera (JB):  	    0.570

  Skew:	           0.359	   Prob(JB):          	    0.752

  Kurtosis:	       1.671	   Cond. No.          	     17.9

```
from statsmodels.formula.api import ols as OLS

data = pd.DataFrame(dict(x=x, y=y))
cubic_fit = OLS('y ~ x + I(x**2)', data).fit()

cubic_fit.summary()
```

OLS Regression Results

  Dep. Variable:	            y	          R-squared:         	    0.929

  Model:	                   OLS	         Adj. R-squared:    	    0.881

  Method:	             Least Squares	    F-statistic:       	    19.50

  Date:	             Sun, 23 Feb 2014	   Prob (F-statistic):	  0.0191

  Time:	                 10:40:27	       Log-Likelihood:    	  -11.056

  No. Observations:	           6	        AIC:               	    28.11

  Df Residuals:	               3	        BIC:               	    27.49

  Df Model:	                   2

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	 [95.0% Conf. Int.]

  Intercept	   -11.0748	     6.013	    -1.842	  0.163	   -30.211     8.062

  x	             6.0577	     2.482	     2.441	  0.092	    -1.840    13.955

  I(x ** 2)	    -0.3027	     0.241	    -1.257	  0.298	    -1.069     0.464

  Omnibus:	          nan	   Durbin-Watson:     	    2.711

  Prob(Omnibus):	    nan	   Jarque-Bera (JB):  	    0.655

  Skew:	          -0.809	   Prob(JB):          	    0.721

  Kurtosis:	       2.961	   Cond. No.          	     270.

## Model Selection

How do we choose among competing models for a given dataset? More parameters are not necessarily better, from the standpoint of model fit. For example, fitting a 9-th order polynomial to the sample data from the above example certainly results in an overfit.

```
def calc_poly(params, data):
        x = np.c_[[data**i for i in range(len(params))]]
        return np.dot(params, x)

ssp = lambda theta, x, y: np.sum((y - calc_poly(theta, x)) ** 2)
betas = fmin(ssp, np.zeros(10), args=(x,y), maxiter=1e6)
plt.plot(x, y, 'ro')
xvals = np.linspace(0, max(x), 100)
plt.plot(xvals, calc_poly(betas, xvals))
```

One approach is to use an information-theoretic criterion to select the most appropriate model. For example **Akaike's Information Criterion (AIC)** balances the fit of the model (in terms of the likelihood) with the number of parameters required to achieve that fit. We can easily calculate AIC as:

$$AIC = n \log(\hat{\sigma}^2) + 2p$$

where $p$ is the number of parameters in the model and $\hat{\sigma}^2 = RSS/(n-p-1)$.

Notice that as the number of parameters increase, the residual sum of squares goes down, but the second term (a penalty) increases.

To apply AIC to model selection, we choose the model that has the **lowest** AIC value.

```
n = len(x)

aic = lambda rss, p, n: n * np.log(rss/(n-p-1)) + 2*p

RSS1 = ss(fmin(ss, [0,1], args=(x,y)), x, y)
RSS2 = ss2(fmin(ss2, [1,1,-1], args=(x,y)), x, y)

print aic(RSS1, 2, n), aic(RSS2, 3, n)
```

    Optimization terminated successfully.
             Current function value: 21.375000
             Iterations: 79
             Function evaluations: 153
    Optimization terminated successfully.
             Current function value: 14.001110
             Iterations: 198
             Function evaluations: 372
    15.7816583572 17.6759368019

Hence, we would select the 2-parameter (linear) model.

## Logistic Regression

Fitting a line to the relationship between two variables using the least squares approach is sensible when the variable we are trying to predict is continuous, but what about when the data are dichotomous?

- male/female
- pass/fail
- died/survived

Let's consider the problem of predicting survival in the Titanic disaster, based on our available information. For example, lets say that we want to predict survival as a function of the fare paid for the journey.

```
titanic = pd.read_excel("data/titanic.xls", "titanic")
titanic.name
```

    0                      Allen, Miss. Elisabeth Walton
    1                     Allison, Master. Hudson Trevor
    2                       Allison, Miss. Helen Loraine
    3               Allison, Mr. Hudson Joshua Creighton
    4    Allison, Mrs. Hudson J C (Bessie Waldo Daniels)
    5                                Anderson, Mr. Harry
    6                  Andrews, Miss. Kornelia Theodosia
    7                             Andrews, Mr. Thomas Jr
    8      Appleton, Mrs. Edward Dale (Charlotte Lamson)
    9                            Artagaveytia, Mr. Ramon
    ...
    1298                  Wittevrongel, Mr. Camille
    1299                        Yasbeck, Mr. Antoni
    1300    Yasbeck, Mrs. Antoni (Selini Alexander)
    1301                       Youseff, Mr. Gerious
    1302                          Yousif, Mr. Wazli
    1303                      Yousseff, Mr. Gerious
    1304                       Zabour, Miss. Hileni
    1305                      Zabour, Miss. Thamine
    1306                  Zakarian, Mr. Mapriededer
    1307                        Zakarian, Mr. Ortin
    1308                         Zimmerman, Mr. Leo
    Name: name, Length: 1309, dtype: object

```
jitter = np.random.normal(scale=0.02, size=len(titanic))
plt.scatter(np.log(titanic.fare), titanic.survived + jitter, alpha=0.3)
plt.yticks([0,1])
plt.ylabel("survived")
plt.xlabel("log(fare)")
```

I have added random jitter on the y-axis to help visualize the density of the points, and have plotted fare on the log scale.

Clearly, fitting a line through this data makes little sense, for several reasons. First, for most values of the predictor variable, the line would predict values that are not zero or one. Second, it would seem odd to choose least squares (or similar) as a criterion for selecting the best line.

```
x = np.log(titanic.fare[titanic.fare>0])
y = titanic.survived[titanic.fare>0]
betas_titanic = fmin(ss, [1,1], args=(x,y))
```

    Optimization terminated successfully.
             Current function value: 277.621917
             Iterations: 55
             Function evaluations: 103

```
jitter = np.random.normal(scale=0.02, size=len(titanic))
plt.scatter(np.log(titanic.fare), titanic.survived + jitter, alpha=0.3)
plt.yticks([0,1])
plt.ylabel("survived")
plt.xlabel("log(fare)")
plt.plot([0,7], [betas_titanic[0], betas_titanic[0] + betas_titanic[1]*7.])
```

If we look at this data, we can see that for most values of `fare`, there are some individuals that survived and some that did not. However, notice that the cloud of points is denser on the "survived" (y=1) side for larger values of fare than on the "died" (y=0) side.

### Stochastic model

Rather than model the binary outcome explicitly, it makes sense instead to model the *probability* of death or survival in a **stochastic** model. Probabilities are measured on a continuous [0,1] scale, which may be more amenable for prediction using a regression line. We need to consider a different probability model for this exerciese however; let's consider the **Bernoulli** distribution as a generative model for our data:

$$f(y|p) = p^{y} (1-p)^{1-y}$$

where $y = \{0,1\}$ and $p \in [0,1]$. So, this model predicts whether $y$ is zero or one as a function of the probability $p$. Notice that when $y=1$, the $1-p$ term disappears, and when $y=0$, the $p$ term disappears.

So, the model we want to fit should look something like this:

$$p_i = \beta_0 + \beta_1 x_i + \epsilon_i$$

However, since $p$ is constrained to be between zero and one, it is easy to see where a linear (or polynomial) model might predict values outside of this range. We can modify this model sligtly by using a **link function** to transform the probability to have an unbounded range on a new scale. Specifically, we can use a **logit transformation** as our link function:

$$\text{logit}(p) = \log\left[\frac{p}{1-p}\right] = x$$

Here's a plot of $p/(1-p)$

```
logit = lambda p: np.log(p/(1.-p))
unit_interval = np.linspace(0,1)
plt.plot(unit_interval/(1-unit_interval), unit_interval)
```

And here's the logit function:

```
plt.plot(logit(unit_interval), unit_interval)
```

The inverse of the logit transformation is:

$$p = \frac{1}{1 + \exp(-x)}$$

So, now our model is:

$$\text{logit}(p_i) = \beta_0 + \beta_1 x_i + \epsilon_i$$

We can fit this model using maximum likelihood. Our likelihood, again based on the Bernoulli model is:

$$L(y|p) = \prod_{i=1}^n p_i^{y_i} (1-p_i)^{1-y_i}$$

which, on the log scale is:

$$l(y|p) = \sum_{i=1}^n y_i \log(p_i) + (1-y_i)\log(1-p_i)$$

We can easily implement this in Python, keeping in mind that `fmin` minimizes, rather than maximizes functions:

```
invlogit = lambda x: 1. / (1 + np.exp(-x))

def logistic_like(theta, x, y):
    p = invlogit(theta[0] + theta[1] * x)
    # Return negative of log-likelihood
    return -np.sum(y * np.log(p) + (1-y) * np.log(1 - p))
```

Remove null values from variables

```
x, y = titanic[titanic.fare.notnull()][['fare', 'survived']].values.T
```

... and fit the model.

```
b0 ,b1 = fmin(logistic_like, [0.5,0], args=(x,y))
b0, b1
```

    Optimization terminated successfully.
             Current function value: 827.015955
             Iterations: 47
             Function evaluations: 93

    (-0.88238984528338194, 0.012452067664164127)

```
jitter = np.random.normal(scale=0.01, size=len(x))
plt.plot(x, y+jitter, 'r.', alpha=0.3)
plt.yticks([0,.25,.5,.75,1])
plt.xvals = np.linspace(0, 600)
plt.plot(xvals, invlogit(b0+b1*xvals))
```

As with our least squares model, we can easily fit logistic regression models in `statsmodels`, in this case using the `GLM` (generalized linear model) class with a binomial error distribution specified.

```
logistic = sm.GLM(y, sm.add_constant(x), family=sm.families.Binomial()).fit()
logistic.summary()
```

Generalized Linear Model Regression Results

  Dep. Variable:	          y	          No. Observations:  	    1308

  Model:	                 GLM	         Df Residuals:      	    1306

  Model Family:	       Binomial	       Df Model:          	       1

  Link Function:	        logit	        Scale:             	    1.0

  Method:	               IRLS	         Log-Likelihood:    	  -827.02

  Date:	           Sun, 23 Feb 2014	   Deviance:          	   1654.0

  Time:	               10:40:31	       Pearson chi2:      	 1.33e+03

  No. Iterations:	         6

    	       coef	     std err	      t	      P>|t|	 [95.0% Conf. Int.]

  const	    -0.8824	     0.076	   -11.684	  0.000	    -1.030    -0.734

  x1	        0.0125	     0.002	     7.762	  0.000	     0.009     0.016

## Bootstrapping

Parametric inference can be **non-robust**:

* inaccurate if parametric assumptions are violated
* if we rely on asymptotic results, we may not achieve an acceptable level of accuracy

Parmetric inference can be **difficult**:

* derivation of sampling distribution may not be possible

An alternative is to estimate the sampling distribution of a statistic *empirically* without making assumptions about the form of the population.

We have seen this already with the kernel density estimate.

### Non-parametric Bootstrap

The bootstrap is a resampling method discovered by [Brad Efron](http://www.jstor.org/discover/10.2307/2958830?uid=3739568&uid=2&uid=4&uid=3739256&sid=21102342537691) that allows one to approximate the true sampling distribution of a dataset, and thereby obtain estimates of the mean and variance of the distribution.

Bootstrap sample:

$$S_1^* = \{x_{11}^*, x_{12}^*, \ldots, x_{1n}^*\}$$

$S_i^*$ is a sample of size $n$, **with** replacement.

The NumPy function `permutation` can be used to generate a random sample of some data without replacement:

```
np.random.permutation(titanic.name)[:5]
```

    array([u'Ahlin, Mrs. Johan (Johanna Persdotter Larsson)',
           u'McCormack, Mr. Thomas Joseph', u'Reynaldo, Ms. Encarnacion',
           u'Cumings, Mr. John Bradley',
           u'Coutts, Master. Eden Leslie "Neville"'], dtype=object)

Similarly, we can use the `random.randint` method to generate a sample *with* replacement, which we can use when bootstrapping.

```
random_ind = np.random.randint(0, len(titanic), 5)
titanic.name[random_ind]
```

    308     White, Mrs. John Stuart (Ella Holmes)
    1004           McDermott, Miss. Brigdet Delia
    1126                  Petranec, Miss. Matilda
    773                          Dimic, Mr. Jovan
    945                              Lam, Mr. Ali
    Name: name, dtype: object

We regard S as an "estimate" of population P

> population : sample :: sample : bootstrap sample

The idea is to generate replicate bootstrap samples:

$$S^* = \{S_1^*, S_2^*, \ldots, S_R^*\}$$

Compute statistic $t$ (estimate) for each bootstrap sample:

$$T_i^* = t(S^*)$$

```
n = 10
R = 1000
# Original sample (n=10)
x = np.random.normal(size=n)
# 1000 bootstrap samples of size 10
s = [x[np.random.randint(0,n,n)].mean() for i in range(R)]
_ = plt.hist(s, bins=30)
```

### Bootstrap Estimates

From our bootstrapped samples, we can extract *estimates* of the expectation and its variance:

$$\bar{T}^* = \hat{E}(T^*) = \frac{\sum_i T_i^*}{R}$$

$$\hat{\text{Var}}(T^*) = \frac{\sum_i (T_i^* - \bar{T}^*)^2}{R-1}$$

```
boot_mean = np.sum(s)/R
boot_mean
```

    -0.26109460708189697

```
boot_var = ((np.array(s) - boot_mean) ** 2).sum() / (R-1)
boot_var
```

    0.089079064843898145

Since we have estimated the expectation of the bootstrapped statistics, we can estimate the **bias** of T:

$$\hat{B}^* = \bar{T}^* - T$$

```
boot_mean - np.mean(x)
```

    -0.011697963201919293

### Bootstrap error

There are two sources of error in bootstrap estimates:

1. **Sampling error** from the selection of $S$.
2. **Bootstrap error** from failing to enumerate all possible bootstrap samples.

For the sake of accuracy, it is prudent to choose at least R=1000

### Bootstrap Percentile Intervals

An attractive feature of bootstrap statistics is the ease with which you can obtain an estimate of *uncertainty* for a given statistic. We simply use the empirical quantiles of the bootstrapped statistics to obtain percentiles corresponding to a confidence interval of interest.

This employs the *ordered* bootstrap replicates:

$$T_{(1)}^*, T_{(2)}^*, \ldots, T_{(R)}^*$$

Simply extract the $100(\alpha/2)$ and $100(1-\alpha/2)$ percentiles:

$$T_{[(R+1)\alpha/2]}^* \lt \theta \lt T_{[(R+1)(1-\alpha/2)]}^*$$

```
s_sorted = np.sort(s)
s_sorted[:10]
```

    array([-1.21473328, -1.20296609, -1.14257574, -1.12029849, -1.09395259,
           -1.03641168, -1.03146359, -1.01978558, -1.00359632, -0.94009612])

```
s_sorted[-10:]
```

    array([ 0.39140486,  0.39440803,  0.40575741,  0.40891146,  0.43260227,
            0.43500618,  0.49686382,  0.54923955,  0.55645112,  0.59660475])

```
alpha = 0.05
s_sorted[[(R+1)*alpha/2, (R+1)*(1-alpha/2)]]
```

    array([-0.84960042,  0.28246507])


```python
import numpy as np
from sympy import *
import matplotlib.pyplot as plt
import itertools
import seaborn as sns
sns.set_style('whitegrid')
import plotly.express as px
from plotly import graph_objects as go
%config InlineBackend.figure_format = 'retina'
```

    /usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
      import pandas.util.testing as tm

Link to slides: https://docs.google.com/presentation/d/1f_OykdISf2Ik9vKKgW9mwbrdZuvxvbiBbiDpkITVWTw/edit?usp=sharing

Link to talk: ...

### Similarities between Hopfield Networks and other algorithms
1. Small region of brain. Unipartite Graph where all neurons in principle can be connected with each other.
2. Phase space of the generalized coordinates with K* stable points. Any ball inserted into the phase space
near a stable point k_i will fall into it. In HN we have additional features: we can put specific vectors inside each stable point. We  have specific update rules for the generalized coordinates ...
3. Spinglass. A 3d array of magnetic dipoles (spins) which can take only two position +1/-1.
4. Coinsicion Matrix of a Random forest, each tree in which does only one split for the same feature.
5. V (features) matrix in SVD, but we additionally project each vector on the corner of a binary hypercube

1. Interactive example: http://faculty.etsu.edu/knisleyj/neural/neuralnet3.html
2. Math of binary HN (Hinton) https://www.youtube.com/watch?v=DS6k0PhBjpI
3. Math of binary HN (lecture) https://www.youtube.com/watch?v=yl8znINLXdg
4. Modern (binary) HN (krotov) https://www.youtube.com/watch?v=lvuAU_3t134
5. Modern HN example (MNIST classification) https://github.com/DimaKrotov/Dense_Associative_Memory/blob/master/Dense_Associative_Memory_training.ipynb

### Binary Hopfield Networks

```python
# Quick Reminder: expression of the form 0.5*q.T@W@q (where q is vector [1,N]) bassically means that we sum up
# all the interaction terms between elements of the q and weight them by coefficients stored in W.
# We can think about it as a total innervation (total synaptic energy) of a biological neural network
# given particular pattern of activations and particular connectome (synaptic strength for each pair of neurons)
# although, importantly, here, unlike in real brains (afaik) when only one neuron in the strongly linked couple gets
# activated - this decreases total Energy of the system.
A = np.random.rand(100,100)
c = np.random.rand(100)
print(np.sum(np.outer(c,c)*A), c@A@c)
```

    1168.136713053695 1168.1367130536948

```python
### Test V.T@M@V, where M is constructed with Hebbian rule
v = np.array([-1,-1,1,1])
M = np.outer(v,v)
v2 = np.array([1,-1,1,1])
v3 = np.array([1,1,-1,-1])
print('Memorized vector gives maximum energy:', v.T@M@v)
print('Corrupted vector gives lower energy:', v2.T@M@v2)
print('Fully inverted vector gives same energy:', v3.T@M@v3)
```

    Memorized vector gives maximum energy: 16
    Corrupted vector gives lower energy: 4
    Fully inverted vector gives same energy: 16

```python
def generate_binary_Hopfield_Net(N: int, K: int, zero_diag=False):
    """
        Generate {k1,k2,...,k_K} random binary vectors of length N.
        Obtain Hebbian 'coincision' matrices for each individual vector k.
        Sum Hebbian matrices to obtain final W matrix of the Hopfield Network.
        inputs:
            - N: length of each pattern (number of neurons representing one pattern)
            - K: number of 'memorized' patterns which will be placed in the
            local minima of the energy function E
            - zero_diag: if True, diagonal of W is set ot zero (classical HN)
        outputs:
            - W: Hopfield Net matrix with Hebbian weigths [N,N]
            - {k_i}: set of randomly generated and 'memorized' patterns of length N
    """
    ks = np.random.normal(size=[K,N])
    ks = np.sign(ks)
    W = ks.T@ks
    # same as:
    # W = np.array([np.outer(k,k) for k in ks]).sum(axis=0)
    if zero_diag:
        np.fill_diagonal(W,0)
    return W, ks.astype(int)
```

```python
# does not render in collab without TeX installed
W, ks = generate_binary_Hopfield_Net(4,3)
E = MatrixSymbol('-E',1,1)
Eq(E, MatMul(Matrix(ks[0]).T, MatMul(Matrix(W), Matrix(ks[0])))) ## add final value
```

    Eq(-E, Matrix([[-1, 1, 1, -1]])*(Matrix([
    [ 3.0, -1.0, -1.0, -1.0],
    [-1.0,  3.0,  3.0, -1.0],
    [-1.0,  3.0,  3.0, -1.0],
    [-1.0, -1.0, -1.0,  3.0]])*Matrix([
    [-1],
    [ 1],
    [ 1],
    [-1]])))

## Energy Landscape

### plot Functions defined

```python
def plot_Energy_Landscape(Es, Vs, kinds, mirrored=False, trajectory=None):
    """
        Makes plotly interactive graph of an Energy function for a small Hopfield Network
        inputs:
            Es: vector of Energies ordered by Vs
            Vs: generated by permutations binary vectors
            kinds: indices of K vecots on which this Hopfiel Network was build (should have minimal energies)
    """
    if W.shape[0]<8:
        tt = np.array([str(tuple(l)) for l in (Vs.T).tolist()]) # generate text vector representations
    else:
        tt = []

    K_Es = Es[kinds] # select energies for K patterns
    ttk = tt[kinds] if len(tt)>0 else []

    fig = go.Figure()
    fig.update_layout(
        height=450,
        title_text="Energy Landscape" + "<br>(hover to see vectors)" if W.shape[0]<8 else "",
        yaxis=dict(title_text="Energy"))
    fig.add_trace(go.Scatter(y=Es, text=tt, mode='lines+markers', name='E-landscape', hoverinfo='text+x+y'))
    fig.add_trace(go.Scatter(x=kinds, y=K_Es, text=ttk, mode='markers', name='K-patterns', hoverinfo='text+x+y'))

    if mirrored:
        kinds_star = np.mean(np.arange(len(Es)))-(kinds-np.mean(np.arange(len(Es)))) # flip off central axis
        fig.add_trace(go.Scatter(x=kinds_star, y=K_Es, text=ttk, mode='markers',
                             name='K-mirrored', hoverinfo='text+x+y'))

    if trajectory:
        #trajectory[0] = np.mean(np.arange(len(Es)))-(trajectory[0]-np.mean(np.arange(len(Es)))) # flip off central axis
        fig.add_trace(go.Scatter(x=trajectory[0], y=trajectory[1], text=ttk, mode='markers',
                     name='mirrored_traj', hoverinfo='text+x+y'))
    fig.show()
```

```python
def make_Energy_landscape(W: np.ndarray):
    """
        Explicitely defines every possible binary vector and calculates
        Energy of the particular Hopfield Network for each.
        inputs:
            - W: Hopfield Net weigths Matrix [N,N]
        output:
            - Vs: array of generated binary vectors
            - Es: vector of scalar Energies of size [1,N]
    """
    if W.shape[0]>10:
        raise ValueError('The dimension of the Hopfield Net is too large \n to generate all possible vecs by explicit permutations.')
    Vs = np.array(list(itertools.product([-1, 1], repeat=W.shape[0]))).T
    Es = -0.5*(Vs.T*(W@Vs).T).sum(axis=1)
    return Vs, Es

def locale_any_vectors(Vs: np.ndarray, ks: np.ndarray):
    """
        Locates positions of any k vectors [n1,N] in the array Vs of other vectors [N,n2] by elementwise comparison
        This should return unique indices as long as Vs contains unique vectors in its rows.
        NOTE:
            This method expects Vs to be *transposed* relative to 'locale_K_vectors' method!
        inputs:
            - Vs: array of vectors of size [N,n2] in which to search for specified vectors
            - k: query vectors of size [n1,N]
        output:
            - kinds: indices of k query vectors in the Vs array
    """
    kinds = []
    for k in ks:
        kinds.append(np.argmax((Vs==k).sum(axis=1)))
    return kinds

def locale_K_vectors(Vs: np.ndarray, ks: np.ndarray):
    """
        *This is only suitable for finding vectors with largest norms.*
        Locates positions of specified query vectors k [n1,N] in the array of other vectors [N,n2]
        using dot product as signal.
        inputs:
            - Vs: array of vectors of size [N,n2] in which to search for specified vectors
            - k: query vectors of size [n1,N]
        output:
            - kinds: indices of k query vectors in the Vs array
    """
    kinds = (ks@Vs).argmax(axis=1)
    return kinds
```

### operations

```python
### Lets check energy landscape. Here we can simply permute through all possible states of our neural network ('query' vectors)
### and calculate energy function for each. TODO: think how Energy can be projected onto 1/2 dimensional rep. of vectors
Vs, Es = make_Energy_landscape(W)
kinds = locale_any_vectors(Vs.T, ks)
```

```python
plot_Energy_Landscape(Es, Vs, kinds, mirrored=True)
```

<meta charset="utf-8" />

```python
### Try changing N and K
# 1. Generate Hopfield Net (W) with N neurons and K memorized patterns
# 2. Make energy landscape for W by explicitly probing each 2**N vectors (I've set a limit N<12, in order to save colab's memory)
# 3. Find memorized K vectors in the energy landscape
# 4. Plot Energy landscape with marked positions of the memorized patterns
W, ks = generate_binary_Hopfield_Net(N=5,K=5)
print(ks)
Vs, Es = make_Energy_landscape(W)
kinds = locale_K_vectors(Vs, ks)
plot_Energy_Landscape(Es, Vs, kinds, mirrored=False)
```

    [[-1 -1 -1  1  1]
     [ 1 -1 -1  1  1]
     [-1 -1  1  1 -1]
     [-1  1  1  1  1]
     [-1  1 -1  1 -1]]

<meta charset="utf-8" />

# Update Rule

### Functions defined

```python
def update_rule_sync(q: np.ndarray, W: np.ndarray, eps=1e-6, bias=0):
    """
        Simplest synchronous update rule for binary Hopfield Network
        If the 'field' acting on an element q_i has different sign (direction) than q_i:
        change q_i sign to the opposite. If q_i is aligned with the field - don't change it.
        Note:
            added eps because sometimes 0 values occured after update.
    """
    return np.sign(W@q+bias+eps).astype(int)

def update_rule_async_random(q: np.ndarray, W: np.ndarray, eps=1e-6, bias=0):
    """
        Simplest Asynchronous update rule for binary Hopfield Network
        If the 'field' acting on an element q_i has different sign (direction) than q_i:
        change q_i sign to the opposite. If q_i is aligned with the field - don't change it.
    """

    for qi in np.random.choice(np.arange(len(q)), size=len(q), replace=False):
        q[qi] = np.sign(W[qi,:]@q)
    return q.astype(int)

def retrive_memorized(q: np.ndarray, W: np.ndarray, update_rule, bias=0, maxiteration=500):
    """
        Update cycle which will drive query vector to one of the local minima of the binary Hopfield Network.
        If number of stored (in W) patterns K is below K* (K* << N^2) (memory limit for HN with a given update rule)
        it should guarantee to retrive one of this vectors.
        If K>K*, there is no guarantee that the retrived pattern will be among K.
        Note:
            TODO: This function will not converge if q* will fall into metastable state
            (subset of minimas transitioning into one another in cycle)
        inputs:
            - q: query vector
            - W: Hopfield Network weigths matrix
            - update_rule: function describing the update rule of the Hopfiel Network
        outputs:
            - q: retrived memorized vector closest to q
            - E_traj: Energy trajectory for q updates
            - q_traj: ordered list of updated q vectors with the final form on the right
    """
    dE = np.inf
    Ef = lambda x: -0.5*x.T@W@x-np.sum(bias*x)
    E1, E0 = Ef(q), np.inf
    E_traj = [E1]
    q_traj = [q]
    itr = 0
    while (E1<E0) and itr<maxiteration:
        q = update_rule(q,W,bias)
        E1,E0 = Ef(q), E1
        E_traj.append(E1)
        q_traj.append(q)
        itr += 1
    return q, E_traj, q_traj
```

### operation

```python
# Do optimization starting from the Energy maximum
# Check if it converged to any stored pattern
# Plot Energy landscape with the optimization trace
# (Hm, something is wrong with the coordinates of the trace values, anyway they jump over the whole landscape)
W, ks = generate_binary_Hopfield_Net(8, 5)
Vs, Es = make_Energy_landscape(W)
kinds = locale_any_vectors(Vs.T, ks)
print('Memorized patterns:')
print(ks)
peak = Vs.T[np.argmax(Es)]
q, E_traj, q_traj = retrive_memorized(peak, W, update_rule=update_rule_async_random)
trajectory = (locale_any_vectors(Vs.T, q_traj), E_traj)
print('Number of steps:', len(E_traj))
print('Energy trajectory:')
print(E_traj)
print('Updated query:')
print(np.stack(q_traj))
print('Converged to minima?')
print('Yes!' if ((ks==q).all(axis=1)|(ks==(-1)*q).all(axis=1)).any() else 'No!')
plot_Energy_Landscape(Es, Vs, kinds, trajectory=trajectory)
```

    Memorized patterns:
    [[ 1 -1 -1  1  1 -1  1  1]
     [ 1 -1  1 -1 -1 -1  1  1]
     [ 1 -1  1  1 -1 -1 -1  1]
     [ 1 -1  1  1 -1 -1 -1 -1]
     [ 1 -1  1 -1  1 -1  1  1]]
    Number of steps: 4
    Energy trajectory:
    [-4.0, -54.5, -62.0, -62.0]
    Updated query:
    [[ 1 -1  1  0 -1 -1 -1  1]
     [ 1 -1  1  1 -1 -1 -1  1]
     [ 1 -1  1  1 -1 -1 -1  1]
     [ 1 -1  1  1 -1 -1 -1  1]]
    Converged to minima?
    Yes!

<meta charset="utf-8" />

# Simulations
Here we conduct some simulations in order to empricially prove several statements about binary Hopfield Nets:
1. K* = 0.14*N (we can not reliably store more than 0.14*N patterns using Hebbian learning rule (outer product))
2. Retrival capability drops with the distance from the memorized patterns (and does so unevenly for networks with different K/N)
3. Asynchronous update is more accurate than synchronous

### functions defined

```python
def corrupt_vector(v, h):
    assert h<len(v), f'Number of corrupted bits can not be higher than the dimension of the vector {len(v)}'
    cind = np.random.choice(len(v), h, replace=False)
    v[cind] *= (-1)
    return v
```

```python
def retrival_rate_from_random(W, ks, update_rule, reps=40,):
    """
        For a number of random pattern 'reps' of dimension [1,W.shape[0]] we run Hopfield updates and return number
        of converged results.
        inputs:
            - W: HN weights [N,N]
            - ks: HN stored patterns [K,N]
            - update_rule: function of the update rule (use lambda to specify parameters)
            - reps: number of replicates (R)
        outputs:
            - res: array of shape [R,K]
    """
    K, N = ks.shape
    res = []
    queries = np.random.choice([-1,1], size=[reps,N])
    for q in queries:
        q, et, vt = retrive_memorized(q, W, update_rule)
        hits = ((ks==q).all(axis=1)|(ks==(-1)*q).all(axis=1))
        res.append(hits.astype(int))
    return np.array(res)

def retrival_rate_vs_error(W, ks, update_rule, distances, reps=40, use_first_k=-1):
    """
        This function repeatedly runs 'retrival operation' on the redacted (noised) stored patterns of the
        binary Hopfield Net W. For each Hammind distance (number of erronious bits introduced) it creates multiple
        replicates (randomly) and optimized each one, storing number of exact hits
        (complete convergence back to *one* of the stored patterns, but not necesserily the original one).
        inputs:
            - W: HN weights [N,N]
            - ks: HN stored patterns [K,N]
            - update_rule: function of the update rule (use lambda to specify parameters)
            - distances: array [1,D] containing Hamming distance values
            - reps: number of replicates (R) to check for each distance, for each original pattern used as seed
            - use_first_k: number of original patterns to use as seeds (K_th). if '-1' use all ks.
        outputs:
            - res: array of shape [K_th,D,R,K] containing hits for every replica. First dimension corresponds
                to the number of seeds used, second to the number of error thresholds used, third to the number of
                replicates, fourth - to the number of patterns (ks) exact correspondence with which was checked
            - meta: scalar values describing each optimization. Length - number of update steps,
                    dE - energy decrease from the 0 step to the last step.
    """
    res = []
    meta = []
    for i in range(len(ks[:use_first_k])):
        res.append([])
        meta.append([])
        for j in range(len(distances)):
            res[i].append([])
            meta[i].append([])
            for _ in range(reps):
                cv = corrupt_vector(ks[i], distances[j])
                cv, et, vt = retrive_memorized(cv, W, update_rule)
                metas = (len(et), et[-1]-et[0])
                meta[i][j].append(metas)
                hits = ((ks==cv).all(axis=1)|(ks==(-1)*cv).all(axis=1))
                res[i][j].append(hits.astype(int))
    res = np.array(res)
    meta = np.array(meta)
    return res, meta
```

```python
def check_retrival_over_N_and_K(Ns, Ks, update_rule):
  reses = []
  for i,N in enumerate(Ns):
      reses.append([])
      for K in Ks:
          W, ks = generate_binary_Hopfield_Net(N, K)
          res = retrival_rate_from_random(W, ks, update_rule, reps=100)
          reses[i].append(res.any(axis=-1))
  return np.array(reses)
```

### plot Functions defined

```python
# We can see in that for a very small Hamming distance synchronous update performes significantly worse than asynchronous
# (it also goes to a local minima in less steps. Not shown)
def plot_Hamming_dist_vs_retival_rate(res, N, K, distances, save=False, suffix=''):
    """
        use plot_Hamming_dist_vs_retival_rate_NEW instead.
    """
    fig, ax = plt.subplots(1,1)
    for i in range(res.shape[0]):
        ax.plot(distances, res[i,...].any(axis=-1).mean(axis=1), label=f'k-{i}')
    ax.set_xlabel('Hamming Distance from stable pattern')
    ax.set_ylabel('Fraction of decorrupted')
   # plt.legend()
    plt.title(f'Retrival rate vs Hamming distance (N={N}, K={K})')
    if save:
        plt.savefig(f'Retrival_rate_vs_Hammim_distance_{suffix}.png', dpi=100)

def plot_Hamming_dist_vs_retrival_rate_NEW(reses, Ns, Ks, distances, save=False, suffix=''):
    fig, ax = plt.subplots(1,1)
    for i,res in enumerate(reses):
        mu = res.any(axis=-1).mean(axis=-1).mean(0)
        sigma = res.any(axis=-1).std(axis=-1).mean(0)
        ax.plot(distances, mu, label=f'N={Ns[i]};K={Ks[i]}')
        ax.fill_between(distances, mu-sigma/np.sqrt(res.shape[2]), mu+sigma/res.shape[2], alpha=0.8)
    ax.set_xlabel('Hamming Distance from stable pattern')
    ax.set_ylabel('Fraction of converged to any from K')
    plt.legend()
    plt.title(f'Retrival rate vs Hamming distance for different K')
    if save:
        plt.savefig(f'Retrival_rate_vs_Hammim_distance_by_K_stored_{suffix}.png', dpi=100)

def plot_Network_capacity_vs_N_for_two_update_rules(retrival_async, retrival_sync, Ns):
  fig, ax = plt.subplots(1,1)
  colors = ['firebrick', 'green', 'blue']
  for i in range(retrival_async.shape[0]):
      mu1, mu2 = retrival_async[i].mean(axis=-1), retrival_sync[i].mean(axis=-1)
      plt.plot(Ks, mu1, color=colors[i], label=f'mu, async; N={Ns[i]}')
      plt.plot(Ks, mu2, color=colors[i], linestyle='--', label=f'mu, sync; N={Ns[i]}')
      plt.xlabel('K stored')
      plt.ylabel('Fraction of converged to any from K')
  plt.legend()
  plt.title('Network capacity K* vs N')
  plt.savefig('Network_capacity_vs_K_sync_vs_async.png', dpi=100)

  def plot_Network_capacity_vs_N(retrival_statistics, Ns):
  fig, ax = plt.subplots(1,1)
  for r, N in zip(retrival_statistics, Ns):
      mu, sigma = r.mean(axis=-1), r.std(axis=-1)
      plt.plot(Ks, mu, label=f'N={N}')
      plt.fill_between(Ks, mu-sigma/np.sqrt(100), mu+sigma/np.sqrt(100), alpha=0.8)
      plt.xlabel('K stored')
      plt.ylabel('Fraction of converged to any from K')
  plt.legend()
  plt.title('Network capacity K* vs N')
  plt.savefig('Network_capacity_vs_K.png', dpi=100)
```

```python

```

```python
def make_heatmap(x,y,z, **kwargs):
    return go.Heatmap(x=x, y=y, z=z, xgap=1.5, ygap=1.5, zmin=0, zmax=res[0].mean(axis=0).max(), **kwargs)

def make_heatmap_frame(step, **kwargs):
    return go.Frame(data=[go.Heatmap(**kwargs)],
                    layout=go.Layout(title_text=title(step))
                    )

def make_animated_heatmap(res):
  """
    Plots animated confusion matrix for stored patterns K. Main diagonal contains fraction of de-corrupted patterns
    which were aligned with their initial minima. Off-diagonal cells show de-corrupted patterns which ended up aligning
    with memorized stable minima different from their initial one (better when falling into spurious state, but still erroneous retirval)
    inputs:
      - res: np.array, output of the 'retrival_rate_vs_error' function
  """
  title = lambda x: f"Confusion Matrix for updated vectors <br> Hamming Distance: {x}"
  xxs = np.arange(res.shape[0])
  yys = np.arange(res.shape[0])[::-1]
  button_dict = dict(label="Play", method="animate", args=[None, {"frame": {"duration": 700}}])

  fig = go.Figure(make_heatmap(xxs, yys, res[:,0,...].mean(axis=1)))
  fig.update_layout({'title':title(0), 'width':500, 'height':500, 'autosize':False})
  fig.update_layout({'updatemenus':[{'type':"buttons", 'buttons':[button_dict]}]
                    })
  fig.frames = [make_heatmap_frame(step=i, x=xxs, y=yys, z=res[:,i,:,:].mean(axis=1))
            for i in range(res.shape[1])]
  fig.show()
```

### operations
### Compare Network capacity for the different K/N ratios

```python
Ns, Ks = [50,100,150], [5, 8, 10, 12, 15, 17, 20, 25]
retrival_async = check_retrival_over_N_and_K(Ns, Ks, update_rule=update_rule_async_random)
retrival_sync = check_retrival_over_N_and_K(Ns, Ks, update_rule=update_rule_sync)
```

```python
plot_Network_capacity_vs_N(retrival_async, Ns)
```

```python
plot_Network_capacity_vs_N_for_two_update_rules(retrival_async, retrival_sync, Ns)
```

Here we plot retrival rate (to *any* stable point) vs Hamming distance from a given stable point.
We observe that small deviations (e.g. 5 errors for 100 dimensional vector) are successfully linked to stored patterns even for K>K*=0.14*N. However, as we probe vectors with more errors (occupying unseen parts of the energy landscape), de-corruption ability of the Hopfield Networks with K>0.14*N drops much quicker.

```python
### Plot Hamming Distance (state of corruption) vs probability to retrive memorized state
N, K = 30, 5
hammings = np.arange(15)
W, ks = generate_binary_Hopfield_Net(N, K)
res, meta = retrival_rate_vs_error(W, ks, update_rule_async_random, hammings,)
plot_Hamming_dist_vs_retival_rate(res, N, K, hammings)
```

```python
make_animated_heatmap(res)
```

<meta charset="utf-8" />

```python
# This might take a while ~3 min
Ns, Ks = [100]*7, [5, 8, 10, 12, 15, 17, 20]
reses = []
for N,K in zip(Ns, Ks):
    W, ks = generate_binary_Hopfield_Net(N, K)
    hammings = np.arange(0, N//3, 3)
    res, meta = retrival_rate_vs_error(W, ks, update_rule_async_random, hammings, use_first_k=5)
    reses.append(res)
```

```python
plot_Hamming_dist_vs_retrival_rate_NEW(reses, Ns, Ks, hammings, save=True, suffix='combined')
```

### Theoretical limitations of the binary Hopfield Net's storage capacity result from the Hebbian Rule:
(TODO)

```python
# If we build W from N vectors and every vector will be orthogonal - when *any* random vector will be stored (placed in local minima)
# however it is tricky to make orthogonalization of binary vectors (at least I couldn't invent quick solution)
ks = np.ones([30,30])
np.fill_diagonal(ks, -1)
W = ks@ks.T
```

```python
q=np.random.choice([-1,1], 30)
np.sign(W@q)==q
```

    array([ True,  True,  True,  True,  True,  True,  True,  True,  True,
            True,  True,  True,  True,  True,  True,  True,  True,  True,
            True,  True,  True,  True,  True,  True,  True,  True,  True,
            True,  True,  True])

### Corrupt MNIST

```python
from torchvision.datasets import MNIST
from PIL import Image
dt = MNIST(root='sample_data/', download=True)
```

    Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to sample_data/MNIST/raw/train-images-idx3-ubyte.gz

    HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))

    Extracting sample_data/MNIST/raw/train-images-idx3-ubyte.gz to sample_data/MNIST/raw
    Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to sample_data/MNIST/raw/train-labels-idx1-ubyte.gz

    HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))

    Extracting sample_data/MNIST/raw/train-labels-idx1-ubyte.gz to sample_data/MNIST/raw
    Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to sample_data/MNIST/raw/t10k-images-idx3-ubyte.gz

    HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))

    Extracting sample_data/MNIST/raw/t10k-images-idx3-ubyte.gz to sample_data/MNIST/raw
    Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to sample_data/MNIST/raw/t10k-labels-idx1-ubyte.gz

    HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))

    Extracting sample_data/MNIST/raw/t10k-labels-idx1-ubyte.gz to sample_data/MNIST/raw
    Processing...

    /usr/local/lib/python3.6/dist-packages/torchvision/datasets/mnist.py:469: UserWarning:

    The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)

    Done!

```python
images, classes = [], []
for i in range(3):
    m,c = dt[i]
    #m.save(f'Pictures/mnist_{i}.png')
    m = np.array(m)
    m = 2*((m > 20).astype(int))-1
    images.append(m)
    classes.append(c)
```

### Functions defined

```python
def construct_Hopfield_Network_from_data(ks: np.ndarray, normalize=True, zero_diag=False):
    """
        Construct Hopfield Network from K vectors of size [1,N].
        input:
            - ks: binary vectors
        outputs:
            - W: Hopfield Net matrix with Hebbian weigths [N,N]
    """
    W = ks.T@ks
    if normalize:
        W = W*(1/W.shape[0])
    if zero_diag:
        np.fill_diagonal(W,0)
    return W
```

```python
def corrupt_image(img, p=0.1):
    eps = np.random.choice([-10,0,10], size=[28,28], p=[p*0.5,1-p, p*0.5])
    img = np.clip(img + eps, -1, 1)
    return img
```

### operations

```python
data = np.stack([m.ravel() for m in images])
W = construct_Hopfield_Network_from_data(data, normalize=True, zero_diag=True)
plt.imshow(W)
```

```python
### Even for just a two MNIST digits we get convoluted memories - probably because the signal/background ratio is too low
### so every picture vector [1,784] looks very similar due to dark background
assert len(images)<10, "Nope, human can not comprehend so many MNIST pictures at once!"
fig, ax = plt.subplots(len(images),2, figsize=(5, len(images)*3))
for i in range(len(images)):
  query = corrupt_image(images[i], p=0.25).ravel()
  query_retrived, E_traj, q_traj = retrive_memorized(query.copy(), W, update_rule_async_random)
  ax[i,0].imshow(query.reshape([28,28]))
  ax[i,1].imshow(query_retrived.reshape([28,28]))
#Image.fromarray(np.clip(255*(query+1), 0, 255).reshape([28,28]).astype(np.uint8)).save('Pictures/mnist_corrupted_5.png')
```

```python
# query = corrupt_image(images[0], p=0.25).ravel()
# query_retrived, E_traj, q_traj = retrive_memorized(query.copy(), W, update_rule_async_random)
# fig, ax = plt.subplots(1,2)
# ax[0].imshow(query.reshape([28,28]))
# ax[1].imshow(query_retrived.reshape([28,28]))
# #Image.fromarray(np.clip(255*(query+1), 0, 255).reshape([28,28]).astype(np.uint8)).save('Pictures/mnist_corrupted_5.png')
```

### Modern Hopfield Networks with better Energy Functions (TODO)

```python
# Comparing power law and exponential for different constants

from scipy.signal import find_peaks
find_peaks(1/(Es+1e-5))

t = np.linspace(0,18,200)
plt.plot(t, 2**t)
plt.plot(t, 2.71**(0.65*t))
plt.plot(t, t**3)
plt.plot(t, t**4)
```