```python
import numpy as np
import matplotlib.pyplot as plt
import scipy
from sklearn.model_selection import ParameterGrid
from sklearn.manifold import Isomap
import time
from tqdm import tqdm

import librosa
from librosa import cqt
from librosa.core import amplitude_to_db
from librosa.display import specshow

import os
import glob
```

```python
data_dir = '/Users/sripathisridhar/Desktop/SOL'
```

```python
file_paths= sorted(glob.glob(os.path.join(data_dir, '**', '*.wav')))

file_names= []
for file_path in file_paths:
    file_names.append(os.path.basename(file_path))
```

```python
hop_size= 512
q= 24
```

```python
import h5py

with h5py.File("SOL.h5", "r") as f:
    features_dict = {key:f[key][()] for key in f.keys()}
```

```python
grid = {
    'Q': [24],
    'k': [3],
    'comp': ['log'],
    'instr': ['all'],
    'dyn': ['all']
}

settings = list(ParameterGrid(grid))

for setting in settings:

    if setting["instr"] == 'all':
        setting['instr'] = ''

    if setting['dyn'] == 'all':
        setting['dyn'] = ''
```

```python
batch_str = []
CQT_OCTAVES = 7

features_keys = list(features_dict.keys())

for setting in settings:

    q = setting['Q']
    # Batch process and store in a folder
    batch_str = [setting['instr'], setting['dyn']]

    batch_features = []
    for feature_key in features_keys:
        # Get features that match setting

        if all(x in feature_key for x in batch_str):
            batch_features.append(features_dict[feature_key])

    batch_features = np.stack(batch_features, axis=1)

    # Isomap parameters
    hop_size = 512
    compression = 'log'
    features = amplitude_to_db(batch_features)
    n_neighbors = setting['k']
    n_dimensions = 3
    n_octaves = 3

    # Prune feature matrix
    bin_low = np.where((np.std(features, axis=1) / np.std(features)) > 0.1)[0][0] + q
    bin_high = bin_low + n_octaves*q
    X = features[bin_low:bin_high, :]

    # Z-score Standardization- improves contrast in correlation matrix
    mus = np.mean(X, axis=1)
    sigmas = np.std(X, axis=1)
    X_std = (X - mus[:, np.newaxis]) / (1e-6 + sigmas[:, np.newaxis]) # 1e-6 to avoid runtime division by zero

    # Pearson correlation matrix
    rho_std = np.dot(X_std, X_std.T) / X_std.shape[1]

    # Isomap embedding
    isomap = Isomap(n_components= n_dimensions, n_neighbors= n_neighbors)
    coords = isomap.fit_transform(rho_std)

    # Get note value
    freqs= librosa.cqt_frequencies(q*CQT_OCTAVES, fmin=librosa.note_to_hz('C1'), bins_per_octave=q) #librosa CQT default fmin is C1
    chroma_list= librosa.core.hz_to_note(freqs[bin_low:bin_high])

    notes = []
    reps = q//12
    for chroma in chroma_list:
        for i in range(reps):
            notes.append(chroma)
```

```python
curr_fig= plt.figure(figsize=(5.5, 2.75))
ax= curr_fig.add_subplot(121)
ax.axis('off')

import colorcet as cc
subsampled_color_ids = np.floor(np.linspace(0, 256, q, endpoint=False)).astype('int')
color_list= [cc.cyclic_mygbm_30_95_c78[i] for i in subsampled_color_ids]

# Plot embedding with color
for i in range(coords.shape[0]):
    plt.scatter(coords[i, 0], coords[i, 1], color= color_list[i%q], s=30.0)

plt.plot(coords[:, 0], coords[:, 1], color='black', linewidth=0.2)

# Plot Pearson correlation matrix
rho_frequencies = freqs[bin_low:bin_high]

freq_ticklabels = ['A2', 'A3', 'A4']
freq_ticks = librosa.core.note_to_hz(freq_ticklabels)

tick_bins = []
tick_labels= []
for i,freq_tick in enumerate(freq_ticks):
    tick_bin = np.argmin(np.abs(rho_frequencies-freq_tick))
    tick_bins.append(tick_bin)
    tick_labels.append(freq_ticklabels[i])

plt.figure(figsize=(2.5,2.5))
plt.imshow(np.abs(rho_std), cmap='magma_r')
plt.xticks(tick_bins)
plt.gca().set_xticklabels(freq_ticklabels)
# plt.xlabel('Log-frequency (octaves)')
plt.yticks(tick_bins)
plt.gca().set_yticklabels(freq_ticklabels)
# plt.ylabel('Log-frequency (octaves)')
plt.gca().invert_yaxis()

plt.clim(0, 1)

```

### Circle projection

```python
import circle_fit
import importlib
importlib.reload(circle_fit)
from circle_fit import circle_fit

A = np.transpose(coords[:,:-1])
x, r, circle_residual = circle_fit(A, verbose=True)
```

```python
import matplotlib
matplotlib.rc('font', family='serif')

fig, axes = plt.subplots()
plt.scatter(A[0,:],A[1,:])
plt.plot(x[0],x[1],'rx')

circle = plt.Circle(x, radius=r, fill=False, linestyle='-.')

axes.set_aspect(1)
axes.add_artist(circle)

# axes.set_ylim([-5,6])
# axes.set_xlim([-2,8])

plt.title('Circle fit: TinySOL all instr', pad=10.0)
plt.show()

print(np.sqrt(circle_residual)/72)
```

```python
r
```

    6.355528108736576

```python
def d_squared(a, b):
    # Takes two n-D tuples and returns euclidean distance between them

    # Cast to array for computation
    # Cast first to tuple in case a or b are Sympy Point objects
    p_a = np.array(tuple(a), dtype='float')
    p_b = np.array(tuple(b), dtype='float')

    return np.sum(np.square(p_a - p_b))
```

```python
import sympy

from sympy.geometry import Circle, Point, Line

center = Point(x, evaluate=False)
c = Circle(center, r, evaluate=False)

l = Line(Point(coords[0,:-1]), center, evaluate=False)
points = [tuple(p) for p in l.points]

xy_prime = []

# TODO: Optimize to a more pythonic manner
for x,y in coords[:,:2]:

    intersections = c.intersection(Line(Point(x,y), center, evaluate=False))

    if d_squared((x,y),intersections[0]) < d_squared((x,y), intersections[1]):
        xy_prime.append([float(p) for p in intersections[0]])
    else:
        xy_prime.append([float(p) for p in intersections[1]])

```

```python
fig, axes = plt.subplots()
plt.scatter(np.array(xy_prime)[:,0],np.array(xy_prime)[:,1], s=10,
            label='projected points')
plt.scatter(A[0,:],A[1,:], s=0.5, label='isomap embedding points (2D)')
plt.plot(center[0],center[1],'rx')

circle = plt.Circle([float(p) for p in center], radius=r, fill=False,
                    linestyle='--', label='estimated circle fit')

axes.set_aspect(1)
axes.add_artist(circle)

plt.title('Projected points on circle', pad=10.0)
plt.legend(bbox_to_anchor=(1,1))
plt.show()

```

### Line projection

```python
z = np.arange(len(coords[:,2]))
z_fit = scipy.stats.linregress(z, coords[:,2])
print(z_fit.stderr)
```

    0.0052234352623828605

```python
plt.figure()
plt.title('Line fit: TinySOL all instr')
plt.scatter(np.arange(len(coords[:,2])), coords[:,2])

plt.plot(z_fit.intercept + z_fit.slope*z, 'b')
```

```python
# New line coordinates
z_prime = [i * z_fit.slope + z_fit.intercept for i,_ in enumerate(coords[:,2])]
```

```python
coords_prime = np.append(np.array(xy_prime), np.expand_dims(np.array(z_prime), axis=1), axis=1)
coords_length = coords_prime.shape[0]
```

### Distance matrices

```python
# Projected helix self-distance matrix

D_proj = np.zeros((coords_length, coords_length))
for i in range(coords_length):
    for j in range(i,coords_length):

        D_proj[i][j] = d_squared(coords_prime[i,:], coords_prime[j,:])
```

```python
# Isomap embedding self-distance matrix

D_isomap = np.zeros((coords_length, coords_length)) # Projected points same no. as isomap
for i in range(coords_length):
    for j in range(i, coords_length):

        D_isomap[i][j] = d_squared(coords[i,:], coords[j,:])
```

```python
# Geodesic self-distance matrix

D_geodesic = isomap.dist_matrix_

# Convert to upper triangular sparse matrix
for i in range(coords_length):
    for j in range(i):
        D_geodesic[i,j] = 0
```

```python
## Centering matrix

def centered(A, Q=24, J=3):
    # Returns centered distance matrix

    '''
    Inputs
    -----
    A - squared distance matrix
    Q - quality factor, 24 by default
    J - number of octaves, 3 by default

    Returns
    -----
    tau - MDS style diagonalized matrix of A
    '''

    coords_length = A.shape[0]
    H = np.zeros((coords_length, coords_length))

    const = 1/(Q*J)
    for i in range(coords_length):
        for j in range(coords_length):
            if j==i:
                H[i,j] = 1 - const
            else:
                H[i,j] = -const

    return -0.5 * np.matmul(np.matmul(H, A), H)
```

```python
def frobenius_distance(A, B):
    # Given two nxn matrices, return their 'Frobenius distance'

    return np.sqrt(np.sum(np.square(A - B)))
```

```python
loss_isomap = frobenius_distance(centered(D_geodesic), centered(D_isomap))/coords_length
loss_total = frobenius_distance(centered(D_geodesic), centered(D_proj))/coords_length
loss_proj = frobenius_distance(centered(D_isomap), centered(D_proj))/coords_length
```

```python
print(f"Isomap loss= {loss_isomap}")
print(f"Projection loss= {loss_proj}")
print(f"Total loss= {loss_total}")
```

    Isomap loss= 20.242536163460947
    Projection loss= 2.3225511781614396
    Total loss= 20.914716613612924

```python
(loss_total) - (loss_isomap + loss_proj) < 0
```

    True


```python
#@markdown Configuración Inicial

import os, sys
from IPython.utils import io
from IPython.display import display, display_svg
from IPython.display import Math, Latex
from IPython.display import IFrame, HTML
import time
from datetime import datetime as dt
from pytz import timezone
import random, string

prefix = 'Parallel'
configdict = {
    'Serialpc': "DISABLE_JIT: 1",
    'Parallel': "DISABLE_JIT: 0\n"
                "NUM_THREADS: 2\n"
                "THREADING_LAYER: 'tbb'"
}

################################################################################

workdir = prefix +\
          dt.now(timezone('America/Bogota')).strftime('_%y%m%d-%H%M_') +\
          ''.join(random.sample(string.hexdigits, 4)
          )
os.mkdir(workdir)
os.chdir(workdir)
print('Working Directory:', os.getcwd(), sep='\n')
with open('.numba_config.yaml','w') as cf:
  cf.write(configdict[prefix])
  print('Numba Config:', configdict[prefix], sep='\n', end='\n\n')

################################################################################

# https://stackoverflow.com/a/57883792
# https://stackoverflow.com/a/57113015
with io.capture_output() as cap:
  !pip install tbb
  !pip install setuptools
  # https://matplotlib-axes-aligner.readthedocs.io/en/latest/
  !pip install mpl-axes-aligner
  !pip install watermark
  !pip install tqdm --upgrade
  !pip install gradio

with open('pip_installs.txt', 'w') as f:
    f.write(cap.stdout)
!pip freeze > requirements.txt

################################################################################

import pandas as pd
import numpy as np
from numba import njit, prange, config
from numba.np.extensions import cross2d

if config.DISABLE_JIT==1:
  cross2d = np.cross
  prange = range
  print('JIT DISABLED!')
else:
  print('JIT!')
from IPython.display import set_matplotlib_formats

from numpy import linalg as LA
from numpy import random as rg
from scipy.interpolate import interp1d
from scipy import constants as const
from scipy import stats as st

import plotly.graph_objects as go
import matplotlib.pyplot as plt
import matplotlib.patches as patches
set_matplotlib_formats('pdf', 'svg')
from matplotlib import style
#style.use('classic')

from collections import Counter
from matplotlib.lines import Line2D
from matplotlib.collections import LineCollection
from matplotlib import colors as pltcolors
from matplotlib.colors import ListedColormap, BoundaryNorm
# https://matplotlib-axes-aligner.readthedocs.io/en/latest/
from mpl_axes_aligner import align

import gradio as gr
from tqdm.auto import trange, tqdm
from pprint import pprint

from sympy import Point, Polygon

#https://github.com/rasbt/watermark
%reload_ext watermark
%watermark -v -iv -m

!lscpu
!nvidia-smi
```

    Working Directory:
    /content/Parallel_211122-2102_aE53
    Numba Config:
    DISABLE_JIT: 0
    NUM_THREADS: 2
    THREADING_LAYER: 'tbb'

    JIT!
    Python implementation: CPython
    Python version       : 3.7.12
    IPython version      : 5.5.0

    Compiler    : GCC 7.5.0
    OS          : Linux
    Release     : 5.4.104+
    Machine     : x86_64
    Processor   : x86_64
    CPU cores   : 2
    Architecture: 64bit

    IPython         : 5.5.0
    sys             : 3.7.12 (default, Sep 10 2021, 00:21:48)
    [GCC 7.5.0]
    plotly          : 4.4.1
    scipy           : 1.4.1
    numpy           : 1.19.5
    numba           : 0.51.2
    matplotlib      : 3.2.2
    mpl_axes_aligner: 1.3
    gradio          : 2.4.6
    pandas          : 1.1.5

    Architecture:        x86_64
    CPU op-mode(s):      32-bit, 64-bit
    Byte Order:          Little Endian
    CPU(s):              2
    On-line CPU(s) list: 0,1
    Thread(s) per core:  2
    Core(s) per socket:  1
    Socket(s):           1
    NUMA node(s):        1
    Vendor ID:           GenuineIntel
    CPU family:          6
    Model:               79
    Model name:          Intel(R) Xeon(R) CPU @ 2.20GHz
    Stepping:            0
    CPU MHz:             2200.140
    BogoMIPS:            4400.28
    Hypervisor vendor:   KVM
    Virtualization type: full
    L1d cache:           32K
    L1i cache:           32K
    L2 cache:            256K
    L3 cache:            56320K
    NUMA node0 CPU(s):   0,1
    Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsaveopt arat md_clear arch_capabilities
    NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running.

```python
#@markdown Definición de Funciones

@njit(
  fastmath=True,
  error_model='numpy',
  )
def inner_angle(Model, i):
  k = len(Model)
  prev = Model[(i-1)%k]
  next = Model[(i+1)%k]
  here = Model[i%k]
  vetcs = prev-here, next-here
  #https://stackoverflow.com/a/14067171
  return -np.arctan2(
      cross2d(*vetcs),
      np.dot(*vetcs)
      )

@njit(
  fastmath=True,
  error_model='numpy',
  )
def grav(Model, XZ):
  k = len(Model)
  lenXZ = len(XZ)
  xietalist = [Model-XZ[i] for i in prange(lenXZ)]
  lenxieta = len(xietalist)
  grav = np.empty(lenxieta)
  for j in prange(lenxieta):
    xi = xietalist[j].T[0]
    eta = xietalist[j].T[1]
    sum = 0
    for i in prange(k):
      A = (xi[i-1]*eta[i] - xi[i]*eta[i-1])/\
          ((xi[i]-xi[i-1])**2 + (eta[i]-eta[i-1])**2)
      B1 = 0.5*(eta[i] - eta[i-1])*\
           np.log((xi[i]**2 + eta[i]**2)/\
          (xi[i-1]**2 + eta[i-1]**2))
      B2 = (xi[i] - xi[i-1])*\
           (np.arctan(xi[i]/eta[i])-\
           np.arctan(xi[i-1]/eta[i-1]))
      sum += A*(B1+B2)
    grav[j] = sum
  return grav

class Model(np.ndarray):
  def __new__(cls, input_array,*args,**kargs):
    return np.asarray(input_array).astype(np.float).view(cls)
  def __str__(self):
    return ','.join(map(str,self.flatten()))
  def area(self):
    # https://stackoverflow.com/a/30408825
    (x,z) = self.T
    return -0.5*(np.dot(x,np.roll(z,1))-np.dot(np.roll(x,1),z))
  def Cgeom(self):
    return np.array(np.mean(self,axis=0))
  def Cmass(self):
    # https://en.wikipedia.org/wiki/Centroid#Of_a_polygon
    k = self.__len__()
    A = self.area()
    (x,z) = self.T
    Cx = np.array(
        [(x[i%k] + x[(i+1)%k])*\
         (x[i%k]*z[(i+1)%k] - x[(i+1)%k]*z[i%k])\
         for i in range(k)]).sum()/A/6
    Cz = np.array(
        [(z[i%k] + z[(i+1)%k])*\
         (x[i%k]*z[(i+1)%k] - x[(i+1)%k]*z[i%k])\
         for i in range(k)]).sum()/A/6
    return np.array([Cx, Cz])
  def move(self, i, r, th):
    k = self.__len__()
    new = self.copy()
    new[i%k] = self[i%k] + r*np.array([np.cos(th), np.sin(th)])
    return new
  def birth(self, i, r, th):
    k = self.__len__()
    p = (self[i%k]+self[(i+1)%k])/2
    return np.insert(self, (i+1)%k, p, axis=0).move(i+1, r, th)
  def death(self, i):
    k = self.__len__()
    return np.delete(self, i%k, axis=0)
  def dvector(self, i):
    k = self.__len__()
    return np.array(
        (self[(i+1)%k] + self[(i-1)%k])/2-
        self[i%k]
        )
  def dradius(self,i):
    return norm(self.dvector(i))
  def dcat(self,i):
    k = self.__len__()
    return norm(self[(i+1)%k] - self[(i-1)%k])/2
  def bcat(self,i):
    k = self.__len__()
    return norm(self[(i+1)%k] - self[i%k])/2
  def vectors(self, i):
    k = self.__len__()
    prev = self[(i-1)%k]
    next = self[(i+1)%k]
    here = self[i%k]
    return np.array([prev-here, next-here])
  @njit(
    fastmath=True,
    error_model='numpy',
    )
  def angle(self, i):
    return inner_angle(self, i)
  def dists(self, i):
    mapvects = map(norm, self.vectors(i))
    return np.fromiter(mapvects, dtype=np.float)
  @njit(
    fastmath=True,
    error_model='numpy',
    )
  def gravitational(self, XZ, rho=600):
    g = grav(self, XZ)
    return np.column_stack((XZ, g))
  def notintersect(self):
  # https://github.com/lycantropos/bentley_ottmann
  # Shamos-Huey algorithms
   return not contour_self_intersects(
      bContour(list(map(lambda _: bPoint(*_), self))
      ))

def RegularModel(p, r, phi=np.pi/2, n=3):
  ths = phi+np.linspace(
      0, -2*np.pi, n,
      endpoint=False
      )
  return Model(
      [p + r*np.array([np.cos(th), np.sin(th)]) for th in ths])

def str2Model(str):
  return Model(eval(str)).reshape(-1, 2)

def Model_from_file(filename):
  with open(filename, 'r') as file:
    l = list(map(str2Model, file.readlines()))
    if len(l)==1:
      return l[0]
    else:
      return l
    file.close()

def Model_from_GeoGebra(filename):
  return Model(np.genfromtxt(filename))

def xz_iterp(XZ, xx, kind='cubic'):
  zz = interp1d(*XZ.T, kind='cubic')(xx)
  return np.c_[xx, zz]

print('Listo')
```

    Listo

```python
%%file ../TrueModel_from_GeoGebra.tsv
188	-95
193	-76
219	-72
228	-50
252	-54
265	-71
265	-93
239	-95
229	-119
204	-116
```

    Overwriting ../TrueModel_from_GeoGebra.tsv

```python
#@markdown Copia y pega la tabla de los vértices separados por tabulaciones para cambiar la forma del modelo de prueba
IFrame("https://www.geogebra.org/classic/cumzsmxp", 1200, 500)
```

```python
def Model_eye(s, q=(1,1), d=(0,0), N=19):
  m = N//2
  n = (N-m)
  q = np.array(q)/2
  gauss = st.norm(0, 1/s)
  f = lambda x: (gauss.pdf(x)-gauss.pdf(1))/\
                (gauss.pdf(0)-gauss.pdf(1))
  ii = np.r_[-1:+1:(n+1)*1j]
  jj = np.r_[+1:-1:(m+1)*1j]
  top    = np.c_[ii, +f(ii)][:-1]
  bottom = np.c_[jj, -f(jj)][:-1]
  eye = q*np.r_[top, bottom] + d
  return Model(eye)

eye_Model = Model_eye(2.5, (200, 200*9/16), (200, -100))
print('Eye Model:\n')
display_svg(Polygon(*map(Point, eye_Model)))
```

    Eye Model:

```python
#@markdown Settings
%%time

sigma = 0.2
gamma = 1.6
Maxk  = 20
ang   = 2*np.pi

old_settings = np.geterr()
new_settings = {
    'divide':  'raise',
    'invalid': 'raise',
    'over':    'ignore',
    'under':   'ignore'
    }
settlist = ['old_settings','new_settings']
display(pd.DataFrame(
    list(map(eval, settlist)),
    index=settlist
    ))
"""
ignore: Take no action when the exception occurs.
warn: Print a RuntimeWarning (via the Python warnings module).
raise: Raise a FloatingPointError.
call: Call a function specified using the seterrcall function.
print: Print a warning directly to stdout.
log: Record error in a Log object specified by seterrcall.
"""

GeoGebraModel = Model_from_GeoGebra('../TrueModel_from_GeoGebra.tsv')
TrueModel = GeoGebraModel # GeoGebraModel, eye_Model

w = np.ones(Maxk)
b = np.ones(Maxk)
d = np.ones(Maxk)
b[-1] = 0
d[2] = 0
ps = np.array([list(p/sum(p)) for p in np.stack((w,b,d)).T])
ps[0:2][:] = np.zeros((2,3))

print('sigma:', sigma)
print('gamma:', gamma)
print('Maxk:', Maxk)
print('ang:', ang)
print(ps)

XZ = (lambda a:
      np.c_[0:500:a,
            4*(1.5+np.cos(2*np.r_[0:2*np.pi:a]))]
      )(26j)

XZg = TrueModel.gravitational(XZ)
g = XZg[:,-1]
np.savetxt(
    '../Gravity.tsv',
    XZg,
    delimiter='\t',
    fmt='%.6g'
    )

XZmin = XZ
XZgrid = xz_iterp(XZ, np.r_[0:500:1001j])

try:
  print('True Model:\n')
  display_svg(Polygon(*map(Point, TrueModel.copy())))
except:
  print('Error or Polygon has intersecting sides.')

Xg    = TrueModel.gravitational(XZgrid)[:,(0,-1)]
Xgmin = TrueModel.gravitational(XZmin)[:,(0,-1)]
```

      divide
      over
      under
      invalid

      old_settings
      warn
      warn
      ignore
      warn

      new_settings
      raise
      ignore
      ignore
      raise

    sigma: 0.2
    gamma: 1.6
    Maxk: 20
    ang: 6.283185307179586
    [[0.         0.         0.        ]
     [0.         0.         0.        ]
     [0.5        0.5        0.        ]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.33333333 0.33333333 0.33333333]
     [0.5        0.         0.5       ]]
    True Model:

    CPU times: user 1.85 s, sys: 38.2 ms, total: 1.88 s
    Wall time: 1.88 s

```python
#@markdown Plot in 0

fig, ax_z = plt.subplots(figsize=np.array([16,9])*3/5)
ax_z.tick_params(
    top=True,
    labeltop=True
    )
ax_g = ax_z.twinx()
ax_z.axhline(c='k', lw=1, zorder=0)

ax_g.plot(
    *Xg.T,
    color='k'
    )
ax_g.scatter(
    *Xgmin.T,
    marker='.',
    color='k'
    )
lineg = Line2D(
  [0],[0],
  marker='.',
  color='k',
  label='Verdadera'
  )

ax_z.plot(
    *XZgrid.T,
    color='tab:green',
    label='_nolegend_'
)
ax_z.fill_between(
    *XZgrid.T,
    color='tab:green',
    alpha=3/5
)
ax_z.scatter(
    *XZmin.T,
    marker='.',
    color='tab:green',
    label='_nolegend_'
)
linez = Line2D(
  [0],[0],
  marker='.',
  color='tab:green',
  label='Terreno'
  )

Truepoly = patches.Polygon(
    TrueModel,
    closed=True,
    Fill=True,
    edgecolor=None,
    label='Verdadero'
    )
ax_z.add_patch(Truepoly)

ax_z.scatter(
    *TrueModel.Cmass(),
    marker='x',
    color='r',
    zorder=2
    )

ax_z.set_xlabel(r'$x$')
ax_z.set_ylabel(r'$z$', rotation=0)
ax_g.set_ylabel(r'$\quad\Delta{g}$', rotation=0, y=5/6)

ax_z.set_xlim(0, 500)
ax_z.set_ylim(ymin=-200)
ax_g.set_yticks(np.r_[0:100:5j])
align.yaxes(ax_z, 0, ax_g, 0, 2/3)

ax_g.legend(
    handles=[lineg, linez],
    title='Gravimetría',
    bbox_to_anchor=(0.25, -.1),
    ncol=3,
    loc='upper center'
    )
ax_z.legend(
    handles=[Truepoly],
    title='Modelo',
    bbox_to_anchor=(0.75, -.1),
    ncol=2,
    loc='upper center'
    )
plt.savefig('InitPlot.pdf', bbox_inches='tight')
plt.show()
```

```python
!cat ../Gravity.tsv
```

    0	10	5.00469
    20	9.50523	5.81984
    40	8.14331	6.80133
    60	6.25116	8.02252
    80	4.29688	9.59481
    100	2.76393	11.6699
    120	2.03154	14.4198
    140	2.28089	17.9639
    160	3.4503	22.2207
    180	5.25047	26.7409
    200	7.23607	30.6812
    220	8.91587	33.0171
    240	9.87433	32.9741
    260	9.87433	30.536
    280	8.91587	26.4886
    300	7.23607	21.8943
    320	5.25047	17.5795
    340	3.4503	13.9663
    360	2.28089	11.1482
    380	2.03154	9.03614
    400	2.76393	7.47582
    420	4.29688	6.31369
    440	6.25116	5.42419
    460	8.14331	4.71545
    480	9.50523	4.12576
    500	10	3.61728

```python
#@title Gradio
examples = [
["""56	-54
106	-31
154	-34
196	-52
207	-82
181	-96
163	-120
124	-120
81.	-98
58.	-69"""],
["""108	-108
144	-48
192	-96
120	-120
96	-192
48	-144"""]
]

data = examples[0][0]
def my_fun(data):
  df=pd.DataFrame([x.split('\t')
  for x in data.split('\n')], dtype=np.float)
  TrueModel=Model(df)

  XZ = (lambda a:
      np.c_[0:500:a,
            4*(1.5+np.cos(2*np.r_[0:2*np.pi:a]))]
      )(26j)

  XZmin = XZ
  XZgrid = xz_iterp(XZ, np.r_[0:500:1001j])

  Xg    = TrueModel.gravitational(XZgrid)[:,(0,-1)]
  Xgmin = TrueModel.gravitational(XZmin)[:,(0,-1)]

  fig, ax_z = plt.subplots(figsize=np.array([16,9])*3/5)
  ax_z.tick_params(
      top=True,
      labeltop=True
      )
  ax_g = ax_z.twinx()
  ax_z.axhline(c='k', lw=1, zorder=0)

  ax_g.plot(
      *Xg.T,
      color='k'
      )
  ax_g.scatter(
      *Xgmin.T,
      marker='.',
      color='k'
      )
  lineg = Line2D(
    [0],[0],
    marker='.',
    color='k',
    label='Verdadera'
    )

  ax_z.plot(
      *XZgrid.T,
      color='tab:green',
      label='_nolegend_'
  )
  ax_z.fill_between(
      *XZgrid.T,
      color='tab:green',
      alpha=3/5
  )
  ax_z.scatter(
      *XZmin.T,
      marker='.',
      color='tab:green',
      label='_nolegend_'
  )
  linez = Line2D(
    [0],[0],
    marker='.',
    color='tab:green',
    label='Terreno'
    )

  Truepoly = patches.Polygon(
      TrueModel,
      closed=True,
      Fill=True,
      edgecolor=None,
      label='Verdadero'
      )
  ax_z.add_patch(Truepoly)

  ax_z.scatter(
      *TrueModel.Cmass(),
      marker='x',
      color='r',
      zorder=2
      )

  ax_z.set_xlabel(r'$x$')
  ax_z.set_ylabel(r'$z$', rotation=0)
  ax_g.set_ylabel(r'$\quad\Delta{g}$', rotation=0, y=5/6)

  ax_z.set_xlim(0, 500)
  ax_z.set_ylim(ymin=-200)
  ax_g.set_yticks(np.linspace(0, 100, 5))
  align.yaxes(ax_z, 0, ax_g, 0, 2/3)

  ax_g.legend(
      handles=[lineg, linez],
      title='Gravimetría',
      bbox_to_anchor=(0.25, -.1),
      ncol=3,
      loc='upper center'
      )
  ax_z.legend(
      handles=[Truepoly],
      title='Modelo',
      bbox_to_anchor=(0.75, -.1),
      ncol=2,
      loc='upper center'
      )
  fig.tight_layout()
  plt.close()
  return fig

title = "Metodo Directo"
description = 'Copia y pega la tabla de los vértices separados por tabulaciones para cambiar la forma del modelo de prueba'
article = """<p style='text-align: center'>

    <a href='https://edwardptera.github.io/Gravimetro/'>Gravimetro</a> |
    <a href='https://github.com/edwardptera/Gravimetro'>Github</a>
    </p>"""

articl0 = """<p style='text-align: center'>
    <a href='https://edwardptera.github.io/Gravimetro/'>Gravimetro</a> |
    <a href='https://github.com/edwardptera/Gravimetro'>Github</a>
    </p>"""

grtest = gr.Interface(
    fn=my_fun,
    inputs=gr.inputs.Textbox(
        lines=11,
        placeholder=data,
        # default=data
        ),
    # inputs=gr.inputs.Dataframe(
    #     col_count=2,
    #     # default=w,
    #     datatype='number'
    #     ),
    outputs='plot',
    live=True,
    allow_flagging=False,
    allow_screenshot=False,
    title=title,
    description=description,
    article=article,
    examples=examples,
    theme='huggingface', # "default", "compact" or "huggingface"
    layout='unaligned' # 'horizontal', 'unaligned', 'vertical'
    )
# grtest.launch(inline=True, debug=True)
with io.capture_output() as captured:
    grtest.launch(inline=True)
print(grtest.share_url)
IFrame(src=grtest.share_url, width=1500, height=700)
# grtest.close()
```

    https://47988.gradio.app

```python
#@markdown Archivo `.zip`

!rm -f zfiles.zip
!7z a -bso0 ../{workdir}.zip *.* -x!*.zip
!7z l ../{workdir}.zip
```

      0M Scan             0%
    7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
    p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)

    Scanning the drive for archives:
      0M Scan ../             1 file, 27329 bytes (27 KiB)

    Listing archive: ../Parallel_211122-2102_aE53.zip

    --
    Path = ../Parallel_211122-2102_aE53.zip
    Type = zip
    Physical Size = 27329

       Date      Time    Attr         Size   Compressed  Name
    ------------------- ----- ------------ ------------  ------------------------
    2021-11-23 02:03:13 .....        29116        22289  InitPlot.pdf
    2021-11-23 02:03:06 .....         7683         1092  pip_installs.txt
    2021-11-23 02:03:07 .....         7887         3502  requirements.txt
    ------------------- ----- ------------ ------------  ------------------------
    2021-11-23 02:03:13              44686        26883  3 files


Sebastian Raschka 2014

```python
%load_ext watermark
```

```python
%watermark -v -u -d -p scipy,scikit-learn,numpy,matplotlib
```

    last updated: 2016-07-11

    CPython 3.5.1
    IPython 4.2.0

    scipy 0.17.1
    scikit-learn 0.17.1
    numpy 1.11.0
    matplotlib 1.5.1

[More information](https://github.com/rasbt/watermark) about the `watermark` magic command extension.

I would be happy to hear your comments and suggestions.
Please feel free to drop me a note via
[twitter](https://twitter.com/rasbt), [email](mailto:bluewoodtree@gmail.com), or [google+](https://plus.google.com/+SebastianRaschka).

# Kernel tricks and nonlinear dimensionality reduction via PCA

## Sections

- [Principal Component Analysis](#Principal-Component-Analysis)
- [PCA and linear dimensionality reduction](#PCA-and-linear-dimensionality-reduction)
- [Nonlinear dimensionality reduction](#Nonlinear-dimensionality-reduction)
- [Kernel functions and the kernel trick](Kernel-functions-and-the-kernel-trick)
- [Gaussian RBF Kernel PCA](#Gaussian-RBF-Kernel-PCA)
- [Implementing the RBF kernel PCA step-by-step](#Implementing-the-RBF-kernel-PCA-step-by-step)
- [Examples of RBF Kernel PCA](#Examples-of-RBF-Kernel-PCA)
    - [Half-moon shapes](#Half-moon-shapes)
    - [Concentric circles](#Concentric-circles)
    - [Swiss roll](#Swiss-roll)
- [Appendix A: Projecting new data](#Appendix-A:-Projecting-new-data)
- [References](#References)

**Most machine learning algorithms have been developed and statistically validated for linearly separable data. Popular examples are linear classifiers like Support Vector Machines (SVMs) or the (standard) Principal Component Analysis (PCA) for dimensionality reduction. However, most real world data requires nonlinear methods in order to perform tasks that involve the analysis and discovery of patterns successfully.**

**The focus of this article is to briefly introduce the idea of kernel methods and to implement a Gaussian radius basis function (RBF) kernel that is used to perform nonlinear dimensionality reduction via BF kernel principal component analysis (kPCA).**

## Principal Component Analysis

The main purpose of principal component analysis (PCA) is the analysis of data to identify patterns that represent the data "well." The principal components can be understood as new axes of the dataset that maximize the variance along those axes (the eigenvectors of the covariance matrix). In other words, PCA aims to find the axes with maximum variances along which the data is most spread.

## PCA and linear dimensionality reduction

[[back to top](#Sections)]

A common application of PCA is to reduce the dimensions of the dataset with minimal loss of information.
Here, the entire dataset (*d* dimensions) is projected onto a new subspace (*k* dimensions where *k* < *d*).
This method of projection is useful in order to reduce the computational costs and the error of parameter estimation ("curse of dimensionality").

The standard PCA approach can be summarized in six simple steps:

More details can be found in a previous article ["Implementing a Principal Component Analysis (PCA) in Python step by step"](http://sebastianraschka.com/Articles/2014_pca_step_by_step.html).

## Nonlinear dimensionality reduction

[[back to top](#Sections)]

The "classic" PCA approach described above is a linear projection technique that works well if the data is linearly separable. However, in the case of linearly inseparable data, a nonlinear technique is required if the task is to reduce the dimensionality of a dataset.

## Kernel functions and the kernel trick

[[back to top](#Sections)]

The basic idea to deal with linearly inseparable data is to project it onto a higher dimensional space where it becomes linearly separable. Let us call this nonlinear mapping function $\phi$ so that the mapping of a sample $\mathbf{x}$ can be written as $\mathbf{x} \rightarrow \phi (\mathbf{x})$, which is called "kernel function."

Now, the term "kernel" describes a function that calculates the dot product of the images of the samples $\mathbf{x}$ under $\phi$.

\begin{equation}\kappa(\mathbf{x_i, x_j}) =  \phi (\mathbf{x_i}) \phi (\mathbf{x_j})^T    \end{equation}

More details about the derivation of this equation are provided in this excellent review article by Quan Wang: [Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models](http://arxiv.org/abs/1207.3538).[[1](#References)]

In other words, the function $\phi$ maps the original d-dimensional features into a larger, k-dimensional feature space by creating nononlinear combinations of the original features. For example, if $\mathbf{x}$ consists of 2 features:

\begin{equation} \mathbf{x} = \big[x_1 \quad x_2\big]^T \quad \quad \mathbf{x} \in I\!R^d\\
\Downarrow \phi \\
 \mathbf{x}' = \big[x_1 \quad x_2 \quad x_1 x_2 \quad x_{1}^2 \quad x_1 x_{2}^3 \quad \dots \big]^T \quad \quad \mathbf{x} \in I\!R^k (k >> d) \end{equation}

Often, the mathematical definition of the RBF kernel is written and implemented as

\begin{equation} \kappa(\mathbf{x_i, x_j}) = exp\bigg(- \gamma \; \lVert\mathbf{x_i - x_j }\rVert^{2}_{2} \bigg)\end{equation}

where $\textstyle\gamma = \tfrac{1}{2\sigma^2}$ is a free parameter that is to be optimized.

## Gaussian radial basis function (RBF) Kernel PCA

[[back to top](#Sections)]

In the linear PCA approach, we are interested in the principal components that maximize the variance in the dataset. This is done by extracting the eigenvectors (principle components) that correspond to the largest eigenvalues based on the covariance matrix:

\begin{equation}\text{Cov} = \frac{1}{N} \sum_{i=1}^{N} \mathbf{x_i} \mathbf{x_i}^T \end{equation}

Bernhard Scholkopf ([Kernel Principal Component Analysis](http://dl.acm.org/citation.cfm?id=299113) [[2](#References)]) generalized this approach for data that was mapped onto the higher dimensional space via a kernel function:

\begin{equation}\text{Cov} = \frac{1}{N} \sum_{i=1}^{N} \phi(\mathbf{x_i}) \phi(\mathbf{x_i})^T \end{equation}

However, in practice the the covariance matrix in the higher dimensional space is not calculated explicitly (kernel trick). Therefore, the implementation of RBF kernel PCA does not yield the principal component axes (in contrast to the standard PCA), but the obtained eigenvectors can be understood as projections of the data onto the principal components.

## Implementing the RBF kernel PCA step-by-step

[[back to top](#Sections)]

In order to implement the RBF kernel PCA we just need to consider the following two steps.

##### 1. Computation of the kernel (similarity) matrix.

In this first step, we need to calculate

\begin{equation} \kappa(\mathbf{x_i, x_j}) = exp\bigg(- \gamma \; \lVert\mathbf{x_i - x_j }\rVert^{2}_{2} \bigg)\end{equation}

for every pair of points. E.g., if we have a dataset of 100 samples, this step would result in a symmetric 100x100 kernel matrix.

##### 2. Eigendecomposition of the kernel matrix.

Since it is not guaranteed that the kernel matrix is centered, we can apply the following equation to do so:

\begin{equation} K' = K - \mathbf{1_N} K - K \mathbf{1_N} + \mathbf{1_N} K \mathbf{1_N} \end{equation}

where $\mathbf{1_N}$ is (like the kernel matrix) a $N\times N$ matrix with all values equal to $\frac{1}{N}$. [[3](#References)]

Now, we have to obtain the eigenvectors of the centered kernel matrix that correspond to the largest eigenvalues. Those eigenvectors are the data points already projected onto the respective principal components.

Below, we implement those steps in Python to see how those computations work.

```python
from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh
import numpy as np

def stepwise_kpca(X, gamma, n_components):
    """
    Implementation of a RBF kernel PCA.

    Arguments:
        X: A MxN dataset as NumPy array where the samples are stored as rows (M),
           and the attributes defined as columns (N).
        gamma: A free parameter (coefficient) for the RBF kernel.
        n_components: The number of components to be returned.

    """
    # Calculating the squared Euclidean distances for every pair of points
    # in the MxN dimensional dataset.
    sq_dists = pdist(X, 'sqeuclidean')

    # Converting the pairwise distances into a symmetric MxM matrix.
    mat_sq_dists = squareform(sq_dists)

    # Computing the MxM kernel matrix.
    K = exp(-gamma * mat_sq_dists)

    # Centering the symmetric NxN kernel matrix.
    N = K.shape[0]
    one_n = np.ones((N,N)) / N
    K = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n)

    # Obtaining eigenvalues in descending order with corresponding
    # eigenvectors from the symmetric matrix.
    eigvals, eigvecs = eigh(K)

    # Obtaining the i eigenvectors that corresponds to the i highest eigenvalues.
    X_pc = np.column_stack((eigvecs[:,-i] for i in range(1,n_components+1)))

    return X_pc
```

# Examples of RBF Kernel PCA

[[back to top](#Sections)]

In this section, we will apply the RBF kernel PCA to different nonlinear sample data in order to perform dimensionality reduction.

## Half-moon shapes

[[back to top](#Sections)]

We will start with a simple example of 2 half-moon shapes generated by the [`make_moons`](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_moons.html) function from [scikit-learn](http://scikit-learn.org/stable/index.html).

```python
%matplotlib inline
```

```python
import matplotlib.pyplot as plt

from sklearn.datasets import make_moons
X, y = make_moons(n_samples=100, random_state=123)

plt.figure(figsize=(8,6))

plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5)
plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5)

plt.title('A nonlinear 2Ddataset')
plt.ylabel('y coordinate')
plt.xlabel('x coordinate')

plt.show()
```

### Linear PCA

[[back to top](#Sections)]

Since the two half-moon shapes are linearly inseparable, we expect that the "classic" PCA will fail to give us a "good" representation of the data in 1D space. Here, we will use the [`PCA`](http://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html) class that is implemented in scikit-learn to perform the dimensionality reduction.

```python
from sklearn.decomposition import PCA

scikit_pca = PCA(n_components=2)
X_spca = scikit_pca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_spca[y==0, 0], X_spca[y==0, 1], color='red', alpha=0.5)
plt.scatter(X_spca[y==1, 0], X_spca[y==1, 1], color='blue', alpha=0.5)

plt.title('First 2 principal components after Linear PCA')
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()
```

```python
import numpy as np
scikit_pca = PCA(n_components=1)
X_spca = scikit_pca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_spca[y==0, 0], np.zeros((50,1)), color='red', alpha=0.5)
plt.scatter(X_spca[y==1, 0], np.zeros((50,1)), color='blue', alpha=0.5)

plt.title('First principal component after Linear PCA')
plt.xlabel('PC1')

plt.show()
```

As we can see, the resulting principal components do not yield a subspace where the data is linearly separated well. Note that PCA is a unsupervised method and does not "consider" class labels in order to maximize the variance in contrast to [Linear Discriminant Analysis](http://sebastianraschka.com/Articles/2014_python_lda.html). Here, the colors blue and red are just added for visualization purposes to indicate the degree of separation.

### Gaussian RBF kernel PCA

[[back to top](#Sections)]

Next, we will perform dimensionality reduction via RBF kernel PCA on our half-moon data. The choice of $\gamma$ depends on the dataset and can be obtained via hyperparameter tuning techniques like Grid Search. Hyperparameter tuning is a broad topic itself, and here I will just use a $\gamma$-value that I found to produce "good" results.

```python
X_pc = stepwise_kpca(X, gamma=15, n_components=2)

plt.figure(figsize=(8,6))
plt.scatter(X_pc[y==0, 0], X_pc[y==0, 1], color='red', alpha=0.5)
plt.scatter(X_pc[y==1, 0], X_pc[y==1, 1], color='blue', alpha=0.5)

plt.title('First 2 principal components after RBF Kernel PCA')
plt.text(-0.18, 0.18, 'gamma = 15', fontsize=12)
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()
```

```python
plt.figure(figsize=(8,6))
plt.scatter(X_pc[y==0, 0], np.zeros((50)), color='red', alpha=0.5)
plt.scatter(X_pc[y==1, 0], np.zeros((50)), color='blue', alpha=0.5)

plt.title('First principal component after RBF Kernel PCA')
plt.text(-0.17, 0.007, 'gamma = 15', fontsize=12)
plt.xlabel('PC1')
plt.show()
```

We can clearly see that the projection via RBF kernel PCA yielded a subspace where the classes are separated well. Such a subspace can then be used as input for linear classification models, such as Support Vector Machines or naive Bayes classifiers, which will be covered in future articles.

### scikit RBF kernel PCA

[[back to top](#Sections)]

For our convenience, there is already an implementation of the [`KernelPCA`](http://scikit-learn.org/stable/modules/generated/sklearn.decomposition.KernelPCA.html) in scikit-learn.
Let us confirm that the results of our implementation is consistent with scikit-learn's approach.

```python
from sklearn.decomposition import KernelPCA

scikit_kpca = KernelPCA(n_components=2, kernel='rbf', gamma=15)
X_skernpca = scikit_kpca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_skernpca[y==0, 0], X_skernpca[y==0, 1], color='red', alpha=0.5)
plt.scatter(X_skernpca[y==1, 0], X_skernpca[y==1, 1], color='blue', alpha=0.5)

plt.text(-0.48, 0.35, 'gamma = 15', fontsize=12)
plt.title('First 2 principal components after RBF Kernel PCA via scikit-learn')
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()
```

```python
scikit_kpca = KernelPCA(n_components=1, kernel='rbf', gamma=15)
X_skernpca = scikit_kpca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_skernpca[y==0, 0], np.zeros((50,1)), color='red', alpha=0.5)
plt.scatter(X_skernpca[y==1, 0], np.zeros((50,1)), color='blue', alpha=0.5)
plt.text(-0.48, 0.007, 'gamma = 15', fontsize=12)
plt.title('First principal component after RBF Kernel PCA')
plt.xlabel('PC1')
plt.show()
```

## Concentric circles

[[back to top](#Sections)]

For our next example, we will have a look at the classic case of 2 concentric circles with random noise produced by scikit-learn's [`make_circles`](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_circles.html).

```python
from sklearn.datasets import make_circles

X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2)

plt.figure(figsize=(8,6))

plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5)
plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5)
plt.title('Concentric circles')
plt.ylabel('y coordinate')
plt.xlabel('x coordinate')
plt.savefig('/Users/Sebastian/Desktop/circles1.pdf')
```

### Linear PCA

[[back to top](#Sections)]

```python
scikit_pca = PCA(n_components=2)
X_spca = scikit_pca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X[y==0, 0], np.zeros((500,1))+0.1, color='red', alpha=0.5)
plt.scatter(X[y==1, 0], np.zeros((500,1))-0.1, color='blue', alpha=0.5)
plt.ylim([-15,15])
plt.text(-0.125, 12.5, 'gamma = 15', fontsize=12)
plt.title('First principal component after Linear PCA')
plt.xlabel('PC1')
plt.savefig('/Users/Sebastian/Desktop/circles2.pdf')
```

Again, the results obtained via the linear PCA approach does not produce a subspace where the 2 classes are linearly well separated.

### Gaussian RBF kernel PCA

[[back to top](#Sections)]

```python
X_pc = stepwise_kpca(X, gamma=15, n_components=1)

plt.figure(figsize=(8,6))
plt.scatter(X_pc[y==0, 0], np.zeros((500,1)), color='red', alpha=0.5)
plt.scatter(X_pc[y==1, 0], np.zeros((500,1)), color='blue', alpha=0.5)
plt.text(-0.05, 0.007, 'gamma = 15', fontsize=12)
plt.title('First principal component after RBF Kernel PCA')
plt.xlabel('PC1')

plt.savefig('/Users/Sebastian/Desktop/circles3.pdf')
```

And again, this 1-dimensional subspace obtained via Gaussian RBF kernel PCA looks much better in terms of linear class separation.

## Swiss roll

[[back to top](#Sections)]

Unrolling the famous Swiss roll is a more challenging task than the examples we have seen above. We will use the [`make_swiss_roll`](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_swiss_roll.html) to create 3-dimensional Swiss roll and start with the linear PCA to project the dataset onto a 2D and 1D feature subspace.

```python
from sklearn.datasets.samples_generator import make_swiss_roll
from mpl_toolkits.mplot3d import Axes3D

X, color = make_swiss_roll(n_samples=800, random_state=123)

fig = plt.figure(figsize=(7,7))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.rainbow)
plt.title('Swiss Roll in 3D')
plt.show()
```

### Linear PCA

[[back to top](#Sections)]

```python
from sklearn.decomposition import PCA

scikit_pca = PCA(n_components=2)
X_spca = scikit_pca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_spca[:, 0], X_spca[:, 1], c=color, cmap=plt.cm.rainbow)

plt.title('First 2 principal components after Linear PCA')
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()
```

```python
scikit_pca = PCA(n_components=1)
X_spca = scikit_pca.fit_transform(X)

plt.figure(figsize=(8,6))
plt.scatter(X_spca, np.zeros((800,1)), c=color, cmap=plt.cm.rainbow)
plt.title('First principal component after Linear PCA')
plt.xlabel('PC1')
plt.show()
```

### Gaussian RBF kernel PCA

[[back to top](#Sections)]

I haven't found a good $\gamma$ parameter for the Gaussian RBF kernel for good linear separation of this dataset. The best result I obtained is shown in the following figures.

```python
X_pc = stepwise_kpca(X, gamma=0.1, n_components=2)

plt.figure(figsize=(8,6))
plt.scatter(X_pc[:, 0], X_pc[:, 1], c=color, cmap=plt.cm.rainbow)

plt.title('First 2 principal components after RBF Kernel PCA')
plt.text(-0.14, 0.14, 'gamma = 0.1', fontsize=12)
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()
```

```python
plt.figure(figsize=(8,6))
plt.scatter(X_pc[:,0], np.zeros((800,1)), c=color, cmap=plt.cm.rainbow)

plt.text(-0.125, 0.007, 'gamma = 0.1', fontsize=12)
plt.title('First principal component after RBF Kernel PCA')
plt.xlabel('PC1')
plt.show()
```

### Locally-Linear Embedding (LLE)

[[back to top](#Sections)]

In 2000, Sam T. Roweis and Lawrence K. Saul ([Nonlinear dimensionality reduction by locally linear embedding](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.111.3313) [[4](#References)]) introduced an unsupervised learning algorithm called locally linear embedding (LLE) that is better suited to identify patterns in the high-dimensional feature space and solves our problem of nonlinear dimensionality reduction for the Swiss roll.
Here, we will use the [`locally_linear_embedding`](http://scikit-learn.org/stable/modules/generated/sklearn.manifold.locally_linear_embedding.html) class from scikit-learn to "unroll" the Swiss roll.

```python
from sklearn.manifold import locally_linear_embedding

X_lle, err = locally_linear_embedding(X, n_neighbors=12, n_components=2)

plt.figure(figsize=(8,6))
plt.scatter(X_lle[:, 0], X_lle[:, 1], c=color, cmap=plt.cm.rainbow)

plt.title('First 2 principal components after Locally Linear Embedding')
plt.show()
```

```python
from sklearn.manifold import locally_linear_embedding

X_lle, err = locally_linear_embedding(X, n_neighbors=12, n_components=1)

plt.figure(figsize=(8,6))
plt.scatter(X_lle, np.zeros((800,1)), c=color, cmap=plt.cm.rainbow)

plt.title('First principal component after Locally Linear Embedding')
plt.show()
```

# Appendix A: Projecting new data

[[back to top](#Sections)]

So far, so good, in the sections above, we have been projecting an dataset onto a new feature subspace. However, in a real application, we are usually interested to map new data points onto the same new feature subspace (e.g., if are working with a training and a test dataset in pattern classification tasks).

Remember, when we computed the eigenvectors \\( \mathbf{\alpha} \\) of the centered kernel matrix, those values were actually already the projected datapoints onto the principal component axis \\( \mathbf{g} \\).

If we want to project a new data point \\( \mathbf{x} \\) onto this principal component axis, we'd need to compute \\(\phi(\mathbf{x})^T  \mathbf{g} \\).

Fortunately, also here, we don't have to compute \\(\phi(\mathbf{x})^T  \mathbf{g} \\) explicitely but use the kernel trick to calculate the RBF kernel between the new data point and every data point \\( j \\) in the training dataset:

$$\phi(\mathbf{x})^T  \mathbf{g}  = \sum_j \alpha_{i} \; \phi(\mathbf{x}) \; \phi(\mathbf{x_j})^T$$

$$=  \sum_j \alpha_{i} \; \kappa(\mathbf{x}, \mathbf{x_j})$$

and the eigenvectors \\( \alpha \\) and eigenvalues \\( \lambda \\) of the Kernel matrix \\(\mathbf{K}\\) satisfy the equation
\\(\mathbf{K} \alpha = \lambda \alpha \\), we just need to normalize the eigenvector by the corresponding eigenvalue.

First, let us modify our original implemenation by returning the corresponding to return also the eigenvalues of the kernel matrix.

```python
from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh
import numpy as np

def stepwise_kpca(X, gamma, n_components):
    """
    Implementation of a RBF kernel PCA.

    Arguments:
        X: A MxN dataset as NumPy array where the samples are stored as rows (M),
           and the attributes defined as columns (N).
        gamma: A free parameter (coefficient) for the RBF kernel.
        n_components: The number of components to be returned.

    Returns the k eigenvectors (alphas) that correspond to the k largest
        eigenvalues (lambdas).

    """
    # Calculating the squared Euclidean distances for every pair of points
    # in the MxN dimensional dataset.
    sq_dists = pdist(X, 'sqeuclidean')

    # Converting the pairwise distances into a symmetric MxM matrix.
    mat_sq_dists = squareform(sq_dists)

    # Computing the MxM kernel matrix.
    K = exp(-gamma * mat_sq_dists)

    # Centering the symmetric NxN kernel matrix.
    N = K.shape[0]
    one_n = np.ones((N,N)) / N
    K_norm = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n)

    # Obtaining eigenvalues in descending order with corresponding
    # eigenvectors from the symmetric matrix.
    eigvals, eigvecs = eigh(K_norm)

    # Obtaining the i eigenvectors (alphas) that corresponds to the i highest eigenvalues (lambdas).
    alphas = np.column_stack((eigvecs[:,-i] for i in range(1,n_components+1)))
    lambdas = [eigvals[-i] for i in range(1,n_components+1)]

    return alphas, lambdas
```

Now, let's make a new half-moon dataset and project it onto a 1-dimensonal subspace using the RBF kernel PCA:

```python
from sklearn.datasets import make_moons
X, y = make_moons(n_samples=100, random_state=123)
alphas, lambdas = stepwise_kpca(X, gamma=15, n_components=1)
```

    /Users/sebastian/miniconda3/envs/py34/lib/python3.4/site-packages/sklearn/datasets/samples_generator.py:612: DeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
      y = np.hstack([np.zeros(n_samples_in, dtype=np.intp),

To confirm that our approach produces the correct results, let's pretend that the 24th point from the half-moon dataset is a new data point  \\( \mathbf{x} \\), and we want to project it onto this new subspace.

```python
x_new = X[25]
X_proj = alphas[25] # original projection
```

```python
x_new
```

    array([ 1.8713187 ,  0.00928245])

```python
X_proj
```

    array([ 0.07877284])

```python
def project_x(x_new, X, gamma, alphas, lambdas):
    pair_dist = np.array([np.sum((x_new-row)**2) for row in X])
    k = np.exp(-gamma * pair_dist)
    return k.dot(alphas / lambdas)

# projection of the "new" datapoint
x_reproj = project_x(x_new, X, gamma=15, alphas=alphas, lambdas=lambdas)
```

```python
x_reproj
```

    array([ 0.07877284])

```python
%matplotlib inline
```

```python
import matplotlib.pyplot as plt
plt.figure(figsize=(8,6))
plt.scatter(alphas[y==0, 0], np.zeros((50)), color='red', alpha=0.5)
plt.scatter(alphas[y==1, 0], np.zeros((50)), color='blue', alpha=0.5)
plt.scatter(X_proj, 0, color='black', label='original projection of point X[24]', marker='^', s=100)
plt.scatter(x_reproj, 0, color='green', label='remapped point X[24]', marker='x', s=500)
plt.legend(scatterpoints=1)
plt.show()
```

# References

[[back to top](#Sections)]

[1] Q. Wang. [Kernel principal component analysis and its applications in face recognition and active shape models](http://arxiv.org/abs/1207.3538). CoRR, abs/1207.3538, 2012.

[2] B. Scholkopf, A. Smola, and K.-R. Muller. [Kernel principal component analysis](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.128.7613). pages 583–588, 1997.

[3] B. Scholkopf, A. Smola, and K.-R. Muller. [Nonlinear component analysis as a kernel eigenvalue problem](http://www.mitpressjournals.org/doi/abs/10.1162/089976698300017467#.VBh9QkuCFHg). Neural computation, 10(5):1299–1319, 1998.

[4] S. T. Roweis and L. K. Saul. [Nonlinear dimensionality reduction by locally linear embedding](http://www.sciencemag.org/content/290/5500/2323.short). Science, 290(5500):2323–2326, 2000.

```python

```


# Theory documentation for `pneumoinfer`

```python
# Add pneumoinfer to the system path
import sys
path = '/Users/Rob/work/pneumoinfer'
sys.path.append(path + '/source/')
from pneumoinfer import pneumoinfer

import numpy as np
import pandas as pd
import scipy.special as spec
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
```

---

## Motivating background

Multi-state models - stochastic processes occupying one of a finite set of states at each moment in time - appear to describe many natural phenomena, but are probably most frequently used in the mathematical modelling of population health. The statistical inference (or selection) of these models for real-world applications frequently involves data in the form of a sequence of individual state observations, which are often coupled with some diagnostic uncertainty.

There are over 90 known capsular serotypes of _Streptococcus pneumoniae_, which persist despite their mutual competition for the same ecological niche (the nasopharynx) and a known fitness gradient. Motivated by the global pneumococcal disease burden, a specific class of multi-state models has been developed to describe the carriage dynamics which offers a neat explanation of this persistence through immunity-driven stabilisation effects (see [Cobey & Lipsitch (2012)](https://pubmed.ncbi.nlm.nih.gov/22383809/)). This class of model typically uses a counting memory of past state (or serotype) occupations (or colonisations) as a model for human immunity (see, e.g., [Flasche et al. (2013)](https://royalsocietypublishing.org/doi/10.1098/rspb.2013.1939) for an alternative formulation and [Løchen & Anderson (2020)](https://pubmed.ncbi.nlm.nih.gov/31055164/) for a general review of the carriage transmission models). Building from these mathematical models, a range of statistical approaches have also been used to infer the pneumococcal carriage through a given population from nasopharyngeal swab sample data (e.g., [Lipsitch et al. (2012)](https://pubmed.ncbi.nlm.nih.gov/22441543/) and [Numminen et al. (2013)](https://pubmed.ncbi.nlm.nih.gov/23822205/)). All of this is obviously really important, e.g., to understanding more precisely how a vaccine covering a restricted range of serotypes can impact colonisation in a given community or region.

The design of policies for gathering data will always have a direct impact on the quality and utility of information that can be learned about a model via statistical inference. Therefore, it's typically useful to know _a priori_ the fundamental constraints a given policy might impose on this procedure. The purpose of the `pneumoinfer` class is to provide researchers with a rigorous framework to investigate these limitations for the inference of multi-state models with a counting memory - which are structurally inspired by the pneumococcus carriage models of [Cobey & Lipsitch (2012)](https://pubmed.ncbi.nlm.nih.gov/22383809/) and [Lipsitch et al. (2012)](https://pubmed.ncbi.nlm.nih.gov/22441543/). The framework should also useful in model inference with real data.

In this documentation, we're going to analyse the master equation of a stochastic model which includes memory effects from individual immunity and investigate a novel (to our knowledge) approximate ODE description for the dynamics, while assessing its validity. By then exploiting the new efficient ODE description, we will be able to develop a new method of inference that is very rapid in comparison to simulated likelihoods (or even ABC/likelihood-free inference methods). This is the main inference method that is implemented in the `pneumoinfer` class.

---

## The fixed $\Lambda_i$ model

Let's now construct a multi-state model which incorporates a counting memory of past state occupations. This will include: an event rate of state clearance $\tilde{\mu}_i$ - the rate at which an individual occupying the $i$-th indexed state returns to the null state; an event rate of susceptibility $\tilde{\Lambda}_i$ for an individual moving to the $i$-th state from the null state; and a state-specific factor matrix $f_{ii'}$ which rescales $\tilde{\Lambda}_{i'}$ to create an event rate for an individual moving directly between the $i$-th and $i'$-th states.

Now consider $\tilde{\mu}_i=\tilde{\mu}_i( \dots, n_{i}, \dots )$, i.e., a function of all previous state occupations by the individual, where $n_i$ are the state-specific counts of past occupations. The rate $\tilde{\mu}_i$ hence maintains a 'record' of past state occupations and updates accordingly through this memory. Additionally, we will make each rate $\tilde{\Lambda}_i=\tilde{\Lambda}_i(n_{i})$, i.e., a function _only_ of the state-specific count associated to each rate, respectively. The choice in the latter case comes from interpreting the counting memory as a model for capsular immunity - this will also turn out to be quite important for our approximation further on.

Note that in [Cobey & Lipsitch (2012)](https://pubmed.ncbi.nlm.nih.gov/22383809/), the models of nonspecific and specific immunity suggest choosing the following functions

$$
\begin{align}
\tilde{\mu}_i( \dots, n_{i}, \dots ) &= \mu_{\rm max} + (\mu_i - \mu_{\rm max})\exp \bigg( -\epsilon \sum_{\forall i'}n_{i'} \bigg) \\
\tilde{\Lambda}_i(n_i) &= \Lambda_{i}{\bf 1}_{n_i=0} + \sigma \Lambda_{i}{\bf 1}_{n_i>0} \,.
\end{align}
$$

In the expressions above: $\epsilon$ governs the level of (immune system maturation) with respect to the number of past state occupations; ${\bf 1}_A$ denotes an indicator function whose argument is unity when $A$ is satisfied, else $0$; and the susceptibility of an individual is assumed to be reduced by a constant factor of $\sigma$ after having occupied that state once or more.

The multi-state process that we're going to consider would be normally be described as a non-Markovian phenomenon. However, the modelling approach we will take is instead a bit more similar to the formal concept of a Markov embedding (as studied, e.g., recently in [Kanazawa & Sornette (2021)](https://arxiv.org/abs/2102.00242)). By creating a binary state occupation variable $x_i$ for the $i$-th serotype, and the probability of occupying state $(\dots , x_i , \dots , n_i , \dots )$ at time $t$ as $P(\dots , x_i , \dots , n_i , \dots , t)$, we may write a Markovian master equation for the process. Let's now define

$$
\begin{align}
p_i(\dots ,n_i,\dots ,t) &\equiv P(\dots, x_{i}=1, x_{i'}=0, \dots ,n_{i},\dots ,t)\quad  \forall i'\neq i \\
q(\dots ,n_i,\dots ,t) &\equiv P(\dots, x_{i}=0, \dots ,n_{i},\dots ,t) \quad \forall i\,.
\end{align}
$$

Using these definitions, it is straightforward to show that the master equation satisfies

$$
\begin{align}
\frac{{\rm d}}{{\rm d}t}p_i(\dots ,n_i,\dots ,t) &= \tilde{\Lambda}_i(n_i-1)q(\dots ,n_{i}-1,\dots ,t) + \sum_{\forall i' \neq i}f_{i'i}\tilde{\Lambda}_i (n_i-1)p_{i'}(\dots ,n_{i'}, n_i-1,\dots ,t) \\
&\quad - \tilde{\mu}_i( \dots, n_{i}, \dots ) p_i(\dots ,n_i,\dots ,t) - \sum_{\forall i'\neq i}f_{ii'}\tilde{\Lambda}_{i'} (n_{i'}) p_i(\dots ,n_i,\dots ,t) \\
\frac{{\rm d}}{{\rm d}t}q(\dots ,n_i,\dots ,t) &= \sum_{\forall i}\tilde{\mu}_i( \dots, n_{i}, \dots )p_i(\dots ,n_i,\dots ,t) - \sum_{\forall i}\tilde{\Lambda}_i(n_i) q(\dots ,n_i,\dots ,t) \,.
\end{align}
$$

By defining the state space to encode the memory of past state occupations using the $n_i$ values themselves, the process is Markovian over the full $(\dots , x_i,\dots ,n_i,\dots)$ space. Note also that one may obtain the time-dependent joint distribution over the $(\dots ,n_i,\dots)$ space, i.e., $P(\dots, n_i, \dots, t)$, through marginalisation at any time

$$
\begin{equation}
P(\dots, n_i, \dots, t) = q(\dots, n_i, \dots, t) + \sum_{\forall i} p_i(\dots, n_i, \dots, t) \,.
\end{equation}
$$

Though we intend our analysis of this class of multi-state models to apply more generally beyond immediate applications to pneumococcus, it also is worth noting that restricting individuals to occupy a single state at a time only approximates the full pneumococcal carriage dynamics. The true process actually allows for some individuals to carry more than one serotype at at time. However, due to the relatively low and variable reported prevalence of simultaneous serotype carriers (or 'co-colonised' individuals) across different studies (see, e.g., [Gratten et al. (1989)](https://pubmed.ncbi.nlm.nih.gov/2639508/), [Huebner et al. (2000)](https://journals.lww.com/pidj/fulltext/2000/10000/lack_of_utility_of_serotyping_multiple_colonies.19.aspx) and many others...), the single-state occupation model should still a good tracer model of the underlying dynamical behaviour of the system. Note also that this additional complexity in the dynamics should be straightforward to incorporate into our framework for future analyses.

Let's now try an approximation for the joint distributions of $p_i(\dots, n_i, \dots, t)$ and $q(\dots, n_i, \dots, t)$ which assumes separability, such that

$$
\begin{align}
\ p_i(\dots, n_i, \dots, t) &\simeq p_i(t)P(\dots, n_i, \dots, t) \\
\ q(\dots, n_i, \dots, t) &\simeq q(t)P(\dots, n_i, \dots, t) \,.
\end{align}
$$

We shall evaluate the quality of this approximation later on (so don't worry) under different parametric conditions, but for now, let's just treat it as an ansatz.

By marginalising over states in the master equation, then substituting in the approximations above, and finally marginalising (each a summation from $n_{i'}=0$ to $\infty$) over the resulting relation $\forall n_{i'} \,\, i'\neq i$, one finds that the following time evolution equation is separately satisfied by each marginal $P(n_i,t)$ distribution

$$
\begin{align}
\frac{{\rm d}}{{\rm d}t}P(n_i,t) &= \bigg[ \tilde{\Lambda}_i(n_i-1)q(t) + \sum_{\forall i'\neq i}  f_{i'i}\tilde{\Lambda}_{i} (n_{i}-1) p_{i'}(t) \bigg] P(n_{i}-1,t) \\
&\quad - \bigg[ \tilde{\Lambda}_i(n_i)q(t) + \sum_{\forall i'\neq i} f_{ii'}\tilde{\Lambda}_{i'} (n_{i'}) p_i(t)\bigg] P(n_i,t) \,.
\end{align}
$$

In addition to the separability assumption, the key point which allowed us to derive this one-step marginal master equation was the dependence of $\tilde{\Lambda}_i$ on _only_ $n_i$; in contrast to all of the past recorded states $(\dots, n_i, \dots)$ like $\tilde{\mu}_i$.

From this point on we'll focus on the specific pneumococcus model by inserting the rate function definitions from [Cobey & Lipsitch (2012)](https://pubmed.ncbi.nlm.nih.gov/22383809/) that we introduced at the start into the marginal master equation for $P(n_i,t)$. The `pneumoinfer` class is currently written for only these models (i.e., with just these choices of function), but it's useful to see how the steps above could be performed for more general models too. The solution to the marginal master equation with these function substitutions is simply a Poisson distribution $P(n_i,t) = {\rm Poisson}[n_i;{\rm E}_t(n_i)]$, where

$$
\begin{equation}
{\rm E}_t (n_i) = {\rm E}_{t_{\rm init}}(n_i) + \int^t_{t_{\rm init}}{\rm d}t'\bigg[ \sigma \Lambda_iq(t') +\sum_{\forall i'\neq i}  f_{i'i}\sigma \Lambda_{i} p_{i'}(t')\bigg] \,.
\end{equation}
$$

Exploiting the properties of this Poisson distribution, if we now return to the full master equation and marginalise them over all $n_i$, while noting that

$$
\begin{align}
\ p_i(t) &= \sum_{\forall n_i}\sum_{n_{i}=0}^\infty p_i(\dots, n_i, \dots, t) \\
\ q(t) &= \sum_{\forall n_i}\sum_{n_{i}=0}^\infty q(\dots, n_i, \dots, t) \,,
\end{align}
$$

one arrives at the following finite (implicitly integro-differential) system

$$
\begin{align}
\frac{{\rm d}}{{\rm d}t}p_i(t) &= \Lambda_iF_{it} q(t) + \sum_{\forall i'\neq i} f_{i'i} \Lambda_iF_{it} p_{i'}(t) - \mu_iG_{it} p_i(t)-\sum_{\forall i'\neq i}f_{ii'}\Lambda_{i'}F_{i't} p_i(t) \\
\frac{{\rm d}}{{\rm d}t}q(t) &= \sum_{\forall i}\mu_iG_{it}p_i(t) - \sum_{\forall i}\Lambda_iF_{it}q(t)\,,
\end{align}
$$

where, to avoid repetition, we have defined

$$
\begin{align}
\ F_{it} &= P(n_i=0,t)+\sigma P(n_i>0,t) = e^{-{\rm E}_t(n_i)}+\sigma \big[ 1-e^{-{\rm E}_t(n_i)}\big] \\
\ G_{it} &= \frac{\mu_{\rm max}}{\mu_i} + \bigg( 1-\frac{\mu_{\rm max}}{\mu_i}\bigg) e^{\sum_{\forall i}{\rm E}_t(n_i)(e^{-\epsilon}-1)}\,,
\end{align}
$$

where to derive $G_{it}$ we have had to assume conditional independence between $n_i$ and $n_{i'}\,\,\forall i'\neq i$. The equation for ${\rm E}_t (n_i)$ can be differentiated to provide an equation for the time derivative of ${\rm E}_t(n_i)$ - evolving this equation alongside the system defined above yields an explicit finite ODE system. Note also that this approximation technique should apply to other forms of memory functions used for $\tilde{\mu}_i(\dots, n_i, \dots)$ and $\tilde{\Lambda}_i(n_i)$ by simply marginalising over their $n_i$ values, and so this approximate approach appears to be quite generalisable to other simlar systems.

In order to analyse the system properties and check the validity of the approach above, we're now going to make some decisions about the parameter space to explore. Let's independently draw the $(\mu_i,\Lambda_i)$ values from Gamma distributions with shapes $(\mu_\alpha,\Lambda_\alpha)$ and rates $(\mu_\beta,\Lambda_\beta)$. Let's also constrain the matrix values $f_{ii'}=f_{i}{\bf I}_{i'}$ (where ${\bf I}_{i'}$ denotes the elements of a simple vector of ones) which also happens to be consistent with pneumococcus data anyway (see, e.g., [Lipsitch et al. (2012)](https://pubmed.ncbi.nlm.nih.gov/22441543/)). We'll also need a metric of comparison between the marginalised distribution outputs from the fully simulated master equation and our approximation. To this end, it probably makes sense to look at the [Kullback-Leibler divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) between the marginal distributions for $x_i$ and $n_i$ in a full stochastic simulation and our approximation. In other words

$$
\begin{align}
\ D^{(x)}_{{}_{\rm KL}} &= \sum_{\forall i} p_{i, {\rm sim}}(t) \ln \Bigg[ \frac{p_{i, {\rm sim}}(t)}{p_i(t)} \Bigg]  \\
&\simeq - \sum_{\forall i} \frac{\ln Z_{\rm sim}(x_i, t)}{Z_{\rm sim}(x_i, t)} -\sum_{\forall i} \frac{\ln p_i(t)}{Z_{\rm sim}(x_i, t)} \\
\ D^{(n_i)}_{{}_{\rm KL}} &= \sum_{n_i=0}^{\infty} P_{\rm sim}(n_i, t) \ln \Bigg[ \frac{P_{\rm sim}(n_i, t)}{P(n_i,t)} \Bigg] \\
&\simeq - \sum_{n_i=0}^{\infty}\frac{\ln Z_{\rm sim}(n_i, t)}{Z_{\rm sim}(n_i, t)} - \sum_{n_i=0}^{\infty} \frac{\ln {\rm Poisson}[n_i;{\rm E}_t(n_i)]}{Z_{\rm sim}(n_i, t)} \\
&\simeq - \sum_{n_i=0}^{\infty}\frac{\ln Z_{\rm sim}(n_i, t)}{Z_{\rm sim}(n_i, t)} - \sum_{n_i=0}^{\infty} \bigg[ \frac{n_i\ln {\rm E}_t(n_i) - {\rm E}_t(n_i) - \ln \Gamma (n_i+1)}{Z_{\rm sim}(n_i, t)} \bigg] \,,
\end{align}
$$

where $Z_{\rm sim}(x_i, t)$ and $Z_{\rm sim}(n_i, t)$ denote the marginal frequency counts in each state space derived from a stochastic simulation of the master equation. Note that for the whole $(\dots, n_i, \dots)$ space, a better comparison would involve Monte Carlo integration of the joint counts $Z_{\rm sim}(\dots, n_i, \dots, t)$. However, this is quite a lot more challenging with many dimensions (usually necessitating nested sampling) and so we'll consider it to be beyond the present scope.

Using the `run_sim` method of the `pneumoinfer` class, and the equations above, we can generate numerically-approximate plots of the Kullback-Leibler divergence on the marginal distributions over a range of numbers of states, parameters and points in time.

```python
# Choose the number of states
nstat = 10

# Initialise pneumoinfer
pn = pneumoinfer(nstat)

# Create population members with gamma-distributed
# rate parameters
numpeople = 1
kmu = 1.0
param_dic = {
    'Curr'  : 0,
    'npast' : np.zeros(nstat),
    'Lam'   : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'mu'    : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'f'     : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'eps'   : 1.0/5.0,
    'sig'   : 1.0,
    'mumax' : 1.0
}
pn.create_members(numpeople,param_dic)

# Run the ODE and full simulation (the latter for a
# given number of realisations
nreals = 10000
ode_tstepsize = 0.001
sim_trejectionstepsize = 0.01
total_time = 10.0
times = [0.2*float(i) + 0.1 for i in range(0,40)]
pn.run_ode(total_time,ode_tstepsize)
pn.run_sim(nreals,total_time,sim_trejectionstepsize,time_snaps=times)

# Setup plots for the DKL values using the outputs
DKLx, DKLn = [], []
for t in times:
    counts = pd.Series(pn.sim_output['Curr'][t].flatten()).value_counts()
    co = np.zeros(nstat + 1)
    co[counts.index.values.astype(int)] = counts.values
    co = co[1:]
    it = np.argmin((pn.ode_output['time']-t)**2)
    DKLx.append(
        - np.sum(np.log(co[co>0])/co[co>0])
        - np.sum(np.log(pn.ode_output['probCurr'][it][co>0])/co[co>0])
    )
    DKLnsum = 0.0
    for i in range(0,nstat):
        ncounts = pd.Series(pn.sim_output['npastsum'][t][i].flatten()).value_counts()
        nco = np.zeros(1000)
        ns = np.arange(0, 1000, 1)
        nco[ncounts.index.values.astype(int)] = ncounts.values
        DKLnsum += (
            - np.sum(np.log(nco[nco>0])/nco[nco>0])
            - np.sum(ns[nco>0] * np.log(pn.ode_output['Expnpast'][it][i])/nco[nco>0])
            + np.sum(pn.ode_output['Expnpast'][it][i]/nco[nco>0])
            + np.sum(spec.loggamma(ns[nco>0]+1)/nco[nco>0])
        )
    DKLn.append(DKLnsum)
```

```python
fig, ax = plt.subplots(1, 2, figsize=(15,5))
ax[0].plot(times,DKLx)
ax[1].plot(times,np.asarray(DKLn)/np.asarray(times))
ax[0].set_xlabel('Time')
ax[0].set_ylabel(r'$D^{(x)}_{\rm KL}$')
ax[1].set_xlabel('Time')
ax[1].set_ylabel(r'$\sum_{\forall i}D^{(n_i)}_{\rm KL}\,/\,$Time')
plt.show()
```

The value of $D_{{}_{\rm KL}}^{(x)}$ generally stays small (and stable) throughout for most parameter choices. Interestingly, the same cannot be said for the $D_{{}_{\rm KL}}^{(n_i)}$ values, which appear to tend towards a deviation which is linearly proportional in time. If we now plot the time evolution of each set of quantities explicitly in time, we can see this is consistent with the observed deviations between the simulation and the ODE approximation.

```python
fig, ax = plt.subplots(1,2,figsize=(15,5))
colours = sns.color_palette()
probs, ncounts = [], []
for t in times:
    counts = pd.Series(pn.sim_output['Curr'][t].flatten()).value_counts()
    nco = np.sum(pn.sim_output['npastsum'][t],axis=1)
    ncounts.append(nco/nreals)
    pr = np.zeros(nstat + 1)
    pr[counts.index.values.astype(int)] = counts.values/float(nreals)
    probs.append(list(pr))
for i in range(0,nstat):
    ax[0].plot(times,np.asarray(probs)[:,i+1],label='State '+str(i+1),color=colours[i])
    ax[0].plot(pn.ode_output['time'],pn.ode_output['probCurr'][:,i],color=colours[i])
    ax[1].plot(times,np.asarray(ncounts)[:,i],label='State ' + str(i+1),color=colours[i])
    ax[1].plot(pn.ode_output['time'],pn.ode_output['Expnpast'][:,i],'--',color=colours[i])
ax[0].set_xlabel('Time')
ax[0].set_ylabel(r'$p_i$')
ax[1].set_xlabel('Time')
ax[1].set_ylabel(r'$n_i$')
ax[1].legend(bbox_to_anchor=(1.3,1.0))
plt.show()
```

---

## A varying $\Lambda_{iu}$ model

We're now ready to introduce an alternative model which accounts for a stochastically-varying susceptibility $\Lambda_{iu}$ (a possible model for community exposure to infectious individuals), which is now additionally indexed by '$u$' which corresponds to each individual. In this model, we have

$$
\begin{equation}
\Lambda_{iu} = \Lambda_{\rm min} + \lambda\sum_{\forall u'\neq u}\beta_{uu'} \frac{x_{iu'}}{N_{\rm p}}\,,
\end{equation}
$$

where: the total population number is $N_{\rm p}$; $\beta_{uu'}$ are elements of a 'contact matrix' which rescales the event rate according to the spreading behaviour between the $u$-th and $u'$-th individuals; $\lambda$ is a constant normalisation for $\beta_{uu'}$; and $x_{iu'}$ records the state of the $u'$-th individual.

Extending the master equation we introduced in the previous section to include the susceptibility above and the states of $N_{\rm p}$ individuals, one can easily adapt the argument of the previous section to arrive at the following generalisation of the ODE system we found earlier

$$
\begin{align}
\frac{{\rm d}}{{\rm d}t}p_{iu}(t) &= {\rm E}_t(\Lambda_{iu})F_{it} q_u(t) + \sum_{\forall i'\neq i} f_{i'i} {\rm E}_t(\Lambda_{iu})F_{it} p_{i'u}(t) - \mu_iG_{it} p_{iu}(t)-\sum_{\forall i'\neq i}f_{ii'}{\rm E}_t(\Lambda_{i'u})F_{i't} p_{iu}(t) \\
\frac{{\rm d}}{{\rm d}t}q_u(t) &= \sum_{\forall i}\mu_iG_{it}p_{iu}(t) - \sum_{\forall i}{\rm E}_t(\Lambda_{iu})F_{it}q_u(t)\,.
\end{align}
$$

In the equations above, the state occupation probabilities of separate $u$-indexed individuals (or $u$-indexed categories of individual) are $p_{iu}(t)$ and $q_u(t)$, and we've also computed the expectation

$$
\begin{equation}
{\rm E}_t(\Lambda_{iu}) = \Lambda_{\rm min} + \lambda\sum_{\forall u'\neq u}\beta_{uu'} \frac{p_{iu'}(t)}{N_{\rm p}}\,.
\end{equation}
$$

The `pneumoinfer` class also implements ODE and full simulations for the varying $\Lambda_{iu}$ model, and we plot an example run of this method below.

```python
# Once again, setup things as before
nstat = 10
pn = pneumoinfer(nstat)

# Now we add a contact matrix as well as its referenced
# index to each population member
pn._cont_mat = np.ones((3,3))
kmu = 1.0
param_dic = {
    'npast' : np.zeros(nstat),
    'Lam'   : np.zeros(nstat),
    'mu'    : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'f'     : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'eps'   : 1.0/5.0,
    'sig'   : 1.0,
    'mumax' : 1.0,
}
nind = 1000
for i in range(0,nind):
    group_param_dic = param_dic.copy()
    group_param_dic['Curr'] = np.random.randint(0,nstat+1)
    group_param_dic['cind'] = np.random.randint(0,3)
    pn.create_members(1,group_param_dic)

# Running the ODE and full sim in much the same way
# but only 1 realisation will be used for speed
nreals = 1
ode_tstepsize = 0.001
sim_trejectionstepsize = 0.01
total_time = 20.0
times = [0.2*float(i) + 0.1 for i in range(0,80)]
pn.run_ode(total_time,ode_tstepsize)
pn.run_sim(nreals,total_time,sim_trejectionstepsize,time_snaps=times)
```

```python
colours = sns.color_palette()
probs = []
for t in times:
    counts = pd.Series(pn.sim_output['Curr'][t].flatten()).value_counts()
    pr = np.zeros(nstat + 1)
    pr[counts.index.values.astype(int)] = counts.values/float(nreals)
    probs.append(list(pr))
for i in range(0,nstat):
    plt.plot(times,np.asarray(probs)[:,i+1]/float(nind),label='State '+str(i+1),color=colours[i])
    plt.plot(pn.ode_output['time'],pn.ode_output['probCurr'][:,i],color=colours[i])
plt.legend(bbox_to_anchor=(1.3,1.0))
ax = plt.gca()
ax.set_xlabel('Time')
ax.set_ylabel(r'$p_i$')
plt.show()
```

---

## Computing the log-likelihood for inference

We're now ready to apply our ODE approximation to the statistical inference of the full simulation. We're going to assume that all data sets considered come in the form of a sequence of state observations (or longitudinally-monitored swab samples if you're talking about pneumococcus) for each sampled individual from the population which takes the form of counts, times and associated sample sizes ${\cal D}\equiv \{d_c,d_t,d_s\}$ (where $d_c=\{c_{ij}\}$, $d_t=\{t_j \, : \, t_{\rm init}<t_j\leq t_{\rm end}\}$ and $d_s=\{s_j\}$). Note that this sequence of state observations may also be regarded as a form of _filtration_ of the underlying stochastic process. Given a likelihood ${\cal L}$ of ${\cal D}$, these time intervals are independent of each other and so one may decompose ${\cal L}$ for the whole population as follows

$$
\begin{equation}
\ {\cal L}({\cal D} \vert \Theta ) = \prod_{\forall j \, : \, t_j\,\in \,d_t}{\cal L}( t_j \vert \Theta ) \,.
\end{equation}
$$

Assuming here the fixed $\Lambda_i$ model, the ODE solution then allows us to easily compute the multinomial likelihoods for the initial condition and into the future

$$
\begin{equation}
{\cal L}( t_j \vert \Theta ) = \frac{\Gamma (s_j+1)}{\prod_{\forall i}\Gamma (c_{ij}+1)}q(t_j)^{s_j-\sum_{\forall i}c_{ij}} \prod_{\forall i}p_i(t_j)^{c_{ij}} \,.
\end{equation}
$$

Note that the independence of the expression above with respect to previous timepoints is not trivial and relies on the Markovian embedding of the system that we have already discussed. Note also that the above relations can be easily adapted to accomodate the varying $\Lambda_{iu}$ model (and these are also implemented in the `pneumoinfer` log-likelihood code).

Let's now run a full simulation using the fixed $\Lambda_i$ model and use its output at different timepoints to construct a mock data set of state observations.

```python
# Rerun a standard sim for the fixed Lamda model
# from which we will generate a mock data set
nstat = 10
pn = pneumoinfer(nstat)
numpeople = 1
kmu = 1.0
param_dic = {
    'Curr'  : 0,
    'npast' : np.zeros(nstat),
    'Lam'   : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'mu'    : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'f'     : np.random.gamma(kmu,1.0/kmu,size=nstat),
    'eps'   : 1.0/5.0,
    'sig'   : 1.0,
    'mumax' : 1.0
}
pn.create_members(numpeople,param_dic)
nreals = 1000
ode_tstepsize = 0.001
sim_trejectionstepsize = 0.01
total_time = 10.0
times = [0.2*float(i) + 0.1 for i in range(0,40)]
pn.run_ode(total_time,ode_tstepsize)
pn.run_sim(nreals,total_time,sim_trejectionstepsize,time_snaps=times)
```

```python
# Create some mock state observation data
# out of the simulation runs above
data_times = np.random.choice(times,nstat,replace=False)
df = pd.Series(pn.sim_output['Curr'][data_times[0]].flatten()).value_counts().reset_index()
df['Time'] = data_times[0]
for t in data_times[1:]:
    new_df = pd.Series(pn.sim_output['Curr'][t].flatten()).value_counts().reset_index()
    new_df['Time'] = t
    df = pd.concat([df, new_df], ignore_index=True)
df = df.rename(columns={'index' : 'Curr', 0 : 'Count'})
df.Curr = df.Curr.astype(int)
df
```

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Curr</th>
      <th>Count</th>
      <th>Time</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>7</td>
      <td>240</td>
      <td>6.3</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>141</td>
      <td>6.3</td>
    </tr>
    <tr>
      <th>2</th>
      <td>0</td>
      <td>135</td>
      <td>6.3</td>
    </tr>
    <tr>
      <th>3</th>
      <td>5</td>
      <td>133</td>
      <td>6.3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>8</td>
      <td>128</td>
      <td>6.3</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>105</th>
      <td>4</td>
      <td>80</td>
      <td>0.9</td>
    </tr>
    <tr>
      <th>106</th>
      <td>1</td>
      <td>29</td>
      <td>0.9</td>
    </tr>
    <tr>
      <th>107</th>
      <td>9</td>
      <td>10</td>
      <td>0.9</td>
    </tr>
    <tr>
      <th>108</th>
      <td>10</td>
      <td>6</td>
      <td>0.9</td>
    </tr>
    <tr>
      <th>109</th>
      <td>6</td>
      <td>5</td>
      <td>0.9</td>
    </tr>
  </tbody>
</table>
<p>110 rows × 3 columns</p>
</div>

Note the form of the `pandas` dataframe given above: the `'Curr'` column denotes the state label (or serotype index) and the `'Count'` and `'Time'` columns describe the population number and time of the observation, respectively. Using this mock data, we can now compute the log-likelihood using the method provided in `pneumoinfer` to show that the sampler correctly identifies the maximum likelihood (up to some observation noise which explains the infrequent samples with a sightly higher log-likelihood).

```python
# Usual setup again, but this time we
# add in the data via the keyword
nstat = 10
pn = pneumoinfer(nstat)
pn.create_members(1,param_dic,data=df)

# Generate some random samples away
# from the known log-likelihood maximum
maxLL = pn.lnlike(ode_tstepsize)
LLs = []
maxLams = pn.params['ode']['Lams']
maxmus = pn.params['ode']['mus']
maxfs = pn.params['ode']['fs']
for i in range(0,100):
    tempLams = maxLams + np.random.normal(0.0,0.01,size=nstat).reshape(nstat,1)
    tempmus = maxmus + np.random.normal(0.0,0.01,size=nstat).reshape(nstat,1)
    tempfs = maxfs + np.random.normal(0.0,0.01,size=nstat).reshape(nstat,1)
    pn.params['ode']['Lams'] = tempLams*(tempLams>0.0)
    pn.params['ode']['mus'] = tempmus*(tempmus>0.0)
    pn.params['ode']['fs'] = tempfs*(tempfs>0.0)
    LLs.append(pn.lnlike(ode_tstepsize))
```

```python
plt.plot(np.asarray(LLs)-maxLL)
ax = plt.gca()
ax.set_ylabel('LL-max(LL)')
plt.show()
```

---

## Additional notes: a method to compute the gradient of the log-likelihood

The current version of `pneumoinfer` does not support a gradient calculation for the log-likelihood (mainly because I was eager to move onto some other stuff!). However, to assist anyone wanting to implement this themselves, I thought it would be helpful to go through the calculation which computes the gradient (in principle) without resorting to numerical derivatives. This makes use of the 'multiple-adjoint' method as implemented in [Zhuang et al. (2021)](https://arxiv.org/abs/2006.02493). Consider the following 'data Lagrangian'

$$
\begin{align}
L &=  \sum_{\forall j \, : \, t_j\,\in \,d_t} L_j\\
L_j &= \ln{\cal L}( t_j \vert \Theta ) + \int^{t_{j}}_{t_{j-1}}{\rm d}t \,{\sf h}(t)^{\rm T}\bigg[ \frac{{\rm d}}{{\rm d}t}{\sf V}(t) - {\sf M}_\Theta (t)\bigg] \\
&= \ln{\cal L}( t_j \vert \Theta) + {\sf h}(t_j)^{\rm T}{\sf V}(t_j)-{\sf h}(t_{j-1})^{\rm T}{\sf V}(t_{j-1}) - \int^{t_{j}}_{t_{j-1}}{\rm d}t \bigg[\frac{{\rm d}}{{\rm d}t}{\sf h}(t)^{\rm T} {\sf V}(t) + {\sf h}(t)^{\rm T}{\sf M}_\Theta (t)\bigg] \,,
\end{align}
$$

where ${\sf V}(t)=[\dots, {\rm E}_t(n_i),\dots, p_i(t), \dots , q(t)]^{\rm T}$, ${\sf h}(t)$ is a dynamical vector of Lagrange multipliers and ${\sf M}_\Theta (t)$ is just compact notation for the vector of ODE terms on the RHS. Varying $L_j$ with respect to the boundary condition ${\sf V}(t_j)$ and ${\sf V}(t)$, we obtain the constraints

$$
\begin{align}
\frac{\partial L_j}{\partial {\sf V}(t_j)} &= 0 \quad \Longleftrightarrow \quad \frac{\partial}{\partial {\sf V}(t_j)}\ln{\cal L}( t_j \vert \Theta ) + {\sf h}(t_j ) = 0 \\
\frac{\delta L_j}{\delta {\sf V}(t)} &= 0 \quad \Longleftrightarrow \quad \frac{{\rm d}}{{\rm d}t}{\sf h}(t) + \bigg[\frac{\partial}{\partial {\sf V}(t)}{\sf M}_\Theta (t)\bigg]^{\rm T}{\sf h}(t) = 0\,,
\end{align}
$$

Let us also note that if we vary $L_j$ with respect to $\Theta$ and optimise the likelihood, one obtains

$$
\begin{align}
\frac{\partial L_j}{\partial \Theta} &= \frac{\partial}{\partial \Theta}\ln{\cal L}( t_j \vert \Theta ) - \int^{t_{j}}_{t_{j-1}}{\rm d}t \,{\sf h}(t)^{\rm T}\frac{\partial}{\partial \Theta}{\sf M}_\Theta (t) \\
&\underset{{\rm opt}}{\longrightarrow} \int_{t_{j}}^{t_{j-1}}{\rm d}t \,{\sf h}(t)^{\rm T}\frac{\partial}{\partial \Theta}{\sf M}_\Theta (t)\,.
\end{align}
$$

The method proposed from here would be something like:
- Treat initial values of ${\rm E}_{t_0}(n_i)$ as a prior that should be varied to test the robustness of the inference.
- From the initialised states of the set of individuals run the forward ODE in time to obtain the value of $\frac{\partial}{\partial \Theta}{\sf M}_\Theta (t)$ at every observed timestep.
- For each interval edge determine ${\sf h}(t_j)$ using the first constraint equation and the ${\sf V}(t_j)$-gradient of the forward-time likelihood.
- For each interval solve the second equation to get its backward-time trajectory ${\sf h}(t)$.
- Integrate over ${\sf h}(t)$ and $\frac{\partial}{\partial \Theta}{\sf M}_\Theta (t)$ to determine the gradient in the last expression.

Seems like overkill, but could be interesting to implement in future if a large number of states/parameters are varied, e.g., for HMC sampling of the model posterior from a decent data set.

```python

```