# Linear prediction (very naive implementation)

Linear prediction has a long history  (see Makhoul, J. Linear prediction: A tutorial review. Proc. IEEE 63, 561–580 (1975)).

This tutorial only covers the very basics, in a very sloppy manner:
1. auto-correlation
2. Levison-Durvison recursion
3. reflection coefficients

Functions are implemented based on Numpy.

```python
import os
import sys
import numpy as np
import scipy.io.wavfile

import tool_lib
import plot_lib
import IPython.display
```

```python
import torch
import torch.nn as torch_nn
import torch.nn.functional as torch_nn_func
import sys

# wrapper functions for plotting
import matplotlib
import matplotlib.pyplot as plt
matplotlib.rcParams['figure.figsize'] = (10, 6)
matplotlib.rcParams.update({'font.size': 12})

def plot_data(x_data, y_data, fig, axis, xlabel, ylabel, title):
    axis.plot(x_data, y_data, 'k')
    axis.grid(True)
    axis.set_xlabel(xlabel)
    axis.set_ylabel(ylabel)
    axis.set_title(title)
    return fig, axis
```

A severly simplified flowchart to conduct LPC analysis and synthesis:
```c

Input waveform
   |
   | (pre-emphasis)
   | framing and windowing
   V
Framed waveforms
for each frame:
        |
        | auto-correlation
        V
  Auto-corr coefficients
        |
        | LP analysis (Levison-Durbin recursion)
        V
   LPC coefficients
   Reflection coefficients
   Residual / excitation waveform
        |
        | LP synthesis
        v
Framed signals
     |
     | overlap-add
     v
Reconstructed waveform
```

## LP API definition

Here is the definition of the LPC API.

It is also available in `core_scripts/data_io/dsp_tools.py`. (This API is used in before data is converted into torch.tensors, see [this note](#finalnote) at the bottom of this notebook)

We here assumes the LPC model (z-domain) to be

$\begin{align}
H(z) &= \frac{1}{1 - bz^{-1}} \frac{G}{a_0 + \sum_{i=1}^{M}a_i z^{-i}} \\
     &= \frac{1}{1 - bz^{-1}} \frac{G}{a_0 + a_1 z^{-1} + \cdots + a_{M} z^{M}}
\end{align}$

where $a_0=1$,  $G$ is the gain,  and $b$ is the optional coefficient for pre-emphasis filter.

In the code, we call $M$ as the order of the model. An $M$-order LPC model has $M$ poles.

We will explain each method in this API one by one.

```python
class LPClite(object):
    """ A lite LPC analyzer & synthesizr
    Note that this is based on numpy, not Pytorch
    It can be used for pre-processing when loading data, or use
    it as data transformation function
    (see message at top)

    Example:
        # load waveform
        sr, wav = wav_tools.waveReadAsFloat(wav_path)
        m_lpc = LPClite(320, 80)
        # LPC analysis
        lpc_coef, _, rc, gain, err, err_overlapped = m_lpc.analysis(wav)
        # LPC synthesis
        wav_re = m_lpc.synthesis(lpc_coef, err, gain)
        # rc to LPC
        lpc_coef_tmp = m_lpc._rc2lpc(lpc_coef)
        np.std(lpc_coef_tmp - lpc_coef)
    """
    def __init__(self, fl=320, fs=80, order=29, window='blackman',
                 flag_emph=True, emph_coef=0.97):
        """LPClite(fl=320, fs=80, order=30, window='blackman')
        Args
        ----
          fl: int, frame length
          fs: int, frame shift
          order: int, order of LPC, [1, a_1, a_2, ..., a_order]
          window: str, 'blackman' or 'hanning'
          flag_emph: bool, whether use pre-emphasis (default True)
          emph_coef: float, coefficit for pre-emphasis filter (default 0.97)

        Note that LPC model is defined as:
           1                          Gain
        -------- ---------------------------------------------
        1- bz^-1 a_0 + a_1 z^-1 + ... + a_order z^-(order)

        b = emph_coef if flag_emph is True
        b = 0 otherwise
        """
        self.fl = fl
        self.fs = fs
        #
        self.order = order
        self.flag_emph = flag_emph
        self.emph_coef = emph_coef

        if np.abs(emph_coef) >= 1.0:
            print("Warning: emphasis coef {:f} set to 0.97".format(emph_coef))
            self.emph_coef = 0.97

        if window == 'hanning':
            self.win = np.hanning(self.fl)
        else:
            self.win = np.blackman(self.fl)

        return

    def analysis(self, wav):
        """lpc_coef, ld_err, gamma, gain, framed_err, err_signal = analysis(wav)

        LPC analysis on each frame

        input
        -----
          wav: np.array, (length, 1)

        output
        ------
          lpc_coef:   np.array, LPC coeff, (frame_num, lpc_order + 1)
          ld_err:     np.array, LD analysis error, (frame_num, lpc_order + 1)
          gamma:      np.array, reflection coefficients, (frame_num,lpc_order)
          gain:       np.array, gain, (frame_num, 1)
          framed_err: np.array, LPC error per frame, (frame_num, frame_length)
          eer_signal: np.array, overlap-added excitation (length, 1)

        Note that framed_err is the excitation signal from LPC analysis on each
        frame. eer_signal is the overlap-added excitation signal.
        """
        if self.flag_emph:
            wav_tmp = self._preemphasis(wav)
        else:
            wav_tmp = wav

        # framing & windowing
        frame_wined = self._windowing(self._framing(wav_tmp[:, 0]))

        # auto-correlation
        auto = self._auto_correlation(frame_wined)

        # LD analysis
        lpc_coef, lpc_err, gamma_array, gain = self._levison_durbin(auto)

        # get LPC excitation signals in each frame
        framed_err = self._lpc_analysis_core(lpc_coef, frame_wined, gain)

        # overlap-add for excitation signal
        err_signal = self._overlapadd(framed_err)

        return lpc_coef, lpc_err, gamma_array, gain, framed_err, err_signal

    def synthesis(self, lpc_coef, framed_err, gain):
        """wav = synthesis(lpc_coef, framed_err, gain):

        LPC synthesis (and overlap-add)

        input
        -----
          lpc_coef:   np.array, LPC coeff, (frame_num, lpc_order + 1)
          framed_err: np.array, LPC excitations, (frame_num, frame_length)
          gain:       np.array, LPC gain, (frame_num, 1)

        output
        ------
          wav: np.array, (length, 1)

        This function does LPC synthesis in each frame and create
        the output waveform by overlap-adding
        """
        framed_x = self._lpc_synthesis_core(lpc_coef, framed_err, gain)
        wav_tmp = self._overlapadd(framed_x)
        if self.flag_emph:
            wav_tmp = self._deemphasis(wav_tmp)
        return wav_tmp

    def _preemphasis(self, wav):
        """ wav_out = _preemphasis(wav)

        input
        -----
          wav: np.array, (length)

        output
        ------
          wav: np.array, (length)
        """
        wav_out = np.zeros_like(wav) + wav
        wav_out[1:] = wav_out[1:] - wav_out[0:-1] * self.emph_coef
        return wav_out

    def _deemphasis(self, wav):
        """ wav_out = _deemphasis(wav)

        input
        -----
          wav: np.array, (length)

        output
        ------
          wav: np.array, (length)
        """
        wav_out = np.zeros_like(wav) + wav
        for idx in range(1, wav.shape[0]):
            wav_out[idx] = wav_out[idx] + wav_out[idx-1] * self.emph_coef
        return wav_out

    def _framing(self, wav):
        """F = _framed(wav)

        Framing the signal

        input
        -----
          wav: np.array, (length)

        output
        ------
          F: np.array, (frame_num, frame_length)
        """
        frame_num = (wav.shape[0] - self.fl) // self.fs + 1
        F = np.zeros([frame_num, self.fl], dtype=wav.dtype)
        for frame_idx in np.arange(frame_num):
            F[frame_idx, :] = wav[frame_idx*self.fs : frame_idx*self.fs+self.fl]
        return F

    def _windowing(self, framed_x):
        """windowing
        """
        return framed_x * self.win

    def _overlapadd(self, framed_x):
        """wav = _overlapadd(framed_x)

        Do overlap-add on framed (and windowed) signal

        input
        -----
          framed_x: np.array, (frame_num, frame_length)

        output
        ------
          wav: np.array, (length, 1)

        length = (frame_num - 1) * frame_shift + frame_length
        """
        # waveform length
        wavlen = (framed_x.shape[0] - 1) * self.fs + self.fl
        wavbuf = np.zeros([wavlen])

        # buf to save overlapped windows (to normalize the signal amplitude)
        protobuf = np.zeros([wavlen])
        win_prototype = self._windowing(self._framing(np.ones_like(protobuf)))

        # overlap and add
        for idx in range(framed_x.shape[0]):
            frame_s = idx * self.fs
            wavbuf[frame_s : frame_s + self.fl] += framed_x[idx]
            protobuf[frame_s : frame_s + self.fl] += win_prototype[idx]

        # remove the impact of overlapped windows
        #protobuf[protobuf<1e-05] = 1.0
        wavbuf = wavbuf / protobuf.mean()
        return np.expand_dims(wavbuf, axis=1)

    def _lpc_analysis_core(self, lpc_coef, framed_x, gain):
        """framed_err = _lpc_analysis_core(lpc_coef, framed_x, gain)

        LPC analysis on frame
        MA filtering: e[n] = \sum_k=0 a_k x[n-k] / gain

        input
        -----
          lpc_coef: np.array, (frame_num, order + 1)
          framed_x: np.array, (frame_num, frame_length)
          gain: np.array,     (frame_num, 1)

        output
        ------
          framed_err: np.array, (frame_num, frame_length)

        Note that lpc_coef[n, :] = (1, a_1, a_2, ..., a_order) for n-th frame
        framed_x[n, :] = (x[0], x[1], ..., x[frame_len]) for n-th frame
        """
        #
        frame_num = framed_x.shape[0]
        frame_len = framed_x.shape[1]

        # lpc order (without the a_0 term)
        order = lpc_coef.shape[1] - 1

        # pad zero, every frame has [0, ..., 0, x[0], x[1], ..., x[frame_len]]
        tmp_framed = np.concatenate(
            [np.zeros([frame_num, order + 1]), framed_x], axis=1)

        # flip to (x[frame_len], ... x[1], x[0], 0, ..., 0)
        tmp_framed = tmp_framed[:, ::-1]

        # LPC excitation buffer
        framed_err = np.zeros_like(framed_x)

        # e[n] = \sum_k=0 a[k] x[n-k]
        # do this for all frames and n simultaneously
        for k in range(self.order + 1):
            # a[k]
            tmp_coef = lpc_coef[:, k:k+1]

            # For each frame
            # RHS = [x[n-k], x[n-k-1], ..., ] * a[k]
            #
            # By doing this for k in [0, order]
            # LHS = [e[n],   e[n-1],   ...]
            #       [x[n-0], x[n-0-1], ..., ] * a[0]
            #     + [x[n-1], x[n-1-1], ..., ] * a[1]
            #     + [x[n-2], x[n-2-1], ..., ] * a[2]
            #     + ...
            # We get the excitation for one frame
            # This process is conducted for all frames at the same time
            framed_err += tmp_framed[:, 0:frame_len] * tmp_coef

            # roll to [x[n-k-1], x[n-k-2], ..., ]
            tmp_framed = np.roll(tmp_framed, -1, axis=1)

        # revese to (e[0], e[1], ..., e[frame_len])
        return framed_err[:, ::-1] / gain

    def _lpc_synthesis_core(self, lpc_coef, framed_err, gain):
        """framed_x = _lpc_synthesis_core(lpc_coef, framed_err, gain)

        AR filtering: x[n] = gain * e[n] - \sum_k=0 a_k x[n-k]
        LPC synthesis on frame

        input
        -----
          lpc_coef:   np.array, (frame_num, order + 1)
          framed_err: np.array, (frame_num, frame_length)
          gain:       np.array, (frame_num, 1)

        output
        ------
          framed_x:   np.array, (frame_num, frame_length)

        Note that
        lpc_coef[n, :] = (1, a_1, a_2, ..., a_order),  for n-th frame
        framed_x[n, :] = (x[0], x[1], ..., x[frame_len]), for n-th frame
        """
        frame_num = framed_err.shape[0]
        frame_len = framed_err.shape[1]
        order = lpc_coef.shape[1] - 1

        # pad zero
        # the buffer looks like
        #  [[0, 0, 0, 0, 0, ... x[0], x[1], x[frame_length -1]],  -> 1st frame
        #   [0, 0, 0, 0, 0, ... x[0], x[1], x[frame_length -1]],  -> 2nd frame
        #   ...]
        framed_x = np.concatenate(
            [np.zeros([frame_num, order]), np.zeros_like(framed_err)], axis=1)

        # flip the cofficients of each frame as [a_order, ..., a_1, 1]
        lpc_coef_tmp = lpc_coef[:, ::-1]

        # synthesis (all frames are down at the same time)
        for idx in range(frame_len):
            # idx+order so that it points to the shifted time idx
            #                                    idx+order
            # [0, 0, 0, 0, 0, ... x[0], x[1], ... x[idx], ]
            # gain * e[n]
            framed_x[:, idx+order] = framed_err[:, idx] * gain[:, 0]

            # [x[idx-1-order], ..., x[idx-1]] * [a_order, a_1]
            pred = np.sum(framed_x[:, idx:idx+order] * lpc_coef_tmp[:, :-1],
                          axis=1)

            # gain * e[n] - [x[idx-1-order], ..., x[idx-1]] * [a_order, a_1]
            framed_x[:, idx+order] = framed_x[:, idx+order] - pred

        # [0, 0, 0, 0, 0, ... x[0], x[1], ... ] -> [x[0], x[1], ...]
        return framed_x[:, order:]

    def _auto_correlation(self, framed_x):
        """ autocorr = _auto_correlation(framed_x)

        input
        -----
          framed_x: np.array, (frame_num, frame_length), frame windowed signal

        output
        ------
          autocorr: np.array, auto-correlation coeff (frame_num, lpc_order+1)
        """
        # (frame_num, order)
        autocor = np.zeros([framed_x.shape[0], self.order+1])

        # loop and compute auto-corr (for all frames simultaneously)
        for i in np.arange(self.order+1):
            autocor[:, i] = np.sum(
                framed_x[:, 0:self.fl-i] * framed_x[:, i:],
                axis=1)
            #print(autocor[0, i])
            #autocor[:, i] = 0
            #for idx in np.arange(self.fl):
            #    if (idx + i) < self.fl:
            #        autocor[:, i] += framed_x[:, idx] * framed_x[:, idx + i]
            #    else:
            #        break
            #print(autocor[0, i])
        # (frame_num, order)
        return autocor

    def _levison_durbin(self, autocor):
        """lpc_coef_ou, lpc_err, gamma_array, gain = _levison_durbin(autocor)
        Levison durbin

        input
        -----
          autocor: np.array, auto-correlation, (frame_num, lpc_order+1)

        output
        ------
          lpc_coef: np.array, LPC coefficients, (frame_num, lpc_order+1)
          lpc_err: np.array, LPC error, (frame_num, lpc_order+1)
          gamma: np.array, reflection coeff, (frame_num, lpc_order)
          gain: np.array, gain, (frame_num, 1)

        Note that lpc_coef[n] = (1, a_2, ... a_order) for n-th frame
        """
        # (frame_num, order)
        frame_num, order = autocor.shape
        order = order - 1
        polyOrder = order + 1

        # to log down the invalid frames
        tmp_order = np.zeros([frame_num], dtype=np.int32) + polyOrder

        lpc_coef = np.zeros([frame_num, 2, polyOrder])
        lpc_err = np.zeros([frame_num, polyOrder])
        gamma_array = np.zeros([frame_num, order])
        gain = np.zeros([frame_num])

        lpc_err[:, 0] = autocor[:, 0]
        lpc_coef[:, 0, 0] = 1.0

        for index in np.arange(1, polyOrder):

            lpc_coef[:, 1, index] = 1.0

            # compute gamma
            #   step1.
            gamma = np.sum(lpc_coef[:, 0, 0:(index)] * autocor[:, 1:(index+1)],
                           axis=1)
            #   step2. check validity of lpc_err
            ill_idx = lpc_err[:,index-1] < 1e-07
            #      also frames that should have been stopped in previous iter
            ill_idx = np.bitwise_or(ill_idx, tmp_order < polyOrder)
            #   step3. make invalid frame gamma=0
            gamma[ill_idx] = 0
            gamma[~ill_idx] = gamma[~ill_idx] / lpc_err[~ill_idx,index-1]
            gamma_array[:, index-1] = gamma
            #   step4. log down the ill frames
            tmp_order[ill_idx] = index

            lpc_coef[:, 1, 0] = -1.0 * gamma
            if index > 1:
                lpc_coef[:, 1, 1:index] = lpc_coef[:, 0, 0:index-1] \
                + lpc_coef[:, 1, 0:1] * lpc_coef[:, 0, 0:index-1][:, ::-1]

            lpc_err[:, index] = lpc_err[:, index-1] * (1 - gamma * gamma)
            lpc_coef[:, 0, :] = lpc_coef[:, 1, :]

        # flip to (1, a_1, ..., a_order)
        lpc_coef = lpc_coef[:, 0, ::-1]

        # output LPC coefficients
        lpc_coef_ou = np.zeros([frame_num, polyOrder])

        # if high-order LPC analysis is not working
        # each frame may require a different truncation length
        for idx in range(frame_num):
            lpc_coef_ou[idx, 0:tmp_order[idx]] = lpc_coef[idx, 0:tmp_order[idx]]

        # get the gain, when tmp_order = polyOrder, tmp_order-2 -> order-1,
        #  last element of the lpc_err buffer
        gain = np.sqrt(lpc_err[np.arange(len(tmp_order)), tmp_order-2])

        # if the gain is zero, it means analysis error is zero,
        gain[gain < 1e-07] = 1.0

        # (frame_num, order)
        return lpc_coef_ou, lpc_err, gamma_array, np.expand_dims(gain, axis=1)

    def _rc2lpc(self, rc):
        """lpc_coef = _rc2lpc(rc)
        from reflection coefficients to LPC coefficients
        forward Levinson recursion

        input
        -----
          rc: np.array, (frame_num, lpc_order)

        output
        ------
          lpc_coef, np.array, (frame_num, lpc_order+1)

        Note that LPC model is defined as:
                            Gain
        ---------------------------------------------
        a_0 + a_1 z^-1 + ... + a_order z^-(order)

        Thus, the reflection coefficitns [gamma_1, ... gamma_order]
        """
        # (frame_num, order)
        frame_num, order = rc.shape
        polyOrder = order + 1

        lpc_coef = np.zeros([frame_num, 2, polyOrder])
        lpc_coef[:, 0, 0] = 1.0
        for index in np.arange(1, polyOrder):
            lpc_coef[:, 1, index] = 1.0
            gamma = rc[:, index-1]
            lpc_coef[:, 1, 0] = -1.0 * gamma
            if index > 1:
                lpc_coef[:, 1, 1:index] = lpc_coef[:, 0, 0:index-1] \
                + lpc_coef[:, 1, 0:1] * lpc_coef[:, 0, 0:index-1][:, ::-1]
            lpc_coef[:, 0, :] = lpc_coef[:, 1,:]

        lpc_coef = lpc_coef[:, 0, ::-1]
        return lpc_coef
```

## Framing and windowing

Framing and windowing is covered in c10_waveform_tools.  There, the framing and overlap function is defined on torch.tensors.

In the LPC API above, we re-implement it using Numpy. The input and output shape are different from those in c10_waveform_tools.

Anyway, the three methods `_framing()`, `_windowing()`, and `_overlapadd()` are similar to those in c10_waveform_tools.

I will just show example usage

```python
# claim an LPC object
frame_shift = 80
frame_length = 320
order = 30

# let's skip pre-emphasis at this stage
m_lpc_api = LPClite(frame_length, frame_shift, order, flag_emph=False)
```

```python
# load waveform
wavfile_path = './data_models/acoustic_features/hn_nsf/slt_arctic_b0474.wav'
sampling_rate, wave_data = tool_lib.waveReadAsFloat(wavfile_path)

# and do framing (and windowing)
wave_framed = m_lpc_api._windowing(m_lpc_api._framing(wave_data))

# Notice that wave_framed has shape (frame_num, frame_length)
frame_num, frame_length = wave_framed.shape
print("From waveform of length {:d}, we get {:d} frames".format(
    wave_data.shape[0], wave_framed.shape[0]))
```

    From waveform of length 44241, we get 550 frames

```python
# Let's plot a few frames (similar)
# This is similar to that in c10_waveform_tools.
#
# function to the time index of one frame
def frame_time_idx_range(frame_idx, frame_length=frame_length, frame_shift=frame_shift):
    # return the start and end time index of the frame
    return frame_idx * frame_shift, frame_idx * frame_shift + frame_length

frame_indices = [100, 101, 102, 103]
#  the time range to plot the waveforms
plot_xlim = [frame_time_idx_range(frame_indices[0])[0], frame_time_idx_range(frame_indices[-1])[-1]]

# whole waveform
length = wave_data.shape[0]
time_idx = np.arange(length)
fig=plt.figure(); axis=fig.add_subplot(511);
axis.plot(time_idx, wave_data, color='k')
axis.set_title('original waveform (black) and framed waveforms (blue)')
axis.set_xlim(plot_xlim)

# plot a few frames
for figure_idx, frame_idx in enumerate(frame_indices):
    axis=fig.add_subplot(5, 1, figure_idx + 2);
    frame_start, frame_end = frame_time_idx_range(frame_idx, frame_length, frame_shift)
    axis.plot(np.arange(frame_start, frame_end), wave_framed[frame_idx])
    axis.set_xlim(plot_xlim)
fig.subplots_adjust(hspace=0.5)
```

## Auto-correlation

In this API, we use auto-correlation method (https://en.wikipedia.org/wiki/Linear_prediction). An alternative is the covariance-based method (Sec. II. B. (Makhoul 1975)).

Given a framed signal $(x_1, \cdots, x_N)$, we want to compute the auto-correlation vector $(R_0, \cdots, R_M)$, where

$R_p = \sum_n x_n x_{n+p}$

The notation here is sloppy. It is recommended to check the definition in Eq.(12-15) of Makhoul, J. Linear prediction: A tutorial review. Proc. IEEE 63, 561–580 (1975).

First, let's try a naive implementation:

```python
# A very naive implementation for one signal is like this:
def auto_correlation(frame_signal, order):
    auto_seq = np.zeros([order])
    for i in np.arange(order):
        auto_seq[i] = 0
        for index in np.arange(len(frame_signal)):
            if (index + i) < len(frame_signal):
                auto_seq[i] += frame_signal[index] * frame_signal[index + i]
            else:
                break
    return auto_seq

# Let's choose one frame as example
frame_idx = 100
order = 30
autocorr = auto_correlation(wave_framed[100], order)

fig=plt.figure(figsize=(10, 3)); axis=fig.add_subplot(111)
axis.plot(autocorr);
axis.set_xlabel('Order p for sum x[n]x[n+p]')

#
# scipy also provides API for this, we can check whether we compute it correctly
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.correlate.html
#
import scipy.signal
autocorr_scipy = scipy.signal.correlate(wave_framed[100], wave_framed[100], mode='same', method='fft')
# notice that autocorr_scipy returns a symmetric vector, we only need half+1
axis.plot(autocorr_scipy[autocorr_scipy.shape[0]//2:autocorr_scipy.shape[0]//2+order], linewidth=5, alpha=0.2)
axis.legend(["from auto_correlation", "from scipy.correlate"])

# Good, the two lines are consistent
```

Since we have multiple frames, we may compute the auto-correlation vectors for all the frames at the same time. This is not a new algorith but just a trick to use Numpy.

Suppose we have two frames, one is [1,2,3,4,5] and the other is [6,7,8,9,10].

To compute $R_0$ for the two frames, we can do

$\begin{align}
\begin{bmatrix}
R_0^{(1)} \\
R_0^{(2)}
\end{bmatrix} =
\text{sum}(\begin{bmatrix}
1 & 2 & 3 & 4 & 5 \\
6 & 7 & 8 & 9 & 10
\end{bmatrix} \odot \begin{bmatrix}
1 & 2 & 3 & 4 & 5 \\
6 & 7 & 8 & 9 & 10
\end{bmatrix}, \text{axis}=1)
 \end{align}$

To compute $R_1$ for the two frames, we can do

$\begin{align}
\begin{bmatrix}
R_1^{(1)} \\
R_1^{(2)}
\end{bmatrix} =
\text{sum}(\begin{bmatrix}
1 & 2 & 3 & 4 \\
6 & 7 & 8 & 9
\end{bmatrix} \odot \begin{bmatrix}
2 & 3 & 4 & 5 \\
7 & 8 & 9 & 10
\end{bmatrix}, \text{axis}=1)
 \end{align}$

To compute $R_2$ for the two frames, we can do

$\begin{align}
\begin{bmatrix}
R_2^{(1)} \\
R_2^{(2)}
\end{bmatrix} =
\text{sum}(\begin{bmatrix}
1 & 2 & 3 \\
6 & 7 & 8
\end{bmatrix} \odot \begin{bmatrix}
3 & 4 & 5 \\
8 & 9 & 10
\end{bmatrix}, \text{axis}=1)
 \end{align}$

...

Then we will have $[R_0^(1), R_1^(1), \cdots]$ and $[R_0^(2), R_1^(2), \cdots]$ for the two frames, respectively.

This is implementation in `LPClite._auto_correlation()` method.

```python
# Let's use the LPClite API
autocorrs = m_lpc_api._auto_correlation(wave_framed)
# the output auto-correlation has shape (frame_num, lpc_order)

# show some numbers
frame_num, order = autocorrs.shape
print("We get {:d} autocorrelation vectors of order {:d}".format(
    frame_num, order))

# plot one frame (same as the figure above)
fig=plt.figure(figsize=(10, 3)); axis=fig.add_subplot(111)
axis.plot(autocorrs[100]);
axis.set_xlabel('Order p for sum x[n]x[n+p]')
axis.legend(["from LPClite._auto_correlation()"])
```

## Levison-Durbin

### Notes on the notations

One efficient way to derive LP coefficients from auto-correlation vector is the Levison-Durbin recursion.

Explanation can be found in Eqs(38) of (Makhoul 1975).

Given an autocorrelation vector $(R_0, R_1, \cdots, R_M)$,

$\begin{align}
E_0 &= R_0 \\
\text{for}&\quad i \in [1, 2, \cdots, M]: \\
  & \gamma_i = - \frac{R_i + \sum_{j=1}^{i-1} a_j^{(i-1)}R_{i - j}}{E_{i-1}} \\
  & a_i^{(i)} = \gamma_i \\
  & a_j^{(i)} = a_j^{(i-1)} + \gamma_i a_{i-j}^{(i-1)} \\
  & E_i = (1 - \gamma_i^2)E_{i-1} \\
\end{align}$

Here $i$ indices the recursion cycle, and $(a_1^{(M)} , \cdots, a_M^{(M)})$ are the output LP coefficients

Different books may have different conventions to write the Levison-Durbin recursion.
A more detailed explanation is given in Section 5.3 of Orfanidis, S. J. Optimum Signal Processing: An Introduction. (Macmillan publishing company, 1988) (PDF is released by the author http://eceweb1.rutgers.edu/~orfanidi/osp2e/). Here, $a_0$ is introduced so that

$\begin{align}
E_0 &= R_0 \\
a_0^{0} &= 1 \\
\text{for}&\quad i \in [1, 2, \cdots, M]: \\
  & \gamma_i = \color{red}{+} \frac{\sum_{\color{red}{j=0}}^{i-1} a_j^{(i-1)}R_{i - j}}{E_{i-1}} = \color{red}{+} \frac{\sum_{\color{red}{k=1}}^{i} a_{\color{red}{i-k}}^{(i-1)}R_{\color{red}{k}}}{E_{i-1}} \\
  & a_i^{(i)} = \color{red}{-}\gamma_i \\
  & a_j^{(i)} = a_j^{(i-1)} \color{red}{-} \gamma_i a_{i-j}^{(i-1)} \\
  & E_i = (1 - \gamma_i^2)E_{i-1} \\
\end{align}$

API in this tutorial follows the convention in (Orfanidis, 1988). See more details in Sec5.3 of http://eceweb1.rutgers.edu/~orfanidi/osp2e/

### Implementation and toy example

```python
# Consider a very simple example

# Consisder order=3, thus we need (R_0, R_1, R_2, R_3)
order = 3
example_autocorr = autocorrs[100][:order+1]

auto_seq = example_autocorr

# create some buffers
# a_j^i, we only need a_j^i and a_j^(i-1) for each recursion cycle,
#  thus just 2 rows
lpc_coef = np.zeros([2, order + 1])
# E_i
lpc_err = np.zeros([order + 1])
# \gammar
gamma_array = np.zeros(order + 1)

# initialization
# E_0 = R_0
lpc_err[0] = example_autocorr[0]

# Follow the convention in (Orfanidis, 1988) a_0^{0} = 1
#   as the recursion goes on, we will save the lpc coeffients in this reversed order
#      lpc_coef[0, :] = [a_M, a_M-1, ..., a_1, 1, 0, 0]
#.  this is convenient when computing \gamma
lpc_coef[0,0] = 1.0

print("Recursion process (note that the float numbers are truncated for visualization)")

for index in np.arange(1, order+1):
    print("\nLD recursion i = {:d}".format(index))

    # a_0^(i-1)
    lpc_coef[1,index] = 1.0

    # compute gamma \frac{\sum_{\color{red}{k=1}}^{i} a_{\color{red}{i-k}}^{(i-1)}R_{\color{red}{k}}}{E_{i-1}}
    # remember that we save lpc_coef[0] as [a_M, a_M-1, ..., a_1, 1]
    # thus, no need to flip it
    gamma = sum(lpc_coef[0, 0:index] * auto_seq[1:(index+1)]) / lpc_err[index-1]
    # save gamma
    gamma_array[index-1] = gamma

    # print information
    print("\t  E_(i-1)\n\t= {:1.3f}\n".format(lpc_err[index-1]))
    print("\t  {:s} ".format(', '.join(["a_{:d}^(i-1)".format(index - (y+1)) for y in range(index)])))
    print("\t= [{:s}]".format(', '.join(["{:2.5f}".format(y) for y in lpc_coef[0, 0:index]])))
    print("\n\t  {:s}".format(', '.join(["R_{:d}".format(y+1) for y in range(index)])))
    print("\t= [{:s}]".format(', '.join(["{:2.5f}".format(y) for y in auto_seq[1:index+1]])))
    print("\n\t  gamma_i\n\t= {:1.5f} \n\t= np.sum(np.array( [{:s}] ) * np.array( [{:s}] )) / {:2.5f}".format(gamma,
        ', '.join(["{:1.5f}".format(y) for _, y in enumerate(lpc_coef[0, 0:index])]),
        ', '.join(["{:1.5f}".format(y) for _,y in enumerate(auto_seq[1:(index+1)])]),
        lpc_err[index-1]))

    # a_i^{(i)} = \color{red}{-}\gamma_i
    lpc_coef[1, 0] = -1.0 * gamma

    #print information
    print("\n\tWe now compute the LP coefficients")
    print("\n\t a_{:d}^i = gamma_i = {:1.3f}".format(index, lpc_coef[1, 0]))

    # a_j^{(i)} = a_j^{(i-1)} \color{red}{-} \gamma_i a_{i-j}^{(i-1)}
    if index > 1:
        for index2 in np.arange(1, index):
            lpc_coef[1, index2] = lpc_coef[0, index2-1] - gamma * lpc_coef[0, index - 1 - index2]

            #print information
            print("\n\t a_{:d}^i = a_{:d}^(i-1) - gamma * a_{:d}^(i-1) = {:1.3f} - {:1.3f} * {:1.3f} = {:1.3f}".format(
                index2, index2, index - index2, lpc_coef[0, index2-1], gamma, lpc_coef[0, index - 1 - index2], lpc_coef[1, index2]))

        #print information
        print("\n\tWe get: \t")
        print("\n\t  {:s} ".format(', '.join(["a_{:d}^(i)".format(index - (y)) for y in range(index+1)])))
        print("\t= [{:s}]".format(', '.join(["{:2.5f}".format(y) for y in lpc_coef[1, 0:index+1]])))

    # E_i = (1 - \gamma_i^2)E_{i-1}
    lpc_err[index] =lpc_err[index-1] * (1 - gamma * gamma)

    # save the lpc_coef for the next recursion cycle
    lpc_coef[0,:] = lpc_coef[1,:]

# now, we need to flip [a_M, a_M-1, ..., a_1, 1] to [1, a_1, a_2, ..., a_M]
lpc_coef_output = lpc_coef[0,::-1]
# we get the gain
gain = np.sqrt(lpc_err[-2])
```

    Recursion process (note that the float numbers are truncated for visualization)

    LD recursion i = 1
    	  E_(i-1)
    	= 5.910

    	  a_0^(i-1)
    	= [1.00000]

    	  R_1
    	= [5.88580]

    	  gamma_i
    	= 0.99593
    	= np.sum(np.array( [1.00000] ) * np.array( [5.88580] )) / 5.90986

    	We now compute the LP coefficients

    	 a_1^i = gamma_i = -0.996

    LD recursion i = 2
    	  E_(i-1)
    	= 0.048

    	  a_1^(i-1), a_0^(i-1)
    	= [-0.99593, 1.00000]

    	  R_1, R_2
    	= [5.88580, 5.81489]

    	  gamma_i
    	= -0.97739
    	= np.sum(np.array( [-0.99593, 1.00000] ) * np.array( [5.88580, 5.81489] )) / 0.04802

    	We now compute the LP coefficients

    	 a_2^i = gamma_i = 0.977

    	 a_1^i = a_1^(i-1) - gamma * a_1^(i-1) = -0.996 - -0.977 * -0.996 = -1.969

    	We get:

    	  a_2^(i), a_1^(i), a_0^(i)
    	= [0.97739, -1.96934, 1.00000]

    LD recursion i = 3
    	  E_(i-1)
    	= 0.002

    	  a_2^(i-1), a_1^(i-1), a_0^(i-1)
    	= [0.97739, -1.96934, 1.00000]

    	  R_1, R_2, R_3
    	= [5.88580, 5.81489, 5.69893]

    	  gamma_i
    	= 0.07101
    	= np.sum(np.array( [0.97739, -1.96934, 1.00000] ) * np.array( [5.88580, 5.81489, 5.69893] )) / 0.00215

    	We now compute the LP coefficients

    	 a_3^i = gamma_i = -0.071

    	 a_1^i = a_1^(i-1) - gamma * a_2^(i-1) = 0.977 - 0.071 * -1.969 = 1.117

    	 a_2^i = a_2^(i-1) - gamma * a_1^(i-1) = -1.969 - 0.071 * 0.977 = -2.039

    	We get:

    	  a_3^(i), a_2^(i), a_1^(i), a_0^(i)
    	= [-0.07101, 1.11722, -2.03874, 1.00000]

    Finally we get, [ 1.         -2.03873676  1.11722131 -0.07100544]

### More notes

Apologize if the above process is not illustrated in a more intuitive way.
(Your suggestion is welcome)

There are two further notes:

1. We can conduct the Levison-Durbin on all the frames simultaneously. Again, this is just a trick using Numpy array, not a new algorithm;

2. We need to check whether $|\gamma|_i > 1$ for each iteration. A $|\gamma|_i > 1$ indicates an unstable synthesis filter. In practise, we can stop the recursion when $|\gamma|_i > 1$ and use the coefficients from the previous cycle $(a_1^{i-1}, a_2^{i-1}, \cdots, a_{i-1}^{i-1})$ as the final output. In other words.

These two points have been considered in `LPClite._levison_durbin()`

1. `LPClite._levison_durbin()` takes as input an array (frame_num, lpc_order+1). It work on all the frames at the same time;

2. `LPClite._levison_durbin()` handles $|\gamma|_i > 1$ in this part. Because $E_i = E_{i-1} * (1- \gamma_i^2)$, $|\gamma|_i > 1$ leads to $E_i < 0$. This is what being monitored. If $E_i < 0$, we set $gamma_i=0$ so that it will stop computation LPC cofficients for later cycles

``` python
#   step2. check validity of lpc_err
ill_idx = lpc_err[:,index-1] < 1e-07
#      also frames that should have been stopped in previous iter
ill_idx = np.bitwise_or(ill_idx, tmp_order < polyOrder)
#   step3. make invalid frame gamma=0
gamma[ill_idx] = 0
gamma[~ill_idx] = gamma[~ill_idx] / lpc_err[~ill_idx,index-1]
gamma_array[:, index-1] = gamma
#   step4. log down the ill frames
tmp_order[ill_idx] = index
```

### Example using the full speech utterance

Now let's try to use the API and analyze all the speech frames

```python
# autocorrs has shape (frame_num, lpc_order + 1)
lpc_coefs, _, gammas, gains = m_lpc_api._levison_durbin(autocorrs)

# lpc_coefs has shape (frame_num, lpc_order + 1)
# gammas has shape (frame_num, lpc_order)
# gains has shape (frame_num, 1)
print("We get LPC coefficients for {:d} frames".format(lpc_coefs.shape[0]))
```

    We get LPC coefficients for 550 frames

Let's plot the frequency response of model and spectrun of the speech frame.
We can also compare it with output of librosa.lpc

```python
# choose one frame to check
frame_idx = 100
frame_waveform = wave_framed[frame_idx]
gain = gains[frame_idx][0]
lpc_coef = lpc_coefs[frame_idx]

# compute the spectrum for this frame
from scipy import signal

fft_number = 1024
plt.figure(figsize=(10,3))
fft_spec = np.fft.fft(frame_waveform, n = fft_number)[0:int(fft_number/2+1)]
plt.plot(20*np.log10(np.abs(fft_spec+1e-20)))

# compute the frequency response
_, lpc_spec = signal.freqz([gain], lpc_coef, worN = int(fft_number/2))
plt.plot(20*np.log10(np.abs(lpc_spec+1e-20)))

# if necessary, compare the results with Librosa.LPC
if True:
    import librosa
    a = librosa.lpc(frame_waveform, 29)
    _, lpc_spec = signal.freqz([gain], a, worN = int(fft_number/2))
    plt.plot(20*np.log10(np.abs(lpc_spec+1e-20)))

plt.xlabel('Frequency bins')
plt.ylabel('Frequency amplitude (dB)')
plt.legend(['speech spectrum', 'LPC frequency response', 'Librosa.lpc output'])
```

Looks acceptable!

## From reflection coefficients to LP coefficients

### Motivation

Many applications such as speech coding do not directly use LP coefficients. (See for example `Paliwal, K. K. & Atal, B. S. Efficient vector quantization of LPC parameters at 24 bits/frame. IEEE Trans. speech audio Process. 1, 3–14 (1993)`)

Alternative representations of LP coeffcients include reflection coefficients, log-area ratio, and line spectrum pairs (https://en.wikipedia.org/wiki/Line_spectral_pairs). They have nice properties, and many TTS systems use line spectrum pairs (see section 3.3.1.2 of `Zen, H., Tokuda, K. & Black, A. W. Statistical parametric speech synthesis. Speech Commun. 51, 1039–1064 (2009)`)

The reflection coefficients $\gamma$ are derived through the Levinson-Durbin recursion, and they have one-to-one mapping to the LP coefficients. A nice property is that as long as $|\gamma|<1$, the corresponding LP synthesis model is stable (i.e., all poles of $\frac{1}{1 + \sum_{p=1}^{M}a_pz^{-p}}$ stay inside the unit circle).

If we want to predict reflection coefficients from other features, a **tanh()** activtion will make sure $|\gamma|<1$.

### rc2lpc

To convert reflection coefficients to LPC coefficients, we use the Levison-Durvin recursion.

$\begin{align}
E_0 &= R_0 \\
a_0^{0} &= 1 \\
\text{for}&\quad i \in [1, 2, \cdots, M]: \\
  & a_i^{(i)} = \color{red}{-}\gamma_i \\
  & a_j^{(i)} = a_j^{(i-1)} \color{red}{-} \gamma_i a_{i-j}^{(i-1)} \\
  & E_i = (1 - \gamma_i^2)E_{i-1} \\
\end{align}$

This is available in API `LPClite._rc2lpc()`

```python
# Let's try one example
lpc_coefs_fromgamma = m_lpc_api._rc2lpc(gammas)

# lpc_coefs has shape (frame_num, lpc_order + 1)
# gammas has shape (frame_num, lpc_order)
# gains has shape (frame_num, 1)
print("We get LPC coefficients from reflection coef. for {:d} frames".format(lpc_coefs_fromgamma.shape[0]))

# they are identical to the lpc_coefs derived before
np.std(lpc_coefs_fromgamma - lpc_coefs)
```

    We get LPC coefficients from reflection coef. for 550 frames

    0.0

### Comparison

We mentioned the impact of quantization error. Let's check it here:

```python
# select one frame
frame_idx = 100
lpc_coef = lpc_coefs[frame_idx]
gamma = gammas[frame_idx]
lpc_coef_gamma = m_lpc_api._rc2lpc(np.expand_dims(gamma, axis=0))[0]

# quantization (very simple, just truncate the float numbers, 256 = 2 ** 8 bits)
lpc_coef_quan = np.ceil(lpc_coef * 256) / 256
gamma_quan = np.ceil(gamma * 256)/256
lpc_coef_gamma_quan = m_lpc_api._rc2lpc(np.expand_dims(gamma_quan, axis=0))[0]

# for plotting figures
from scipy import signal
def log_spec(spec):
    return 20*np.log10(np.abs(spec+1e-20))
frame_waveform = wave_framed[frame_idx]
gain = gains[frame_idx][0]
fft_number = 1024
fft_spec = np.fft.fft(frame_waveform, n = fft_number)[0:int(fft_number/2+1)]

def plot_figures(lpc_coef, lpc_coef_quan, gain, fft_spec, title):
    plt.figure(figsize=(10,3))
    plt.plot(log_spec(fft_spec))

    # compute the frequency response
    _, lpc_spec = signal.freqz([gain], lpc_coef, worN = int(fft_number/2))
    plt.plot(log_spec(lpc_spec))

    # compute the frequency response
    _, lpc_spec_quan = signal.freqz([gain], lpc_coef_quan, worN = int(fft_number/2))
    plt.plot(log_spec(lpc_spec_quan))
    plt.legend(['speech frame spectrum', 'Spectrum from LPC coef.', 'Spectrum from quantized LPC coef.'])
    plt.title(title)
    return

plot_figures(lpc_coef, lpc_coef_quan, gain, fft_spec, 'Using LPC coefficients')
plot_figures(lpc_coef_gamma, lpc_coef_gamma_quan, gain, fft_spec, 'Using Reflection coefficients')
```

From the above figures, we see how reflection coeffcients can be useful.

```python
print("LPC coef.\n", lpc_coef)
print("Quantized LPC coef.\n", lpc_coef_quan)
print("\nReflection coefficients\n", gamma)
print("Quantized Reflection coefficients\n", gamma_quan)
print("\nLPC coef. from reflection coef.\n", lpc_coef_gamma)
print("LPC coef. from quantized reflection coef.\n", lpc_coef_gamma_quan)
```

    LPC coef.
     [ 1.         -2.25759816  1.61893504 -0.80707088  1.37205659 -1.94182127
      2.05942357 -2.52222464  3.03338199 -2.52923725  2.19015224 -3.1165879
      3.38950535 -2.52055657  2.76796376 -3.42291209  2.73144117 -2.30856245
      2.7714963  -2.5343378   1.803059   -1.78397269  1.88781544 -1.21440392
      0.7321409  -1.02763026  0.98354883 -0.54648684  0.46953537 -0.41197257
      0.14360484]
    Quantized LPC coef.
     [ 1.         -2.25390625  1.62109375 -0.8046875   1.375      -1.94140625
      2.0625     -2.51953125  3.03515625 -2.52734375  2.19140625 -3.11328125
      3.390625   -2.51953125  2.76953125 -3.421875    2.734375   -2.3046875
      2.7734375  -2.53125     1.8046875  -1.78125     1.890625   -1.2109375
      0.734375   -1.02734375  0.984375   -0.54296875  0.47265625 -0.41015625
      0.14453125]

    Reflection coefficients
     [ 9.95928594e-01 -9.77387668e-01  7.10054438e-02 -1.94103693e-02
     -2.63050598e-01 -1.03572519e-01  4.04988430e-01 -1.66318296e-03
      2.85567671e-01 -1.29556498e-01 -1.96712431e-01 -1.25371133e-01
      3.73095840e-01 -8.36763721e-03 -1.82631283e-02 -4.59812619e-01
      4.67856238e-02 -1.24739164e-01  2.22494633e-01 -3.00598972e-01
      3.28130010e-04 -9.28826830e-02  2.03047846e-01 -1.53502189e-01
     -1.73160771e-01 -2.24478486e-01  2.08112570e-01 -4.12011614e-02
      8.96186981e-02 -1.43604837e-01]
    Quantized Reflection coefficients
     [ 0.99609375 -0.9765625   0.07421875 -0.015625   -0.26171875 -0.1015625
      0.40625    -0.          0.2890625  -0.12890625 -0.1953125  -0.125
      0.375      -0.0078125  -0.015625   -0.45703125  0.046875   -0.12109375
      0.22265625 -0.296875    0.00390625 -0.08984375  0.203125   -0.15234375
     -0.171875   -0.22265625  0.2109375  -0.0390625   0.08984375 -0.140625  ]

    LPC coef. from reflection coef.
     [ 1.         -2.25759816  1.61893504 -0.80707088  1.37205659 -1.94182127
      2.05942357 -2.52222464  3.03338199 -2.52923725  2.19015224 -3.1165879
      3.38950535 -2.52055657  2.76796376 -3.42291209  2.73144117 -2.30856245
      2.7714963  -2.5343378   1.803059   -1.78397269  1.88781544 -1.21440392
      0.7321409  -1.02763026  0.98354883 -0.54648684  0.46953537 -0.41197257
      0.14360484]
    LPC coef. from quantized reflection coef.
     [ 1.         -2.26104736  1.61934992 -0.79752732  1.36148766 -1.93556773
      2.05280923 -2.51587227  3.0307638  -2.5211543   2.17330658 -3.10371686
      3.38053871 -2.5085728   2.75706327 -3.41251276  2.71345894 -2.28729545
      2.7541263  -2.52040316  1.79133835 -1.77224523  1.87495998 -1.20172389
      0.72179553 -1.02199169  0.9800435  -0.54182982  0.46371315 -0.40602684
      0.140625  ]

### Note

It is also possible to derive reflection coefficients from LP coefficients. Please check `backward Levinson recursion` on this page http://eceweb1.rutgers.edu/~orfanidi/osp2e/

## Analysis and synthesis

### Do analysis and synthesis on one frame

Analysis and synthesis given the LPC coefficients are straightforward

```python
# take one frame as example
frame_idx = 100
lpc_coef = lpc_coefs[frame_idx]
frame_waveform = wave_framed[frame_idx]
gain = gains[frame_idx][0]
order = lpc_coef.shape[0]-1

# analysis

# buffer to store the residual or excitation signal
frame_excit = np.zeros_like(frame_waveform)

# A simple implementation
#        e[n] = (x[n] + \sum_k=1 x[n-k] a_k) / gain
#
for i in range(frame_waveform.shape[0]):
    for k in range(1+order):
        if i-k < 0:
            break
        frame_excit[i] += frame_waveform[i-k] * lpc_coef[k]
    frame_excit[i] = frame_excit[i] / gain

# synthesis

# buffer to store the synthesized signal
frame_synthesized = np.zeros_like(frame_waveform)

# A simple implementation
#       x[n] = e[n] * gain- \sum_k=1 x[n-k] a_k
#
for i in range(frame_waveform.shape[0]):
    frame_synthesized[i] = frame_excit[i] * gain
    for k in range(1, order+1):
        if i-k < 0:
            break
        frame_synthesized[i] -= frame_synthesized[i-k] * lpc_coef[k]

fig = plt.figure()
axis = fig.add_subplot(311)
axis.plot(frame_waveform)
axis.set_title("Original waveform")
axis = fig.add_subplot(312)
axis.plot(frame_excit)
axis.set_title("LP residual")
axis = fig.add_subplot(313)
axis.plot(frame_synthesized)
axis.set_title("Synthesized waveform")
fig.subplots_adjust(hspace=0.5)
```

### The analysis and synthesis API

This is just one example on one frame.

When doing the analysis on multiple frames, we can use Numpy.array and do it more efficiently.

These are provided in `LPClite._lpc_analysis_core()` and `LPClite._lpc_synthesis_core()`

```python
# Analysis on all the frames
# lpc_coefs,   (frame_num, order + 1)
# wave_framed, (frame_num, frame_length)
# gains,       (frame_num, 1)
# frame_excit, (frame_num, frame_length)
frame_excit = m_lpc_api._lpc_analysis_core(lpc_coefs, wave_framed, gains)

# Synthesis on all the frames
frame_wave_synthesized = m_lpc_api._lpc_synthesis_core(lpc_coefs, frame_excit, gains)
```

```python
# Plot some figures

# take one frame as example
frame_idx = 100
fig = plt.figure()
axis = fig.add_subplot(311)
axis.plot(wave_framed[frame_idx])
axis.set_title("Original waveform")
axis = fig.add_subplot(312)
axis.plot(frame_excit[frame_idx])
axis.set_title("LP residual")
axis = fig.add_subplot(313)
axis.plot(frame_wave_synthesized[frame_idx])
axis.set_title("Synthesized waveform")
fig.subplots_adjust(hspace=0.5)
```

## Overlap-add

After doing analysis and synthesis on each frame, we can do overlap-add to "concatenate" the waveform in frames into a single waveform.

This topic is explained in `c10_waveform_tools`. The method is also included as `LPClite._overlapadd()`

```python
# The outptu will be (length, 1)
waveform_reconstructed = m_lpc_api._overlapadd(frame_wave_synthesized)

fig = plt.figure(figsize=(10, 5))
axis = fig.add_subplot(211)
axis.plot(wave_data)
axis.set_title('Original waveform')
axis = fig.add_subplot(212)
axis.plot(waveform_reconstructed)
axis.set_title('LP analysis and synthesized')
fig.subplots_adjust(hspace=0.5)

IPython.display.display('Original waveform')
IPython.display.display(IPython.display.Audio(wave_data, rate=sampling_rate, normalize=False))
IPython.display.display('LP analysis and synthesized')
IPython.display.display(IPython.display.Audio(waveform_reconstructed[:, 0], rate=sampling_rate, normalize=False))
```

We can also overlap-add the excitation waveforms in each frame

```python
# The outptu will be (length, 1)
waveform_excitation = m_lpc_api._overlapadd(frame_excit)
waveform_excitation = waveform_excitation[:, 0]

fig = plt.figure(figsize=(10, 2))
axis = fig.add_subplot(111)
axis.plot(waveform_excitation)
axis.set_title('Excitation waveform')

# If you want to play the audio, please decrease the volume
#IPython.display.display('Excitation waveform')
#IPython.display.display(IPython.display.Audio(waveform_excitation, rate=sampling_rate, normalize=False))

# If you want to check the spectrogram
if False:
    excit_spectrogram = tool_lib.spec_amplitude(waveform_excitation, fft_bins=4096, frame_shift=40, frame_length=240)
    fig=plt.figure(); axis=fig.add_subplot(211);
    plot_lib.plot_spectrogram(excit_spectrogram, fig, axis, "", "frequency (kHz)", sampling_rate = sampling_rate, title='Excitation signal')
```

Note that the above excitation waveform is from $\frac{G}{1 + \sum_{p=1}^{M}a_pz^{-p}}$, with $G$ into consideration.

You may try to do analysis and synthesis without the gain $G$.

## Wrapper functions

The LPClite provides high level APIs to do analysis and synthesis

```python
# claim an LPC object
frame_shift = 80
frame_length = 320
order = 30

m_lpc_api = LPClite(frame_length, frame_shift, order, flag_emph=False)

# load waveform
wavfile_path = './data_models/acoustic_features/hn_nsf/slt_arctic_b0474.wav'
sampling_rate, wave_data = tool_lib.waveReadAsFloat(wavfile_path)

# do analysis
# The API accepts waveform in shape (length, 1)
wave_data = np.expand_dims(wave_data, axis=1)
lpc_coefs, _, gammas, gains, frame_excit, _ = m_lpc_api.analysis(wave_data)

# do synthesis
# The API returns waveform in shape (length, 1)
waveform_reconstructed = m_lpc_api.synthesis(lpc_coefs, frame_excit, gains)
waveform_reconstructed = waveform_reconstructed[:, 0]

# plot figures
fig = plt.figure(figsize=(10, 5))
axis = fig.add_subplot(211)
axis.plot(wave_data)
axis.set_title('Original waveform')
axis = fig.add_subplot(212)
axis.plot(waveform_reconstructed)
axis.set_title('LP analysis and synthesized')
fig.subplots_adjust(hspace=0.5)
```

<a id="finalnote"></a>
## Final note on its usage for Pytorch project
The API above are based on Numpy. I personally use them to get the numpy array before they are converted into torch.tensor.

A good place to plug it in is the data transformation functions in `core_scripts/data_io/default_data_io.py f_post_data_process()`

In this way, LP analysis can be conducted on CPU before the data is converted into a Pytorch tensor. Our implementation of the iLPCNet will use this function and extract LP coefficients and other features.  See more details in s6_demonstration_iLPCNet.ipynb.

```c
waveform data on disk ----> DataSet.__getitem__()  -----> Collate  ---->  Pytorch model
                                numpy.tensor           torch.tensor
                                LP analysis
```

That's all


    Run in Google Colab

    View source on GitHub

# Recommendation Retrieval

A typical recommender system has two phases.  In the first phase (preference learning), which can be done offline, we learn the model from observations.  In the second phase (retrieval), upon the appearance of a user, we retrieve the top-ranked recommendations for that user.  This latter phase needs to be done online.  In this tutorial, we investigate ways to speed up the retrieval phase by avoiding an exhaustive search over all items.  The focus is on matrix factorization-based recommender systems.

## 1. Setup

```
!pip install --quiet cornac==1.6.1 torch>=0.4.1
```

```
import os
import sys
import time
import random
import pickle
from operator import itemgetter
from collections import defaultdict

import tqdm.auto as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

import cornac
from cornac.utils import cache
from cornac.datasets import movielens
from cornac.eval_methods import RatioSplit
from cornac.models import PMF, BPR, IBPR

print(f"System version: {sys.version}")
print(f"Cornac version: {cornac.__version__}")

SEED = 42
VERBOSE = True
```

    System version: 3.6.9 (default, Apr 18 2020, 01:56:04)
    [GCC 8.4.0]
    Cornac version: 1.6.1

## 2. Recommendation Retrieval is a Similarity Search Problem

The two phases of a matrix factorization-based recommender system are:
1. **Learning phase:** derive a $d$-dimensional latent vector $\mathbf{u}_i$ for each user $i$ and a $d$-dimensional latent vector $\mathbf{v}_j$ for each item $j$. The user-item relationship is modeled using the inner product kernel, i.e., the user preference of user $i$ for an item $j$ is given by:

$$\hat{r}_{i,j} = \mathbf{u}_i^T \mathbf{v}_j$$

2. **Retrieval phase:** given a user vector $\mathbf{u}_i$, construct a recommendation list of $K$ items with the highest preference scores. This step requires $\mathcal{O}(d \times M)$ preference score computations for $M$ number of items.

### Probabilistic Matrix Factorization

For the retrieval phase, the straightforward approach is to exhaustively compute the preference score for every item and maintain the top-$K$. Let us take a closer look through an example based on Probabilistic Matrix Factorization (PMF) [1].

For PMF, the objective is to learn for each user $i$ a vector $\mathbf{u}_i$ and each item $j$ a vector $\mathbf{v}_j$, such that the following error function is minimized over the set of observed ratings $\mathbf{R}$:
\begin{equation}
 \mathcal{L} = \sum_{r_{ui}\in \mathbf{R}}(r_{ui} - \mathbf{u}_i^T\mathbf{v}_j)^2 + \lambda \sum_{i = 1}^{N}||\mathbf{u}_i||^2 + \lambda\sum_{j = 1}^{M}||\mathbf{v}_j||^2,
\end{equation}
where $\lambda$ is a regularization term.

We begin by running PMF on the Movielens 1M dataset.

```
# Load the MovieLens 1M dataset
ml_1m = movielens.load_feedback(variant='1M')
```

```
# Instantiate an evaluation method
ratio_split = RatioSplit(
  data=ml_1m, test_size=0.4, rating_threshold=1.0, exclude_unknowns=False
)

# Instatiate a PMF recommender model
pmf = PMF(k=100, max_iter=100, learning_rate=0.01, seed=SEED, verbose=VERBOSE)

# Instantiate evaluation metrics.
rec_10 = cornac.metrics.Recall(k=10)
pre_10 = cornac.metrics.Precision(k=10)

# Instantiate and then run an experiment.
cornac.Experiment(
  eval_method=ratio_split, models=[pmf], metrics=[rec_10, pre_10]
).run()
```

    TEST:
    ...
        | Precision@10 | Recall@10 | Train (s) | Test (s)
    --- + ------------ + --------- + --------- + --------
    PMF |       0.0789 |    0.0164 |  144.9316 |   7.3906

### Exhaustive Search

To retrieve the top-$K$ recommendation for a user $i$, we need to compute the inner product between the user's latent vector $\mathbf{u}_i$ and that of every item $\mathbf{v}_j$.  This is directly proportional to the number of items $M$ as well as the number of latent factors (dimensionality of the item vector) $d$, as we will see below.

**Observation #1: Retrieval time scales linearly with the number of items**

```
comp_time = []

# For each user vector, we compute the inner product score to each item vector
pmf_item_vect = pmf.V
pmf_user_vect = pmf.U

for incr_size in tqdm.trange(0, pmf_item_vect.shape[0], 10, disable=not VERBOSE):
  start_time = time.time()
  pred = np.matmul(pmf_item_vect[:incr_size, :], pmf_user_vect.T)
  end_time = time.time()
  comp_time.append((end_time - start_time) / pmf_user_vect.shape[0])
```

```
# plot the computation time as the number of items increases
plt.plot(range(0, pmf_item_vect.shape[0],10), comp_time, marker='o',
         label='Computation_time')
plt.xlabel('Number of Items')
plt.ylabel('Computation Time (seconds)')
plt.title('Inner product computation time (seconds) vs. number of items')
plt.grid()
plt.gcf().set_size_inches(10, 5)
plt.tight_layout()
plt.show()
```

**Observation #2: Retrieval time scales linearly with the number of latent factors**

```
comp_time = []

for d in tqdm.trange(pmf.k, disable=not VERBOSE):
  start_time = time.time()
  pred = np.matmul(pmf_item_vect[:incr_size, :d], pmf_user_vect[:,:d].T)
  end_time = time.time()

  # average the computation time
  comp_time.append((end_time - start_time) / pmf_user_vect.shape[0])
```

```
# plot the computation time as the number of dimension increases
plt.plot(range(pmf.k), comp_time, marker='o', label='Computation_time')
plt.title('Inner product computation time (seconds) vs. number of features')
plt.xlabel('Number of features')
plt.ylabel('Computation Time (seconds)')
plt.grid()
plt.gcf().set_size_inches(10, 5)
plt.tight_layout()
plt.show()
```

## 3. Scalable Recommendation Retrieval with Locality Sensitive Hashing
From the previous example, we can see that the cost of **exhaustive search** scales linearly to the number of items, i.e., $M$ and number of features, i.e., $d$. For modern real-world systems, the number of items could go up to the scale of millons.  Therefore, it would be fruitful to make the search more efficient.

### Locality Sensitive Hashing
One way to do so is to use Locality Sensitive Hashing (LSH), which is a probabilistic space-partitioning indexing technique. One important element of LSH is the hashing function $h(.)$, which maps a data point into a hash value. This hashing function is usually designed with the *locality-sensitive* property, stated as: *two similar data points are more likely to get the same hash values as compared to two distant data points.*

One of the most popular search protocols using LSH is hash table look-up.  In a nutshell, we first derive the hash codes for all items and construct a hash table that links a hash code to items of that code. When a user appears, we hash the user's latent vector into a hash code.  We then look up items of that hash code and compute the top-$K$ based on inner product *only among those items*, thus drastically reducing the number of items we need to consider.  To increase the search space, we can also use multiple hash tables, in which case we look up the multiple hash tables and compute the top-$K$ across the tables.

```
random.seed(SEED)

# Implementation of Locality Sensitive Hashing for Maximum Cosine Similarity Search
class LSHIndex:

  def __init__(self, hash_family, k, L):
    self.hash_family = hash_family
    self.k = k
    self.L = 0
    self.hash_tables = []
    self.resize(L)

  def resize(self, L):
    """ update the number of hash tables to be used """
    if L < self.L:
      self.hash_tables = self.hash_tables[:L]
    else:
      # initialise a new hash table for each hash function
      hash_funcs = [[self.hash_family.create_hash_func() for h in range(self.k)]
                    for l in range(self.L, L)]
      self.hash_tables.extend([(g, defaultdict(lambda:[])) for g in hash_funcs])

  def hash(self, g, p):
    return self.hash_family.combine([h.hash(p) for h in g])

  def index(self, points):
    """ index the supplied points """
    self.points = points
    for g, table in self.hash_tables:
        for ix, p in enumerate(self.points):
            table[self.hash(g,p)].append(ix)
    # reset stats
    self.tot_touched = 0
    self.num_queries = 0

  def query(self, q, metric, max_results):
    """ find the max_results closest indexed points to q according to the supplied metric """
    candidates = set()
    for g, table in self.hash_tables:
        matches = table.get(self.hash(g,q), [])
        candidates.update(matches)

    # update stats
    self.tot_touched += len(candidates)
    self.num_queries += 1

    # rerank candidates
    candidates = [(ix, metric(q, self.points[ix])) for ix in candidates]
    candidates.sort(key = itemgetter(1))
    return candidates[:max_results]

  def get_avg_touched(self):
    """ mean number of candidates inspected per query """
    return self.tot_touched/self.num_queries

#################################################################################################
#--------------------------------- Cosine LSH Hash Family --------------------------------------#
#################################################################################################
class CosineHashFamily:

  def __init__(self,d):
    self.d = d

  def create_hash_func(self):
    # each CosineHash is initialised with a random projection vector
    return CosineHash(self.rand_vec())

  def rand_vec(self):
    return [random.gauss(0,1) for i in range(self.d)]

  def combine(self, hashes):
    """combine by treating as a bitvector"""
    return sum(2**i if h > 0 else 0 for i,h in enumerate(hashes))

class CosineHash:

  def __init__(self, r):
    self.r = r

  def hash(self, vec):
    return self.sgn(dot(vec, self.r))

  def sgn(self,x):
    return int(x > 0)

#--- inner product ---
def dot(u,v):
	return np.dot(u, v)

#-- cosine distance ---
def cosine_distance(u,v):
	return 1 - dot(u,v)/(dot(u,u)*dot(v,v))**0.5
```

```
# helper function to measure precision, recall, and touched values
# of using LSH for top-k recommendation retrieval
# ------------------------------------------------------------------------
def evaluate_LSHTopK(test_data, item_vect, user_vect, lsh_index, sim_metric, topK):

  lsh_prec = 0.0
  lsh_recall = 0.0

  #build index
  lsh_index.index(item_vect.tolist())

  for id in range(user_vect.shape[0]):
    test_item_ids = list(test_data.csr_matrix.getrow(id).nonzero()[1])

    if len(test_item_ids) > 0:
      # top-k by LSH
      lsh_rec_list = list(map(itemgetter(0), lsh_index.query(user_vect[id, :], sim_metric, topK)))
      lsh_overlaps = list(set(lsh_rec_list).intersection(test_item_ids))

      lsh_prec   += len(lsh_overlaps) * 1.0/topK
      lsh_recall += len(lsh_overlaps) * 1.0/len(test_item_ids)

  touched = lsh_index.get_avg_touched() * 1.0/item_vect.shape[0]
  return lsh_prec/user_vect.shape[0], lsh_recall/user_vect.shape[0], touched
```

### Issue with Using Locality Sensitive Hashing for Recommendation Retrieval

Recommendation retrieval, which relies on inner product computation is not directly compatible with Locality Sensitive Hashing, which is designed for nearest neighbor search or maximum cosine similarity search. This might lead to the significant degeneration of recommendation accuracy after indexing [2].

In the following, we introduce two approaches to this issue:

1.   Vector Augmentation
2.   Indexable Representation

### Solution I: Vector Augmentation
In this solution, we first apply the vector transformation for both user and item vectors, as introduced in the paper [3].

For each item vector $\mathbf{v}_j$:
\begin{equation}
\mathbf{\tilde{v}}_j = [\mathbf{v}_j; \sqrt{\phi^2 - ||\mathbf{v}_j||^2}]  \hspace{1cm}(\mathrm{where}\ \phi = \max\{||\mathbf{v}_j||\})
\end{equation}

For each user vector $\mathbf{u}_i$:
\begin{equation}
\mathbf{\tilde{u}}_i = [\mathbf{u}_i; 0]
\end{equation}

We have the following observation:

\begin{equation}
\frac{\mathbf{\tilde{u}}_i^T\mathbf{\tilde{v}}_j}{||\mathbf{\tilde{u}}_i||\times ||\mathbf{\tilde{v}}_j||} = \frac{\mathbf{\tilde{u}}_i^T\mathbf{\tilde{v}}_j}{\phi ||\mathbf{\tilde{u}}_i||}
\end{equation}

i.e.,

\begin{equation}
\arg\max_{1\leq j\leq M}{\mathbf{{u}}_i^T\mathbf{{v}}_j} = \arg\max_{1\leq j\leq M}\frac{\mathbf{\tilde{u}}_i^T\mathbf{\tilde{v}}_j}{||\mathbf{\tilde{u}}_i||\times ||\mathbf{\tilde{v}}_j||}
\end{equation}

With this vector augmentation, we effectively convert a Maximum Inner Product Search (MIPS) problem to a Maximum Cosine Similarity Search (MCCS), which can be solved with LSH.  Hypothetically, this will make the user and item latent vectors more compatible with LSH.

```
#apply vector augmentation
M = np.linalg.norm(pmf_item_vect, axis=1) # compute item vector norms
max_norm = max(M) # max item norm

xbox_item_vect = np.concatenate(
  (pmf_item_vect, np.sqrt(max_norm**2 - pow(M, 2)).reshape(pmf_item_vect.shape[0], -1)),
  axis=1
)
xbox_user_vect = np.concatenate(
  (pmf_user_vect, np.zeros((pmf_user_vect.shape[0], 1))),
  axis=1
)
```

#### Effectiveness of Vector Augmentation

In this experiment, we build LSH index on the output of PMF algorithm and compare the latent vectors before and after vector augmentation for LSH recommendation retrieval.

Since the augmented user and item vectors are in $(d+1)-$dimensional space, for comparison purpose we append 0s to the original non-augmented user and item vectors (to make them the same length), i.e.,

\begin{equation}
\mathbf{\tilde{v}}_j = [\mathbf{v}_j; 0]
\end{equation}

\begin{equation}
\mathbf{\tilde{u}}_i = [\mathbf{u}_i; 0]
\end{equation}

With this non-augmented "transformation", we have no change to the original inner product:

\begin{equation}
\mathbf{\tilde{u}}_i^T\mathbf{\tilde{v}}_j = \mathbf{{u}}_i^T\mathbf{{v}}_j
\end{equation}

```
padded_user_vect = np.concatenate((pmf_user_vect, np.zeros((pmf_user_vect.shape[0], 1))), axis=1)
padded_item_vect = np.concatenate((pmf_item_vect, np.zeros((pmf_item_vect.shape[0], 1))), axis=1)
```

Specifically, we report three evaluation metrics as follows:

1. lsh_prec@10 = $\text{precision@10 of LSH Indexing}$

2. lsh_rec@10    = $\text{recall@10 of LSH Indexing}$

3. touched = $\frac{\text{Average number of investigated items by LSH}}{\text{Total number of items}}$

```
topK = 10 # @param - top-K value
b_vals = [4, 8] # @param - number of hash function
L_vals = [10, 20] # @param - number of hashtables

test_data  = ratio_split.test_set # testing data

print('#table \t#bit \t ?Augmented \t lsh_prec@{0} \t lsh_recall@{0} \t touched'.format(topK))
for nt in L_vals:
  for b in b_vals:
    #init lsh index:
    #------ hash-family: the LSH scheme/family
    #------ k          : number of hash functions
    #------ L          : number of hash tables
    lsh_index = LSHIndex(hash_family = CosineHashFamily(padded_item_vect.shape[1]), k = b, L=nt)

    #performance without employing vector augmentation
    print('---------------------------------------------------------------------------------')
    prec_1, recall_1, touched_1 = evaluate_LSHTopK(
      test_data, padded_item_vect, -padded_user_vect[:1000], lsh_index, dot, topK
    )
    print("{}\t{}\t{}\t{:.4f}\t{:.4f}\t{:.4f}".format(nt, b, 'No', prec_1, recall_1, touched_1))

    #performance with vector augmentation
    prec_2, recall_2, touched_2 = evaluate_LSHTopK(
      test_data, xbox_item_vect, -xbox_user_vect[:1000], lsh_index, dot, topK
    )
    print("{}\t{}\t{}\t{:.4f}\t{:.4f}\t{:.4f}".format(nt, b, 'Yes', prec_2, recall_2, touched_2))
```

    #table 	#bit 	 ?Augmented 	 lsh_prec@10 	 lsh_recall@10 	 touched
    ---------------------------------------------------------------------------------
    10	4	No	0.1193	0.0108	0.4544
    10	4	Yes	0.1211	0.0110	0.6889
    ---------------------------------------------------------------------------------
    10	8	No	0.0661	0.0051	0.0395
    10	8	Yes	0.0766	0.0062	0.0698
    ---------------------------------------------------------------------------------
    20	4	No	0.1271	0.0124	0.7045
    20	4	Yes	0.1316	0.0124	0.8926
    ---------------------------------------------------------------------------------
    20	8	No	0.0861	0.0067	0.0768
    20	8	Yes	0.0947	0.0075	0.1378

We observe from the results above that with vector augmentation, we can achieve higher recommendation accuracy after using Locality Sensitive Hashing.

What is the effect of increasing the number of bits, or the number of hash tables? Why?

### Solution II: Indexable Representation Learning
Another solution is "Indexable representation", which refers to recommendation algorithms whose user/item vectors are immediately sublinearly searchable. In this tutorial, we are going to experiment with one such model, namely Indexable Bayesian Personalized Ranking or IBPR [5] for short. IBPR is an extension of the popular framework for implicit feedback: Bayesian Personalized Ranking (BPR) [4].

#### Bayesian Personalized Ranking - BPR

Let us see the performance of BPR with exhaustive search.

```
rec_bpr = BPR(k=20, max_iter=100, learning_rate=0.001, seed=SEED, verbose=VERBOSE)

cornac.Experiment(
  eval_method=ratio_split, models=[rec_bpr], metrics=[rec_10, pre_10],
).run()
```

    TEST:
    ...
        | Precision@10 | Recall@10 | Train (s) | Test (s)
    --- + ------------ + --------- + --------- + --------
    BPR |       0.1838 |    0.0448 |   20.5633 |   6.7531

We now measure the performance of BPR after LSH indexing.

```
bpr_user_vect  = rec_bpr.u_factors
bpr_item_vect  = rec_bpr.i_factors

# apply Xbox transformation
M = np.linalg.norm(bpr_item_vect, axis=1) # compute item vector norms
max_norm = max(M) # max item norm

xbox_bpr_item_vect = np.concatenate(
  (bpr_item_vect, np.sqrt(max_norm**2 - pow(M, 2)).reshape(bpr_item_vect.shape[0], -1)),
  axis=1
)
xbox_bpr_user_vect = np.concatenate(
  (bpr_user_vect, np.zeros((bpr_user_vect.shape[0], 1))),
  axis = 1
)
```

```
topK = 10 # @param
b_vals = [4, 8] # @param
L_vals = [10, 20] # @param

test_data  = ratio_split.test_set # testing data

print('#table\t #bit \t model \t lsh_prec@{0} \t lsh_recall@{0} \t touched'.format(topK))
for nt in L_vals:
  print('------------------------------------------------------------------------------')
  for b in b_vals:
    # lsh index:
    #------ hash-family: the LSH scheme/family
    #------ k          : number of hash functions
    #------ L          : number of hash tables
    lsh_index = LSHIndex(hash_family = CosineHashFamily(xbox_bpr_item_vect.shape[1]), k = b, L = nt)

    lsh_bpr_prec, lsh_bpr_recall, touched_bpr = evaluate_LSHTopK(
      test_data, xbox_bpr_item_vect, -xbox_bpr_user_vect[:1000, :], lsh_index, dot, topK
    )
    print("{}\t{}\t{}\t{:.4f}\t{:.4f}\t{:.4f}".format(
      nt, b, 'BPR', lsh_bpr_prec, lsh_bpr_recall, touched_bpr)
    )
```

    #table	 #bit 	 model 	 lsh_prec@10 	 lsh_recall@10 	 touched
    ------------------------------------------------------------------------------
    10	4	BPR	0.0417	0.0024	0.4809
    10	8	BPR	0.0340	0.0022	0.0512
    ------------------------------------------------------------------------------
    20	4	BPR	0.0442	0.0030	0.7138
    20	8	BPR	0.0405	0.0025	0.0969

The results above show that in relative terms, the performance degenerates for both precision and recall.

#### Indexable Bayesian Personalized Ranking - IBPR

We will now test the effectiveness of indexable model IBPR. In this experiment, we train IBPR with the same data we use for BPR and measure the performance after LSH indexing.

```
rec_ibpr = IBPR(k=20, max_iter=15, learning_rate=0.001, verbose=VERBOSE)

cornac.Experiment(
  eval_method=ratio_split, models =[rec_ibpr], metrics=[rec_10, pre_10]
).run()
```

    TEST:
    ...
         | Precision@10 | Recall@10 | Train (s) | Test (s)
    ---- + ------------ + --------- + --------- + --------
    IBPR |       0.1669 |    0.0386 |  627.9537 |   6.4840

```
ibpr_user_vect = rec_ibpr.U
ibpr_item_vect = rec_ibpr.V
```

```
topK = 10 # @param
b_vals = [4, 8] # @param
L_vals = [10, 20] # @param

test_data = ratio_split.test_set # testing data

print('#table\t #bit \t model \t lsh_prec@{0} \t lsh_recall@{0} \t touched'.format(topK))
for nt in L_vals:
  print('------------------------------------------------------------------------------')
  for b in b_vals:
    # lsh index:
    #------ hash-family: the LSH scheme/family
    #------ k          : number of hash functions
    #------ L          : number of hash tables
    lsh_index = LSHIndex(hash_family = CosineHashFamily(ibpr_item_vect.shape[1]), k = b, L = nt)

    lsh_ibpr_prec, lsh_ibpr_recall, touched_ibpr = evaluate_LSHTopK(
      test_data, ibpr_item_vect, -ibpr_user_vect[:1000,:], lsh_index, dot, topK
    )
    print("{}\t{}\t{}\t{:.4f}\t{:.4f}\t{:.4f}".format(
      nt, b, 'IBPR', lsh_ibpr_prec, lsh_ibpr_recall, touched_ibpr)
    )
```

    #table	 #bit 	 model 	 lsh_prec@10 	 lsh_recall@10 	 touched
    ------------------------------------------------------------------------------
    10	4	IBPR	0.0677	0.0041	0.6673
    10	8	IBPR	0.0252	0.0015	0.4574
    ------------------------------------------------------------------------------
    20	4	IBPR	0.0843	0.0056	0.7131
    20	8	IBPR	0.0371	0.0021	0.4916

We observe that the performance after LSH of IBPR (without vector augmentation) could be better than that of BPR (with vector augmentation), which may be attributed to having representations more compatible with indexing.  As a result, we could speed up top-$K$ recommendation retrieval, while better preserving the accuracy.

## References

1. Mnih, Andriy, and Russ R. Salakhutdinov. "Probabilistic matrix factorization." Advances in neural information processing systems. 2008.

2. Stochastically Robust Personalized Ranking for LSH Recommendation Retrieval, Dung D. Le and Hady W. Lauw
AAAI Conference on Artificial Intelligence (AAAI-20)

3. Bachrach, Yoram, et al. "Speeding up the xbox recommender system using a euclidean transformation for inner-product spaces." Proceedings of the 8th ACM Conference on Recommender systems. 2014.

4. Rendle, Steffen, et al. "BPR: Bayesian personalized ranking from implicit feedback." arXiv preprint arXiv:1205.2618 (2012).

5. Le, Dung D., and Hady W. Lauw. "Indexable Bayesian personalized ranking for efficient top-k recommendation." Proceedings of the 2017 ACM on Conference on Information and Knowledge Management. 2017.

6.   Cornac - A Comparative Framework for Multimodal Recommender Systems (https://cornac.preferred.ai/)


# Dropout and maxout
In this lab we will explore the methods of [dropout](https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf), a regularisation method which stochastically drops out activations from the model during training, and [maxout](http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf), another non-linear transformation that can be used in multiple layer models. This is based on material covered in the [fifth lecture slides](http://www.inf.ed.ac.uk/teaching/courses/mlp/2016/mlp05-hid.pdf).

## Exercise 1: Implementing a dropout layer

During training the forward propagation through a dropout layer produces outputs where a subset of the input dimensions are set to zero ('dropped out'). The dimensions to be dropped out are randomly sampled for each new batch, with each dimension having a probability $p$ of being included and the inclusion (or not) of each dimension independent of all the others. If the inputs to a dropout layer are $D$ dimensional vectors then we can represent the dropout operation by an elementwise multiplication by a $D$ dimensional *binary mask* vector $\boldsymbol{m} = \left[m_1 ~ m_2 ~\dots~ m_D\right]^{\rm T}$ where $m_d \sim \text{Bernoulli}(p) ~~\forall d \in \lbrace 1 \dots D\rbrace$.

As a first step implement a `random_binary_mask` function in the cell below to generate a binary mask array of a specified shape, where each value in the outputted array is either a one with probablity `prob_1` or zero with probability `1 - prob_1` and all values are sampled independently.

```python
def random_binary_mask(prob_1, shape, rng):
    """Generates a random binary mask array of a given shape.

    Each value in the outputted array should be an indepedently sampled
    binary value i.e in {0, 1} with the probability of each value
    being 1 being equal to `prob_1`.

    Args:
        prob_1: Scalar value in [0, 1] specifying probability each
            entry in output array is equal to one.
        shape: Shape of returned mask array.
        rng (RandomState): Seeded random number generator object.

    Returns:
        Random binary mask array of specified shape.
    """
    return rng.uniform(size=shape) < prob_1
```

Test your `random_binary_mask` function using the cell below (if your implementation is incorrect you will get an `AssertionError` - look at what the assert statement is checking for a clue as to what is wrong).

```python
import numpy as np
test_shapes = [(1, 1000), (10, 10, 10)]
test_probs = [0.1, 0.5, 0.7]
for i in range(10):
    for shape in test_shapes:
        for prob in test_probs:
            output = random_binary_mask(prob, shape, np.random)
            # Check generating correct shape output
            assert output.shape == shape
            # Check all outputs are binary values
            assert np.all((output == 1.) | (output == 0.))
            # Check proportion equal to one plausible
            # This will be noisy so there is a chance this will error
            # even for a correct implementation
            assert np.abs(output.mean() - prob) < 0.1
```

Given a randomly sampled binary mask $\boldsymbol{m}$, the outputs $\lbrace \boldsymbol{y}^{(b)} \rbrace_{b=1}^B$ of the stochastic forward propagation through a dropout layer given a batch of inputs $\lbrace \boldsymbol{x}^{(b)} \rbrace_{b=1}^B$ can be calculated by simply performing an elementwise multiplication of the inputs with the mask

\begin{equation}
  y^{(b)}_d = m_k x^{(b)}_d \qquad \forall d \in \lbrace 1 \dots D \rbrace
\end{equation}

The corresponding partial derivatives required for implementing back-propagation through a dropout layer are

\begin{equation}
  \frac{\partial y^{(b)}_k}{\partial x^{(b)}_d} =
  \begin{cases}
      m_k & \quad k = d \\
      0   & \quad k \neq d
  \end{cases}
  \qquad \forall k,\,d \in \lbrace 1 \dots D \rbrace
\end{equation}

As discussed in the lecture slides, when using a model trained with dropout at test time dimensions are no longer stochastically dropped out and instead all activations are deterministically fed forward through the model. So that the expected (mean) outputs of each layer are the same at test and training we scale the forward propagated inputs during testing by $p$ the probability of each dimension being included in the output. If we denote the deterministically forward-propagated batch of outputs of a dropout layer at test time as $\lbrace \boldsymbol{z}^{(b)} \rbrace_{b=1}^B$ then we have

\begin{equation}
  z^{(b)}_d =
  \mathbb{E}\left[ y^{(b)}_d \right] =
  \sum_{m_d \in \lbrace 0,1 \rbrace} \left( \mathbb{P}\left[\mathrm{m}_d = m_d\right] m_d x^{(b)}_d \right) =
  (p) (1) x^{(b)}_d  + (1-p) (0) x^{(b)}_d =
  p x^{(b)}_d  \qquad \forall d \in \lbrace 1 \dots D \rbrace
\end{equation}

To allow switching between this stochastic training time behaviour and deterministic test time behaviour, a new abstract `StochasticLayer` class has been defined in the `mlp.layers` module. This acts similarly to the layer objects we have already encountered other than adding an extra boolean argument `stochastic` to the `fprop` method interface. When `stochastic = True` (the default) a stochastic forward propagation should be caculated, for dropout this corresponding to $\boldsymbol{x}^{(b)} \to \boldsymbol{y}^{(b)}$ above. When `stochastic = False` a deterministic forward-propagation corresponding to the expected output of the stochastic forward-propagation should be calculated, for dropout this corresponding to $\boldsymbol{x}^{(b)} \to \boldsymbol{z}^{(b)}$ above.

Using the skeleton `DropoutLayer` class definition below, implement the `fprop` and `bprop` methods. You may wish to store the binary mask used in the forward propagation as an attribute of the class for use in back-propagation - it is fine to assume that the `fprop` and `bprop` will always be called in sync.

```python
from mlp.layers import StochasticLayer

class DropoutLayer(StochasticLayer):
    """Layer which stochastically drops input dimensions in its output."""

    def __init__(self, rng=None, incl_prob=0.5, share_across_batch=True):
        """Construct a new dropout layer.

        Args:
            rng (RandomState): Seeded random number generator.
            incl_prob: Scalar value in (0, 1] specifying the probability of
                each input dimension being included in the output.
            share_across_batch: Whether to use same dropout mask across
                all inputs in a batch or use per input masks.
        """
        super(DropoutLayer, self).__init__(rng)
        assert incl_prob > 0. and incl_prob <= 1.
        self.incl_prob = incl_prob
        self.share_across_batch = share_across_batch

    def fprop(self, inputs, stochastic=True):
        """Forward propagates activations through the layer transformation.

        Args:
            inputs: Array of layer inputs of shape (batch_size, input_dim).
            stochastic: Flag allowing different deterministic
                forward-propagation mode in addition to default stochastic
                forward-propagation e.g. for use at test time. If False
                a deterministic forward-propagation transformation
                corresponding to the expected output of the stochastic
                forward-propagation is applied.

        Returns:
            outputs: Array of layer outputs of shape (batch_size, output_dim).
        """
        if stochastic:
            mask_shape = (1,) + inputs.shape[1:] if self.share_across_batch else inputs.shape
            self._mask = (rng.uniform(size=mask_shape) < self.incl_prob)
            return inputs * self._mask
        else:
            return inputs * self.incl_prob

    def bprop(self, inputs, outputs, grads_wrt_outputs):
        """Back propagates gradients through a layer.

        Given gradients with respect to the outputs of the layer calculates the
        gradients with respect to the layer inputs. This should correspond to
        default stochastic forward-propagation.

        Args:
            inputs: Array of layer inputs of shape (batch_size, input_dim).
            outputs: Array of layer outputs calculated in forward pass of
                shape (batch_size, output_dim).
            grads_wrt_outputs: Array of gradients with respect to the layer
                outputs of shape (batch_size, output_dim).

        Returns:
            Array of gradients with respect to the layer inputs of shape
            (batch_size, input_dim).
        """
        return grads_wrt_outputs * self._mask

    def __repr__(self):
        return 'DropoutLayer(incl_prob={0:.1f})'.format(self.incl_prob)
```

Test your implementation by running the cell below (if your implementation is incorrect you will get an `AssertionError` - look at what the assert statement is checking for a clue as to what is wrong).

```python
seed = 31102016
rng = np.random.RandomState(seed)
test_incl_probs = [0.1, 0.5, 0.7]
input_shape = (5, 10)
for incl_prob in test_incl_probs:
    layer = DropoutLayer(rng, incl_prob)
    inputs = rng.normal(size=input_shape)
    grads_wrt_outputs = rng.normal(size=input_shape)
    for t in range(100):
        outputs = layer.fprop(inputs, stochastic=True)
        # Check outputted array correct shape
        assert outputs.shape == inputs.shape
        # Check all outputs are either equal to inputs or zero
        assert np.all((outputs == inputs) | (outputs == 0))
        grads_wrt_inputs = layer.bprop(inputs, outputs, grads_wrt_outputs)
        # Check back-propagated gradients only non-zero for non-zero outputs
        assert np.all((outputs != 0) == (grads_wrt_inputs != 0))
        assert np.all(grads_wrt_outputs[outputs != 0] == grads_wrt_inputs[outputs != 0])
    det_outputs = layer.fprop(inputs, stochastic=False)
    # Check deterministic fprop outputs are correct shape
    assert det_outputs.shape == inputs.shape
    # Check deterministic fprop outputs scaled correctly
    assert np.allclose(det_outputs, incl_prob * inputs)
```

### Optional extension

Above we assumed the same dropout mask was applied to each input in a batch, as specified in the lecture slides. In practice sometimes a different mask is sampled for each input. As an extension you could try implementing this per-input form of dropout either by defining a new layer or adding an extra argument to the constructor of the above layer which allows you to switch between the two forms.

## Exercise 2: Training with dropout

Experiment with training models with dropout layers to classify MNIST digits. Code has been provided below as a starting point for setting up the model objects though feel free to use any additional adaptive learning rules or learning rule schedulers you wrote during the coursework instead. You may also wish to change the model architecture to use a larger model with more parameters in which the regularisation provided by dropout is likely to have a more pronounced effect. You will probably also find that models with dropout generally need to be trained over more epochs than those without (can you suggest why this might be?).

You should training with a few different `incl_prob` settings for the dropout layers and try to establish how the values chosen affect the training performance. You may wish to experiment with using a different dropout probability at the input than for the intermediate layers (why?).

You may wish to start reading through and implementing exercise 3 while waiting for training runs to complete.

```python
import numpy as np
import logging
from mlp.data_providers import MNISTDataProvider
from mlp.models import MultipleLayerModel
from mlp.layers import ReluLayer, AffineLayer
from mlp.errors import CrossEntropySoftmaxError
from mlp.initialisers import GlorotUniformInit, ConstantInit
from mlp.learning_rules import MomentumLearningRule
from mlp.optimisers import Optimiser
import matplotlib.pyplot as plt
%matplotlib inline

# Seed a random number generator
seed = 31102016
rng = np.random.RandomState(seed)

# Set up a logger object to print info about the training run to stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.handlers = [logging.StreamHandler()]

# Create data provider objects for the MNIST data set
train_data = MNISTDataProvider('train', batch_size=50, rng=rng)
valid_data = MNISTDataProvider('valid', batch_size=50, rng=rng)
```

```python
# Probability of input being included in output in dropout layer
incl_prob = 0.5

input_dim, output_dim, hidden_dim = 784, 10, 125

# Use Glorot initialisation scheme for weights and zero biases
weights_init = GlorotUniformInit(rng=rng, gain=2.**0.5)
biases_init = ConstantInit(0.)

# Create three affine layer model with rectified linear non-linearities
# and dropout layers before every affine layer
model = MultipleLayerModel([
    DropoutLayer(rng, incl_prob),
    AffineLayer(input_dim, hidden_dim, weights_init, biases_init),
    ReluLayer(),
    DropoutLayer(rng, incl_prob),
    AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init),
    ReluLayer(),
    DropoutLayer(rng, incl_prob),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

# Multiclass classification therefore use cross-entropy + softmax error
error = CrossEntropySoftmaxError()

# Use a momentum learning rule - you could use an adaptive learning rule
# implemented for the coursework here instead
learning_rule = MomentumLearningRule(0.02, 0.9)

# Monitor classification accuracy during training
data_monitors={'acc': lambda y, t: (y.argmax(-1) == t.argmax(-1)).mean()}

optimiser = Optimiser(
    model, error, learning_rule, train_data, valid_data, data_monitors, notebook=True)

num_epochs = 100
stats_interval = 5

stats, keys, run_time = optimiser.train(num_epochs=num_epochs, stats_interval=stats_interval)

# Plot the change in the validation and training set error over training.
fig_1 = plt.figure(figsize=(8, 4))
ax_1 = fig_1.add_subplot(111)
for k in ['error(train)', 'error(valid)']:
    ax_1.plot(np.arange(1, stats.shape[0]) * stats_interval,
              stats[1:, keys[k]], label=k)
ax_1.legend(loc=0)
ax_1.set_xlabel('Epoch number')

# Plot the change in the validation and training set accuracy over training.
fig_2 = plt.figure(figsize=(8, 4))
ax_2 = fig_2.add_subplot(111)
for k in ['acc(train)', 'acc(valid)']:
    ax_2.plot(np.arange(1, stats.shape[0]) * stats_interval,
              stats[1:, keys[k]], label=k)
ax_2.legend(loc=0)
ax_2.set_xlabel('Epoch number')
```

## Exercise 3: Implementing maxout

[Maxout](http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf) can be considered a generalisation of the rectified linear transformation implemented in the previous lab.

For a rectified linear (`Relu`) layer the forward propagation corresponds to

\begin{equation}
    y^{(b)}_k =
    \max\left\lbrace 0,\,x^{(b)}_k \right\rbrace
\end{equation}

i.e. each output corresponds to an pairwise maximum of a constant (0) and the input.

Instead of taking the maximum of the input and a constant, we could instead consider taking the maximum over sets of inputs of a fixed size $s$.

\begin{equation}
    y^{(b)}_k =
    \max\left\lbrace x^{(b)}_{(k-1)s + 1},\, x^{(b)}_{(k-1)s + 2},\, \dots ,\, x^{(b)}_{ks} \right\rbrace
\end{equation}

If these inputs $x^{(b)}_d$ are themselves the outputs of an affine layer, then this corresponds to taking the maximum of a series of affine functions of the previous layer outputs. Like a rectified linear layer this leads to piecewise linear input-output relationships (which have well-behaved gradients which do not suffer from the saturation problems of logistic sigmoid / hyperbolic tangent transformations) but unlike the rectified linear case we do not end force a portion of the outputs to be zero.

Experimentally this form of transformation has been found to give good performance, with the name *maxout* chosen because the *out*put is the *max*imum of a set of inputs. Maxout is also commonly used with dropout layers however note they are not directly related - maxout defines a deterministic non-linear transformation which can help improve the representational capacity and trainability of models; dropout defines a stochastic transformation which is mainly aimed at regularising a model to reduce overfitting.

Using layers which take the maximum of fixed sized sets of inputs is also a common technique in models with convolutional layers which we will cover later in the course, with here the layer commonly being termed a *max-pooling* layer (with there being natural generalisation to other choices of reduction functions over pools such as the mean). We will adopt this terminology here for a layer implementing the transformation described above and we will be able to reuse our code implementing this maximum operation when experimenting with convolutional models.

The partial derivatives of this max-pooling transformation are sparse (lots of values are zero), with only the partial derivative of the output of a pool with respect to the maximum input in the pool non-zero. This can be expressed as

\begin{equation}
    \frac{\partial y^{(b)}_k}{\partial x^{(b)}_d} =
    \begin{cases}
      1 & \quad (k-1)s + 1 \leq d  \leq ks \quad\textrm{and} &x^{(b)}_d = \max\left\lbrace x^{(b)}_{(k-1)s + 1},\, x^{(b)}_{(k-1)s + 2},\, \dots ,\, x^{(b)}_{ks} \right\rbrace \\
      0 & \quad \textrm{otherwise}
    \end{cases}.
\end{equation}

Using these definitions implement the `fprop` and `bprop` methods of the skeleton `MaxPoolingLayer` class below.

Some hints

  * One way of organising the inputs into non-overlapping pools is using the `numpy.reshape` function.
  * The `numpy.max` function has an `axis` argument which allows you specify the axis (dimension) of the input array to take the maximum over.
  * It may help to construct a binary mask corresponding to the definitions of the partial derivatives above to allow you to implement the `bprop` method.
  * As with the `DropoutLayer` it is fine to temporarily store values calculated in the `fprop` method as attributes of the object (e.g. `self.val = val`) to use in the `bprop` method (although you don't necessarily need to do this).

```python
from mlp.layers import Layer

class MaxPoolingLayer(Layer):

    def __init__(self, pool_size=2):
        """Construct a new max-pooling layer.

        Args:
            pool_size: Positive integer specifying size of pools over
               which to take maximum value. The outputs of the layer
               feeding in to this layer must have a dimension which
               is a multiple of this pool size such that the outputs
               can be split in to pools with no dimensions left over.
        """
        self.pool_size = pool_size

    def fprop(self, inputs):
        """Forward propagates activations through the layer transformation.

        This corresponds to taking the maximum over non-overlapping pools of
        inputs of a fixed size `pool_size`.

        Args:
            inputs: Array of layer inputs of shape (batch_size, input_dim).

        Returns:
            outputs: Array of layer outputs of shape (batch_size, output_dim).
        """
        assert inputs.shape[-1] % self.pool_size == 0, (
            'Last dimension of inputs must be multiple of pool size')
        pooled_inputs = inputs.reshape(
            inputs.shape[:-1] +
            (inputs.shape[-1] // self.pool_size, self.pool_size))
        pool_maxes = pooled_inputs.max(-1)
        self._mask = pooled_inputs == pool_maxes[..., None]
        return pool_maxes

    def bprop(self, inputs, outputs, grads_wrt_outputs):
        """Back propagates gradients through a layer.

        Given gradients with respect to the outputs of the layer calculates the
        gradients with respect to the layer inputs.

        Args:
            inputs: Array of layer inputs of shape (batch_size, input_dim).
            outputs: Array of layer outputs calculated in forward pass of
                shape (batch_size, output_dim).
            grads_wrt_outputs: Array of gradients with respect to the layer
                outputs of shape (batch_size, output_dim).

        Returns:
            Array of gradients with respect to the layer inputs of shape
            (batch_size, input_dim).
        """
        return (self._mask * grads_wrt_outputs[..., None]).reshape(inputs.shape)

    def __repr__(self):
        return 'MaxPoolingLayer(pool_size={0})'.format(self.pool_size)
```

Test your implementation by running the cell below.

```python
test_inputs = np.array([[-3, -4, 5, 8], [0, -2, 3, -8], [1, 5, 3, 2]])
test_outputs_1 = np.array([[8], [3], [5]])
test_grads_wrt_outputs_1 = np.array([[10], [5], [-3]])
test_grads_wrt_inputs_1 = np.array([[0, 0, 0, 10], [0, 0, 5, 0], [0, -3, 0, 0]])
test_outputs_2 = np.array([[-3, 8], [0, 3], [5, 3]])
test_grads_wrt_outputs_2 = np.array([[3, -1], [2, 5], [5, 3]])
test_grads_wrt_inputs_2 = np.array([[3, 0, 0, -1], [2, 0, 5, 0], [0, 5, 3, 0]])
layer_1 = MaxPoolingLayer(4)
layer_2 = MaxPoolingLayer(2)
# Check fprop with pool_size = 4
assert np.allclose(layer_1.fprop(test_inputs), test_outputs_1)
# Check bprop with pool_size = 4
assert np.allclose(
    layer_1.bprop(test_inputs, test_outputs_1, test_grads_wrt_outputs_1),
    test_grads_wrt_inputs_1
)
# Check fprop with pool_size = 2
assert np.allclose(layer_2.fprop(test_inputs), test_outputs_2)
# Check bprop with pool_size = 2
assert np.allclose(
    layer_2.bprop(test_inputs, test_outputs_2, test_grads_wrt_outputs_2),
    test_grads_wrt_inputs_2
)
```

## Exercise 4: Training with maxout

Use your `MaxPoolingLayer` implementation in a multiple layer models to experiment with how well maxout networks are able to classify MNIST digits. As with the dropout training exercise, code has been provided below as a starting point for setting up the model objects, but again feel free to substitute any components.

If you have time you may wish to experiment with training a model using a combination of maxout and dropout or another regularisation method covered in the last lab notebook.

```python
import numpy as np
import logging
from mlp.data_providers import MNISTDataProvider
from mlp.models import MultipleLayerModel
from mlp.layers import AffineLayer
from mlp.errors import CrossEntropySoftmaxError
from mlp.initialisers import GlorotUniformInit, ConstantInit
from mlp.learning_rules import MomentumLearningRule
from mlp.optimisers import Optimiser
import matplotlib.pyplot as plt
%matplotlib inline

# Seed a random number generator
seed = 31102016
rng = np.random.RandomState(seed)

# Set up a logger object to print info about the training run to stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.handlers = [logging.StreamHandler()]

# Create data provider objects for the MNIST data set
train_data = MNISTDataProvider('train', batch_size=50, rng=rng)
valid_data = MNISTDataProvider('valid', batch_size=50, rng=rng)
```

```python
# Size of pools to take maximum over
pool_size = 2

input_dim, output_dim, hidden_dim = 784, 10, 100

# Use Glorot initialisation scheme for weights and zero biases
weights_init = GlorotUniformInit(rng=rng)
biases_init = ConstantInit(0.)

# Create three affine layer model interleaved with max-pooling layers
model = MultipleLayerModel([
    AffineLayer(input_dim, hidden_dim * pool_size, weights_init, biases_init),
    MaxPoolingLayer(pool_size),
    AffineLayer(hidden_dim, hidden_dim * pool_size, weights_init, biases_init),
    MaxPoolingLayer(pool_size),
    AffineLayer(hidden_dim, output_dim, weights_init, biases_init)
])

# Multiclass classification therefore use cross-entropy + softmax error
error = CrossEntropySoftmaxError()

# Use a momentum learning rule - you could use an adaptive learning rule
# implemented for the coursework here instead
learning_rule = MomentumLearningRule(0.02, 0.9)

# Monitor classification accuracy during training
data_monitors={'acc': lambda y, t: (y.argmax(-1) == t.argmax(-1)).mean()}

optimiser = Optimiser(
    model, error, learning_rule, train_data, valid_data, data_monitors, notebook=True)

num_epochs = 100
stats_interval = 5

stats, keys, run_time = optimiser.train(num_epochs=num_epochs, stats_interval=stats_interval)

# Plot the change in the validation and training set error over training.
fig_1 = plt.figure(figsize=(8, 4))
ax_1 = fig_1.add_subplot(111)
for k in ['error(train)', 'error(valid)']:
    ax_1.plot(np.arange(1, stats.shape[0]) * stats_interval,
              stats[1:, keys[k]], label=k)
ax_1.legend(loc=0)
ax_1.set_xlabel('Epoch number')

# Plot the change in the validation and training set accuracy over training.
fig_2 = plt.figure(figsize=(8, 4))
ax_2 = fig_2.add_subplot(111)
for k in ['acc(train)', 'acc(valid)']:
    ax_2.plot(np.arange(1, stats.shape[0]) * stats_interval,
              stats[1:, keys[k]], label=k)
ax_2.legend(loc=0)
ax_2.set_xlabel('Epoch number')

```

```python

```