# Mollow triplets 2

```python
from functools import partial
import pdb

import pickle
import numpy as np
from scipy.optimize import minimize
from scipy.fftpack import fft, fftshift, fftfreq
from scipy.integrate import quad
from scipy.special import factorial, sinc
import matplotlib.pyplot as plt

import pysme.integrate as integ
import pysme.hierarchy as hier
import pysme.sparse_system_builder as ssb
import qinfo as qi
from qinfo import supops
```

```python
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
```

```python
# define Qubit operators
sx = np.array([[0, 1], [1, 0]], dtype=np.complex)
sy = np.array([[0, -1.j], [1.j, 0]], dtype=np.complex)
sz = np.array([[1, 0], [0, -1]], dtype=np.complex)
Id = np.eye(2, dtype=np.complex)
sp = (sx + 1.j * sy) / 2
sm = (sx - 1.j * sy) / 2
zero = np.zeros((2, 2), dtype=np.complex)
```

```python
plt.style.use('../paper.mplstyle')
```

A coherent drive adds a Hamiltonian term.
\begin{align}
    d\rho&=dt[\beta^*L-\beta L^\dagger,\rho]+dt\mathcal{D}[L]\rho \\
    &=-idt[i\beta^*L-i\beta L^\dagger,\rho]+dt\mathcal{D}[L]\rho \\
    H_{\text{eff}}&=i\beta^*L-i\beta L^\dagger
\end{align}
For $L=\sqrt{\gamma}\sigma_-$, the Rabi frequency is $\Omega=2\sqrt{\gamma}|\beta|$.

(Carmichæl notates the Rabi frequency as $2\Omega$, so in his notation $\Omega=\sqrt{\gamma}|\beta|$.

```python
def Heff_fn(beta, L):
    return 1.j*np.conj(beta)*L - 1.j*beta*L.conj().T
```

Try to reproduce plots from my thesis.

```python
def rect(x, a, b):
    return np.where(x < a, 0, np.where(x < b, 1, 0))

def xi_rect(t, a, b):
    return rect(t, a, b)/np.sqrt(b - a)
```

```python
gamma = 1
beta = 16.j
r = np.log(2)
mu = 0
T = 4
xi_fn = partial(xi_rect, a=0, b=T)
Omega = 2*np.sqrt(gamma)*np.abs(beta)
```

```python
Omega
```

```python
rho0 = (Id - sz) / 2
S = Id
L = np.sqrt(gamma)*sm
H = zero
```

Parameters indicating how precisely we compute the correlators.

```python
tau_final = 2**8
tau_samples = 2**12
taus = np.linspace(0, tau_final, tau_samples)
```

How long we evolve for to get steady-state.

```python
times_ss = np.linspace(0, 32, 2**14)
```

Derived quantities useful for figuring out the scale of the frequency-domain steps.

```python
frequencies = fftshift(fftfreq(taus.shape[0], np.diff(taus)[0]))
omegas = 2*np.pi*frequencies
d_omega = np.diff(omegas)[0]
Rabi_offset = 2*np.abs(beta) // d_omega
```

Functions for calculating the necessary correlation functions.

```python
def calc_white_auto_corr(L, beta, r, mu, times_ss, taus, solve_ivp_kwargs=None):
    N = np.sinh(r)**2
    M_sq = -np.exp(2.j * mu) * np.sinh(r) * np.cosh(r)
    integrator = integ.UncondGaussIntegrator(L, M_sq, N, Heff_fn(beta, L))
    soln_ss = integrator.integrate(Id/2, times_ss)

    rho_ss = soln_ss.get_density_matrices(np.s_[-1])
    sp_ss = np.trace(sp @ rho_ss)
    sm_ss = np.trace(sm @ rho_ss)

    L_0_taus = integrator.integrate_non_herm(rho_ss @ sp, taus, solve_ivp_kwargs=solve_ivp_kwargs)

    Expt_t_taus = L_0_taus.get_expectations(sm, hermitian=False)

    return rho_ss, Expt_t_taus - sp_ss * sm_ss

def calc_hier_auto_corr(xi_fn, L, r, mu, beta, m_max, taus, rho_ss, t, t0=0, timesteps=2**10,
                        solve_ivp_kwargs=None):
    sp_ss = np.trace(sp @ rho_ss)
    sm_ss = np.trace(sm @ rho_ss)
    Id_field = np.eye(m_max + 1, dtype=np.complex)
    factory = hier.HierarchyIntegratorFactory(2, m_max)
    integrator = factory.make_uncond_integrator(xi_fn, Id, L, Heff_fn(beta, L), r, mu)
    times = np.linspace(t0, t, timesteps)
    soln_t = integrator.integrate(rho_ss, times)
    sp_ss_t = soln_t.get_expectations(sp, vac_rho(m_max), idx_slice=np.s_[-1], hermitian=False)
    rho_ss_t = soln_t.get_hierarchy_density_matrices(np.s_[-1])
    L_t_t = rho_ss_t @ np.kron(sp, Id_field)
    L_t_taus = integrator.integrate_hier_init_cond(L_t_t, taus + t,
                                                   solve_ivp_kwargs=solve_ivp_kwargs)
    Expt_t_taus = L_t_taus.get_expectations(sm, vac_rho(m_max), hermitian=False)
    soln_t_taus = integrator.integrate_hier_init_cond(rho_ss_t, taus + t,
                                                      solve_ivp_kwargs=solve_ivp_kwargs)
    sm_ss_t_taus = soln_t_taus.get_expectations(sm, vac_rho(m_max), hermitian=False)
    # Subtract off a bunch of stuff that gets rid of the delta
    return (Expt_t_taus - sp_ss_t * sm_ss - sp_ss * sm_ss_t_taus + sp_ss * sm_ss,
            sp_ss, sm_ss, sp_ss_t, sm_ss_t_taus)

def calc_hier_auto_corr_fock(xi_fn, L, r, mu, beta, m_max, taus, rho_ss, t, t0=0, timesteps=2**10,
                             solve_ivp_kwargs=None):
    sp_ss = np.trace(sp @ rho_ss)
    sm_ss = np.trace(sm @ rho_ss)
    Id_field = np.eye(m_max + 1, dtype=np.complex)
    factory = hier.HierarchyIntegratorFactory(2, m_max)
    integrator = factory.make_uncond_integrator(xi_fn, Id, L, Heff_fn(beta, L), 0, 0)
    times = np.linspace(t0, t, timesteps)
    soln_t = integrator.integrate(rho_ss, times)
    sp_ss_t = soln_t.get_expectations(sp, sqz_rho(r, mu, m_max), idx_slice=np.s_[-1], hermitian=False)
    rho_ss_t = soln_t.get_hierarchy_density_matrices(np.s_[-1])
    L_t_t = rho_ss_t @ np.kron(sp, Id_field)
    L_t_taus = integrator.integrate_hier_init_cond(L_t_t, taus + t,
                                                   solve_ivp_kwargs=solve_ivp_kwargs)
    Expt_t_taus = L_t_taus.get_expectations(sm, sqz_rho(r, mu, m_max), hermitian=False)
    soln_t_taus = integrator.integrate_hier_init_cond(rho_ss_t, taus + t,
                                                      solve_ivp_kwargs=solve_ivp_kwargs)
    sm_ss_t_taus = soln_t_taus.get_expectations(sm, sqz_rho(r, mu, m_max), hermitian=False)
    # Subtract off a bunch of stuff that gets rid of the delta
    return (Expt_t_taus - sp_ss_t * sm_ss - sp_ss * sm_ss_t_taus + sp_ss * sm_ss,
            sp_ss, sm_ss, sp_ss_t, sm_ss_t_taus)
```

```python
def lam_mu(gamma_c, eps):
    lam = gamma_c + eps
    mu = gamma_c - eps
    return lam, mu

def N_degen_PA(omega, omega_A, lam, mu):
    return (lam**2 - mu**2)/4*(1/((omega - omega_A)**2 + mu**2)
                               - 1/((omega - omega_A)**2 + lam**2))

def mod_M_degen_PA(omega, omega_A, lam, mu):
    return (lam**2 - mu**2)/4*(1/((omega - omega_A)**2 + mu**2)
                               + 1/((omega - omega_A)**2 + lam**2))

def deltas_degen_PA(gamma_c, Omega, lam, mu):
    delta_mu = gamma_c*Omega*(lam**2 - mu**2)/(4*mu*(Omega**2 + mu**2))
    delta_lam = gamma_c*Omega*(lam**2 - mu**2)/(4*lam*(Omega**2 + mu**2))
    return delta_mu, delta_lam

def F_G(delta_mu, delta_lam, Phi):
    F_A = -(1j/4)*(delta_mu*(1 + np.cos(Phi))
                   - delta_lam*(1 - np.cos(Phi)))
    G_A = -(1/4)*(delta_mu + delta_lam)*np.sin(Phi)
    return F_A, G_A

def get_degen_PA_params(Omega, omega_A, omega_L, gamma_c, eps, phi_L, phi_s):
    lam, mu = lam_mu(gamma_c, eps)
    N_A = N_degen_PA(omega_A, omega_A, lam, mu)
    N_Om = N_degen_PA(omega_A + Omega, omega_A, lam, mu)
    mod_M_A = mod_M_degen_PA(omega_A, omega_A, lam, mu)
    mod_M_Om = mod_M_degen_PA(omega_A + Omega, omega_A, lam, mu)
    M_A = np.exp(2j*phi_s)*mod_M_A
    M_Om = np.exp(2j*phi_s)*mod_M_Om
    Delta_AL = omega_A - omega_L
    Phi = 2*phi_L - phi_s
    delta_mu, delta_lam = deltas_degen_PA(gamma_c, Omega, lam, mu)
    F_A, G_A = F_G(delta_mu, delta_lam, Phi)
    return N_A, N_Om, M_A, M_Om, Delta_AL, F_A, G_A

def calc_quasi_markoff_degen_PA_auto_corr(
        gamma, Omega, omega_A, omega_L, gamma_c, eps, phi_L, phi_s, times_ss, taus, solve_ivp_kwargs=None):
    N_A, N_Om, M_A, M_Om, Delta_AL, F_A, G_A = get_degen_PA_params(
        Omega, omega_A, omega_L, gamma_c, eps, phi_L, phi_s)
    integrator = integ.QuasiMarkoff2LvlIntegrator(
        gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A)
    soln_ss = integrator.integrate(Id/2, times_ss)

    rho_ss = soln_ss.get_density_matrices(np.s_[-1])
    sp_ss = np.trace(sp @ rho_ss)
    sm_ss = np.trace(sm @ rho_ss)

    L_0_taus = integrator.integrate_non_herm(rho_ss @ sp, taus, solve_ivp_kwargs=solve_ivp_kwargs)

    Expt_t_taus = L_0_taus.get_expectations(sm, hermitian=False)
    return rho_ss, Expt_t_taus - sp_ss * sm_ss
```

```python
def get_fluorescence_spectrum(auto_corr):
    fluorescence = fftshift(fft(auto_corr))
    return fluorescence
```

```python
def gen_save_load_data(data_gen_method, fname, overwrite=False):
    '''Get the data returned by the generating method, running the method only if the data isn't already available.

    If the given filename exists, load and return the data from that file. Otherwise generate the data using the
    supplied method and save and return it.

    Useful for notebooks you imagine running multiple times, but where some of the data is expensive to generate
    and you want to save it to disk to be reloaded for future sessions.

    '''
    try:
        with open(fname, 'xb' if not overwrite else 'wb') as f:
            data = data_gen_method()
            pickle.dump(data, f)
    except FileExistsError:
        print('Data already exist.')
        with open(fname, 'rb') as f:
            data = pickle.load(f)
    return data
```

```python
def gen_Expt_t_taus_wavepacket(
        xi_fn, L, r, mu, beta, m_max, taus, rho_ss_coh, t_final, solve_ivp_kwargs=None):
    '''Generate autocorrelation function for squeezed hierarchy

    Parameters
    ----------
    xi_fn : callable (float -> complex)
        Function returning the wavepacket amplitude at the evaluated time
    L : np.array
        Lindblad operator
    r : float
        Squeezing strength
    mu : float
        Squeezing angle
    beta : complex
        Drive amplitude
    m_max : int
        Hierarchy truncation level
    taus : np.array
        Times at which to evaluate the autocorrelation function
    rho_ss_coh : np.array
        The steady-state density matrix for a coherently driven atom
    t_final : float
        The fiducial time t to compute the auto correlation fn wrt: <sp(t)sm(t+tau)>
    solve_ivp_kwargs : dict
        Keyword arguments for the ``solve_ivp`` call made when integrating the hierarchy

    '''
    return calc_hier_auto_corr(xi_fn, L, r, mu, beta, m_max,
                               taus, rho_ss_coh, t=t_final,
                               solve_ivp_kwargs=solve_ivp_kwargs)
```

```python
def rho_from_ket(ket):
    return np.outer(ket, ket.conj())

def vac_rho(n):
    ket = np.zeros(n + 1, dtype=np.complex)
    ket[0] = 1
    return rho_from_ket(ket)

def make_squeezed_state_vec(r, mu, N, normalized=True):
    r'''Make a truncated squeezed-state vector.

    The squeezed-state vector is :math:`S(r,\mu)|0\rangle`. The truncated
    vector is renormalized by default.

    Parameters
    ----------
    N: positive integer
        The dimension of the truncated Hilbert space, basis {0, ..., N-1}
    r: real number
        Squeezing amplitude
    mu: real number
        Squeezing phase
    normalized: boolean
        Whether or not the truncated vector is renormalized

    Returns
    -------
    numpy.array
        Squeezed-state vector in the truncated Hilbert space, represented in the
        number basis

    '''
    ket = np.zeros(N, dtype=np.complex)
    for n in range(N//2):
        ket[2*n] = (1 / np.sqrt(np.cosh(r))) * ((-0.5 * np.exp(2.j * mu) * np.tanh(r))**n /
                                                factorial(n)) * np.sqrt(factorial(2 * n))
    return ket / np.linalg.norm(ket) if normalized else ket

def sqz_rho(r, mu, n):
    return rho_from_ket(make_squeezed_state_vec(r, mu, n + 1))
```

```python
rho_ss_coh, delta_Expt_t_taus_coh = calc_white_auto_corr(L, beta, 0, 0, times_ss, taus,
                                                         solve_ivp_kwargs={'rtol': 1e-6, 'atol': 1e-9})
```

```python
fluor_spec = get_fluorescence_spectrum(delta_Expt_t_taus_coh)
```

```python
fig, ax = plt.subplots()
ax.plot(omegas, np.abs(fluor_spec))
ax.set_xlabel(r'$(\omega-\Omega)/\Gamma$')
```

    Text(0.5, 0, '$(\\omega-\\Omega)/\\Gamma$')

```python
m_max = 12
t_final = 0.5
Expt_t_taus_wavepacket, _, _, _, _ = gen_save_load_data(partial(gen_Expt_t_taus_wavepacket, xi_fn=xi_fn, L=L, r=r,
                                                                mu=mu, beta=beta, m_max=m_max, taus=taus,
                                                                rho_ss_coh=rho_ss_coh, t_final=t_final,
                                                                solve_ivp_kwargs={'rtol': 1e-6, 'atol': 1e-9}),
                                                        '2020-06-25/Expt_t_taus_wavepacket.pickle')
```

    Data already exist.

```python
Expt_t_taus_wavepacket
```

    array([ 4.99320613e-01+0.j,  1.51640406e-01+0.j,  8.04227137e-02+0.j, ...,
           -1.32136987e-08+0.j, -4.13006597e-09+0.j,  1.75379286e-08+0.j])

```python
fluor_spec_wavepacket = get_fluorescence_spectrum(Expt_t_taus_wavepacket)
```

```python
fig, ax = plt.subplots()
ax.plot(omegas, np.abs(fluor_spec))
ax.plot(omegas, np.abs(fluor_spec_wavepacket))
ax.set_xlabel(r'$(\omega-\Omega)/\Gamma$')
```

    Text(0.5, 0, '$(\\omega-\\Omega)/\\Gamma$')

For a rectangular wavepacket
\begin{align}
    \xi_t
    &=
    \begin{cases}
    1/\sqrt{T} & 0\leq t\leq T
    \\
    0 & \text{otherwise}
    \end{cases}
\end{align}
we have
\begin{align}
    \langle b_\omega^\dagger b_{\omega^\prime}\rangle_{\gamma,\xi}
    &=
    \frac{4s^2}{T}e^{i(\omega^\prime-\omega)T/2}
    \frac{\sin(\omega^\prime T/2)\sin(\omega T/2)}{\omega\omega^\prime}
    \\
    &=
    s^2Te^{i(\omega^\prime-\omega)T/2}
    \frac{\sin(\omega^\prime T/2)}{\omega^\prime T/2}\frac{\sin(\omega T/2)}{\omega T/2}
    \\
    &=
    s^2Te^{i(\omega^\prime-\omega)T/2}\operatorname{sinc}(\omega^\prime T/2\pi)
    \operatorname{sinc}(\omega T/2\pi)
    \\
    \operatorname{sinc}(x)
    &=
    \frac{\sin(\pi x)}{\pi x}
\end{align}

```python
def rect_correlator(omega, omega_p, s, T):
    return (s**2*T*np.exp(1.j*(omega_p - omega)*T/2)
            *sinc(omega_p*T/(2*np.pi))*sinc(omega*T/(2*np.pi)))
```

```python
Omega, Omega_p = np.mgrid[-2*np.pi:2*np.pi:129j,-2*np.pi:2*np.pi:129j]
```

```python
correlations = rect_correlator(Omega, Omega_p, np.sinh(np.log(2)), 4)
fig, ax = plt.subplots()
ax.pcolormesh(Omega, Omega_p, np.abs(correlations), rasterized=True)
ax.set_aspect('equal')
```

```python
correlations = rect_correlator(Omega, Omega_p, np.sinh(np.log(2)), 1)
fig, ax = plt.subplots()
ax.pcolormesh(Omega, Omega_p, np.abs(correlations), rasterized=True)
ax.set_aspect('equal')
```

```python
class QuasiMarkoff2LvlIntegrator(integ.UncondLindbladIntegrator):
    def __init__(self, gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A):
        dim = 2
        basis = ssb.SparseBasis(dim)
        self.basis = basis
        sx = np.array([[0, 1], [1, 0]], dtype=np.complex)
        sy = np.array([[0, -1j], [1j, 0]], dtype=np.complex)
        sz = np.array([[1, 0], [0, -1]], dtype=np.complex)
        sp = (sx + 1j*sy)/2
        sm = (sx - 1j*sy)/2
        # Eqn 16 and 17 in [YB96]
        Sm = sm*np.exp(-1j*phi_L)
        Sp = sp*np.exp(1j*phi_L)
        Y = 1j*sz@(Sp + Sm)
        sz_vec = self.basis.vectorize(sz)
        Sm_vec = self.basis.vectorize(Sm)
        Sp_vec = self.basis.vectorize(Sp)
        Y_vec = self.basis.vectorize(Y)
        H_vec = ((Omega/2 + 1j*F_A)*(Sp_vec + Sm_vec)
                 + (Delta_AL/2)*sz_vec + G_A*Y_vec)

        self.Q = -gamma/4*(
                (1 + N_A)*(basis.make_double_comm_matrix(Sm_vec, 1)
                           - 2*basis.make_diff_op_matrix(Sm_vec))
                + N_A*(basis.make_double_comm_matrix(Sp_vec, 1)
                       - 2*basis.make_diff_op_matrix(Sp_vec))
                + (1 + N_Om)*(-basis.make_double_comm_matrix(Sm_vec, 1)
                            - 2*basis.make_diff_op_matrix(Sm_vec))
                + N_Om*(-basis.make_double_comm_matrix(Sp_vec, 1)
                        - 2*basis.make_diff_op_matrix(Sp_vec))
                - 2*basis.make_double_comm_matrix(Sm_vec, M_A*np.exp(-2j*phi_L))
                + 2*basis.make_real_comm_matrix(M_A*np.exp(-2j*phi_L)*Sp_vec,
                                                Sp_vec)
                + 2*basis.make_real_comm_matrix(np.conjugate(M_A)*np.exp(2j*phi_L)
                                                *Sm_vec, Sm_vec)
                - 2*basis.make_double_comm_matrix(Sm_vec,
                                                  M_Om*np.exp(-2j*phi_L))
                - 2*basis.make_real_comm_matrix(M_Om*np.exp(-2j*phi_L)*Sp_vec,
                                                Sp_vec)
                - 2*basis.make_real_comm_matrix(np.conjugate(M_Om)*np.exp(2j*phi_L)
                                                *Sm_vec, Sm_vec))
        self.Q += basis.make_hamil_comm_matrix(H_vec)
        self.Q += -2*basis.make_real_sand_matrix(F_A*(Sp_vec - Sm_vec), sz_vec)
        self.Q += -2*basis.make_real_sand_matrix(G_A*(Sp_vec + Sm_vec), sz_vec)
```

```python
def get_quasi_markoff_ss_slow(gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A, times_ss):
    integrator = QuasiMarkoff2LvlIntegrator(
        gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A)
    soln_ss = integrator.integrate(Id/2, times_ss)

    rho_ss = soln_ss.get_density_matrices(np.s_[-1])
    return rho_ss
```

```python
def get_quasi_markoff_ss(gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A):
    phi_s = 0.5*np.angle(M_A) # Assumes the angle is independent of frequency
    Phi = 2*phi_L - phi_s
    gamma_Omega = gamma*(N_Om + np.abs(M_Om)*np.cos(Phi))
    gamma_0 = gamma*(N_A - np.abs(M_A)*np.cos(Phi) + 1/2)
    A = np.array([[-gamma_Omega, gamma*np.abs(M_Om)*np.sin(Phi), -2*G_A],
                  [gamma*np.abs(M_A)*np.sin(Phi), -gamma_0, 0.5*Omega + 2.j*F_A],
                  [0, -2*Omega, -gamma_0 - gamma_Omega]])
    bloch_vec_ss = np.linalg.inv(A)@np.array([0, 0, gamma])

    sx = np.array([[0, 1], [1, 0]], dtype=np.complex)
    sy = np.array([[0, -1j], [1j, 0]], dtype=np.complex)
    sz = np.array([[1, 0], [0, -1]], dtype=np.complex)
    rho_ss = 0.5*(np.eye(2) + bloch_vec_ss[0]*sx + bloch_vec_ss[1]*sy + bloch_vec_ss[2]*sz)
    return rho_ss
```

```python
def integrator_to_lind_proc_tensor(integrator, t=0):
    if isinstance(integrator.basis, ssb.SparseBasis):
        op_basis = qi.OperatorBasis(integrator.basis.basis.todense())
    else:
        op_basis = qi.OperatorBasis(integrator.basis.basis)
    lind_proc_mat = integrator.jac(t, rho_vec=None)
    lind_process = lambda rho: supops.act_proc_mat(rho, lind_proc_mat, op_basis)
    lind_proc_tensor = supops.process_to_proc_tensor(lind_process, op_basis.vec_dim)
    return lind_proc_tensor

def get_lind_degen_choi_mat(lind_proc_tensor):
    dim = lind_proc_tensor.shape[0]
    Id_proc_tensor = supops.get_identity_proc_tensor(dim)
    Id_choi_mat = supops.proc_tensor_to_choi_mat(Id_proc_tensor)
    Id_choi_mat_degen_proj = (np.eye(Id_choi_mat.shape[0], dtype=Id_choi_mat.dtype)
                              - qi.rho_from_ket(np.linalg.eigh(Id_choi_mat)[1][:,-1]))
    lind_choi_mat = supops.proc_tensor_to_choi_mat(lind_proc_tensor)
    lind_degen_choi_mat = Id_choi_mat_degen_proj @ lind_choi_mat @ Id_choi_mat_degen_proj
    return lind_degen_choi_mat
```

```python
gamma = 1
r_A = np.log(2)
r_Om = np.log(2)/2
phi_s = 0
Nth_A = 0
Nth_Om = 0
N_A = (2*Nth_A + 1)*np.sinh(r_A)**2 + Nth_A
N_Om = (2*Nth_Om + 1)*np.sinh(r_Om)**2 + Nth_Om
M_A = -(2*Nth_A + 1)*np.exp(2j*phi_s)*np.sinh(r_A)*np.cosh(r_A)
M_Om = -(2*Nth_Om + 1)*np.exp(2j*phi_s)*np.sinh(r_Om)*np.cosh(r_Om)
F_A = 0.1j
G_A = .05
Delta_AL = 0
Omega = 8
phi_L = 0
integrator = QuasiMarkoff2LvlIntegrator(
                gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A)
```

```python
lind_proc_tensor = integrator_to_lind_proc_tensor(integrator)
```

```python
degen_choi_mat = get_lind_degen_choi_mat(lind_proc_tensor)
```

```python
np.round(degen_choi_mat, 4)
```

    array([[-0.   +0.j , -0.05 -0.1j, -0.05 +0.1j,  0.   +0.j ],
           [-0.05 +0.1j,  1.625+0.j ,  0.875+0.j ,  0.05 -0.1j],
           [-0.05 -0.1j,  0.875+0.j ,  0.625+0.j ,  0.05 +0.1j],
           [-0.   +0.j ,  0.05 +0.1j,  0.05 -0.1j,  0.   +0.j ]])

```python
np.linalg.eigvalsh(degen_choi_mat)
```

    array([-1.48401169e-01, -8.87579035e-18,  2.59968716e-01,  2.13843245e+00])

Looks like we get negativity fairly easily as a result of arbitrary `F_A` values.

```python
get_quasi_markoff_ss_slow(1., 1., 1., 1, 1, 1., 1., 0., 0., 0., np.linspace(0, 2**6, 2**12))
```

    array([[ 0.37837838+0.j        , -0.05405405-0.13513513j],
           [-0.05405405+0.13513513j,  0.62162162+0.j        ]])

```python
get_quasi_markoff_ss(1., 1., 1., 1, 1, 1., 1., 0., 0., 0.)
```

    array([[0.38888889+0.j        , 0.        +0.11111111j],
           [0.        -0.11111111j, 0.61111111+0.j        ]])

I seem to have not quite got this part right...

```python
def calc_quasi_markoff_auto_corr(
        gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A, times_ss, taus, solve_ivp_kwargs=None):
    integrator = QuasiMarkoff2LvlIntegrator(
        gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A)
    soln_ss = integrator.integrate(Id/2, times_ss)

    rho_ss = soln_ss.get_density_matrices(np.s_[-1])
    sp_ss = np.trace(sp @ rho_ss)
    sm_ss = np.trace(sm @ rho_ss)

    L_0_taus = integrator.integrate_non_herm(rho_ss @ sp, taus, solve_ivp_kwargs=solve_ivp_kwargs)

    Expt_t_taus = L_0_taus.get_expectations(sm, hermitian=False)
    return rho_ss, Expt_t_taus - sp_ss * sm_ss
```

```python
def get_quasi_fluors(
        T, gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A, times_ss, taus, solve_ivp_kwargs=None):
    _, Expt_t_taus = calc_quasi_markoff_auto_corr(gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A, times_ss, taus, solve_ivp_kwargs)
    fluors = get_fluorescences([Expt_t_taus], T)[1]
    return fluors
```

```python
def get_quasi_cost_fn(target_fluor, T, gamma, beta, omega_A, omega_L, times_ss, taus, solve_ivp_kwargs=None):
    Omega = 2*np.abs(beta)
    phi_L = np.angle(beta)
    def calculate_cost(x):
        gamma_c, eps, phi_s = x
        quasi_fluor = get_quasi_fluors(T, gamma, Omega, omega_A, omega_L, gamma_c, eps, phi_L, phi_s, times_ss, taus)[0]
        return np.sum((quasi_fluor - target_fluor)**2)
    return calculate_cost
```

```python

```

Could try and make things faster by precomputing some stuff that doesn't change when tweaking parameters.

```python
class QuasiMarkoff2LvlIntegratorFactory(UncondLindbladIntegrator):
        def __init__(self, gamma, N_A, N_Om, M_A, M_Om, Delta_AL, Omega, phi_L, F_A, G_A):
        dim = 2
        basis = ssb.SparseBasis(dim)
        self.basis = basis
        sx = np.array([[0, 1], [1, 0]], dtype=np.complex)
        sy = np.array([[0, -1j], [1j, 0]], dtype=np.complex)
        sz = np.array([[1, 0], [0, -1]], dtype=np.complex)
        sp = (sx + 1j*sy)/2
        sm = (sx - 1j*sy)/2
        # Eqn 16 and 17 in [YB96]
        Sm = sm*np.exp(-1j*phi_L)
        Sp = sp*np.exp(1j*phi_L)
        Y = 1j*sz@(Sp + Sm)
        sz_vec = self.basis.vectorize(sz)
        Sm_vec = self.basis.vectorize(Sm)
        Sp_vec = self.basis.vectorize(Sp)
        Y_vec = self.basis.vectorize(Y)
        H_vec = ((Omega/2 + 1j*F_A)*(Sp_vec + Sm_vec)
                 + (Delta_AL/2)*sz_vec + G_A*Y_vec)

        self.N_A_1_op = (basis.make_double_comm_matrix(Sm_vec, 1)
                         - 2*basis.make_diff_op_matrix(Sm_vec))
        self.N_A_op = (basis.make_double_comm_matrix(Sp_vec, 1)
                       - 2*basis.make_diff_op_matrix(Sp_vec))
        self.N_Om_1_op = (-basis.make_double_comm_matrix(Sm_vec, 1)
                          - 2*basis.make_diff_op_matrix(Sm_vec))
        self.N_Om_op = (-basis.make_double_comm_matrix(Sp_vec, 1)
                        - 2*basis.make_diff_op_matrix(Sp_vec))

        self.Q = -gamma/4*(
                (1 + N_A)*self.N_A_1_op
                + N_A*self.N_A_op
                + (1 + N_Om)*self.N_Om_1_op
                + N_Om*self.N_Om_op
                - 2*basis.make_double_comm_matrix(Sm_vec, M_A*np.exp(-2j*phi_L))
                + 2*basis.make_real_comm_matrix(M_A*np.exp(-2j*phi_L)*Sp_vec,
                                                Sp_vec)
                + 2*basis.make_real_comm_matrix(M_A.conj()*np.exp(2j*phi_L)
                                                *Sm_vec, Sm_vec)
                - 2*basis.make_double_comm_matrix(Sm_vec,
                                                  M_Om*np.exp(-2j*phi_L))
                - 2*basis.make_real_comm_matrix(M_Om*np.exp(-2j*phi_L)*Sp_vec,
                                                Sp_vec)
                - 2*basis.make_real_comm_matrix(M_Om.conj()*np.exp(2j*phi_L)
                                                *Sm_vec, Sm_vec))
        self.Q += basis.make_hamil_comm_matrix(H_vec)
        self.Q += -2*basis.make_real_sand_matrix(F_A*(Sp_vec - Sm_vec), sz_vec)
        self.Q += -2*basis.make_real_sand_matrix(G_A*(Sp_vec + Sm_vec), sz_vec)
```


```python
from comet_ml.api import API
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from scipy.stats import entropy
from sympy.ntheory import factorint
from sympy.utilities.iterables import multiset_partitions
from pathlib import Path

comet_api = API(api_key="hRTMB66l59BMb9pPrRzLrbzwp")
```

```python
experiments = comet_api.get(workspace="tensorized-rnn", project_name="speaker-verification")

def fetch_val_eer(experiments, cores_result_dict):
    for exp in experiments:
        tags = exp.get_tags()
        tag = tags[-1]

        if tag in cores_result_dict.keys():
            # best validation eer is the last one recorded.
            best_val_eer_dict = exp.get_metrics("best_eer")[-1]
            best_val_eer = float(best_val_eer_dict['metricValue'])
            # best_val_eer_step = int(best_val_eer_dict['step'])

            cores_result_dict[tag].append(best_val_eer)
    print(cores_result_dict)

    # average results of experiment repeats.
    cores_avg_result_dict = {}
    for tag, eer_list in cores_result_dict.items():
        print("Tag: {}. No of experiments: {}".format(tag, len(eer_list)))
        cores_avg_result_dict[tag] = round(np.mean(eer_list) * 100, 2)

    return cores_avg_result_dict

def safely_get_parameter(api_exp, param_name):
    param_list = api_exp.get_parameters_summary(param_name)
    if len(param_list) == 0:
        raise ValueError("parameter '{}' was not found in experiment '{}'"
                         .format(param_name, api_exp.get_metadata()["experimentKey"]))
    return param_list["valueCurrent"]

###----------FUNCTIONS from t3nsor library for getting auto shape of a tensor-train cores--------####
def _get_all_factors(n, d=3, mode='ascending'):
    p = _factorint2(n)
    if len(p) < d:
        p = p + [1, ] * (d - len(p))

    if mode == 'ascending':
        def prepr(x):
            return tuple(sorted([np.prod(_) for _ in x]))
    elif mode == 'descending':
        def prepr(x):
            return tuple(sorted([np.prod(_) for _ in x], reverse=True))

    elif mode == 'mixed':
        def prepr(x):
            x = sorted(np.prod(_) for _ in x)
            N = len(x)
            xf, xl = x[:N//2], x[N//2:]
            return tuple(_roundrobin(xf, xl))

    else:
        raise ValueError('Wrong mode specified, only {} are available'.format(MODES))

    raw_factors = multiset_partitions(p, d)
    clean_factors = [prepr(f) for f in raw_factors]
    clean_factors = list(set(clean_factors))
    return clean_factors

def _to_list(p):
    res = []
    for k, v in p.items():
        res += [k, ] * v
    return res

def _factorint2(p):
    return _to_list(factorint(p))

def auto_shape(n, d=3, criterion='entropy', mode='ascending'):
    factors = _get_all_factors(n, d=d, mode=mode)
    if criterion == 'entropy':
        weights = [entropy(f) for f in factors]
    elif criterion == 'var':
        weights = [-np.var(f) for f in factors]
    else:
        raise ValueError('Wrong criterion specified, only {} are available'.format(CRITERIONS))

    i = np.argmax(weights)
    return list(factors[i])
###----------FUNCTIONS from t3nsor library for getting auto shape of a tensor-train cores--------####

def lstm_num_params(num_layers, hidden_size, input_size, bias=True, n_gates_and_cell=4):
    result = 0

    # layer 1
    result = n_gates_and_cell * ((input_size + hidden_size + int(bias)) * hidden_size)

    # for higher layers, input_size == hidden_size
    if num_layers > 1:
        num_layers_minus_1 = num_layers - 1
        result += num_layers_minus_1 * n_gates_and_cell * ((hidden_size + hidden_size + int(bias)) * hidden_size)

    return result

def tt_linear_num_params(in_size, out_size, n_cores, rank, bias=True):
#     print('in: {}, out: {}, rank: {}, cores: {}'.format(in_size, out_size, rank, n_cores))
    in_core_shape = auto_shape(in_size, n_cores)
    out_core_shape = auto_shape(out_size, n_cores)

    # number of parameters in a tt linear layer.
    assert len(in_core_shape) == len(out_core_shape)
    n_cores = len(in_core_shape)

    result = 0
    out_size = 1
    for i in range(n_cores):
        if i == 0 or i == n_cores - 1:
            result += rank * (in_core_shape[i]) * out_core_shape[i]
        else:
            result += rank**2 * (in_core_shape[i]) * out_core_shape[i]
        out_size *= out_core_shape[i]

    if bias:
        result += out_size

#     print('linear output ', result)
    return result

def tt_lstm_num_params(num_layers, in_size, hidden_size, n_cores, rank, bias=True, n_gates_and_cell = 4):
    result = 0

    # layer 1
    result = n_gates_and_cell * \
                (tt_linear_num_params(in_size, hidden_size, n_cores, rank, False) \
                    + tt_linear_num_params(hidden_size, hidden_size, n_cores, rank, bias))

    # for higher layers, input_size == hidden_size
    if num_layers > 1:
        num_layers_minus_1 = num_layers - 1
        result += num_layers_minus_1 * n_gates_and_cell * \
                (tt_linear_num_params(hidden_size, hidden_size, n_cores, rank, False) \
                    + tt_linear_num_params(hidden_size, hidden_size, n_cores, rank, bias))

    return result

def tt_lstm_num_params_with_concattrick(num_layers, in_size, hidden_size, n_cores, rank, bias=True, n_gates_and_cell = 4):
    # number of parameters in a tt lstm that uses concatenation trick.
    result = 0

    # layer 1
    result = (tt_linear_num_params(in_size, hidden_size * n_gates_and_cell, n_cores, rank, False)
                      + tt_linear_num_params(hidden_size, hidden_size * n_gates_and_cell, n_cores, rank, bias))

    # for higher layers, input_size == hidden_size
    if num_layers > 1:
        num_layers_minus_1 = num_layers - 1
        result += num_layers_minus_1 * (tt_linear_num_params(hidden_size, hidden_size * n_gates_and_cell, n_cores, rank, False)
                      + tt_linear_num_params(hidden_size, hidden_size * n_gates_and_cell, n_cores, rank, bias))

    return result
```

# Model size and parameters

```python
hidden_sizes = [256, 512, 768]
num_layers = [1, 2, 3]
input_size = 4096
emb_size = 256
n_gates = 4

data = []
for hidden_size in hidden_sizes:
    row = []
    for layer_size in num_layers:
        num_linear_layer = ((hidden_size + 1) * emb_size)
        num_lstm_layers = lstm_num_params(layer_size, hidden_size, input_size, n_gates_and_cell=n_gates)
        row.append(num_lstm_layers + num_linear_layer)
    data.append(row)
result = pd.DataFrame(data, index=hidden_sizes, columns=num_layers)
print("No of parameters")
print(result)
```

    No of parameters
                1         2         3
    256   4523264   5048576   5573888
    512   9570560  11669760  13768960
    768  15142144  19863808  24585472

## Evaluating no of parameters in TT-LSTM

```python
input_size = 4096
emb_size = 256
n_cores_list = [2,3,4]
ranks_list = [1, 2, 3, 4, 5]
n_gates = 4

def print_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list):
    data = []
    for i in range(len(n_cores_list)):
        n_cores = n_cores_list[i]
        row = []
        for rank in ranks_list:
            num_lstm_layers = tt_lstm_num_params_with_concattrick(num_layers, input_size, hidden_size,
                                                          n_cores, rank, bias=True, n_gates_and_cell=n_gates)
            num_linear_layer = tt_linear_num_params(hidden_size, emb_size, n_cores, rank, bias=True)
            row.append(num_lstm_layers + num_linear_layer)
        data.append(row)
    data = pd.DataFrame(data, index=n_cores_list, columns=ranks_list)
    return data

#---------------
hidden_size = 768
num_layers = 1
print("No of parameters ({}/{})".format(num_layers, hidden_size))
print(print_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list))
print("")

#---------------
hidden_size = 256
num_layers = 3
print("No of parameters ({}/{})".format(num_layers, hidden_size))
print(print_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list))
print("")

#---------------
hidden_size = 512
num_layers = 2
print("No of parameters ({}/{})".format(num_layers, hidden_size))
print(print_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list))
print("")

hidden_size = 512
num_layers = 1
print("No of parameters ({}/{})".format(num_layers, hidden_size))
print(print_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list))
print("")
```

    No of parameters (1/768)
           1      2      3      4      5
    2  14592  25856  37120  48384  59648
    3   4640   6848   9952  13952  18848
    4   3824   4816   6304   8288  10768

    No of parameters (3/256)
           1      2      3      4      5
    2  13056  22784  32512  42240  51968
    3   5104   7904  11728  16576  22448
    4   4064   5536   7744  10688  14368

    No of parameters (2/512)
           1      2      3      4      5
    2  18944  33536  48128  62720  77312
    3   6112   9280  13856  19840  27232
    4   5088   6528   8672  11520  15072

    No of parameters (1/512)
           1      2      3      4      5
    2  11776  21248  30720  40192  49664
    3   3424   5440   8352  12160  16864
    4   2752   3648   4992   6784   9024

## Evaluating no of parameters in Naive TT-LSTM

```python
input_size = 4096
emb_size = 256
n_cores_list = [2,3,4]
ranks_list = [1, 2, 3, 4, 5]
n_gates = 4

def print_naive_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list):
    data = []
    for i in range(len(n_cores_list)):
        n_cores = n_cores_list[i]
        row = []
        for rank in ranks_list:
            num_lstm_layers = tt_lstm_num_params(num_layers, input_size, hidden_size,
                                                          n_cores, rank, bias=True, n_gates_and_cell=n_gates)
            num_linear_layer = tt_linear_num_params(hidden_size, emb_size, n_cores, rank, bias=True)
            row.append(num_lstm_layers + num_linear_layer)
        data.append(row)
    data = pd.DataFrame(data, index=n_cores_list, columns=ranks_list)
    return data

#---------------
hidden_size = 512
num_layers = 1
print("No of parameters ({}/{})".format(num_layers, hidden_size))
print(print_naive_tt_lstm_options(num_layers, hidden_size, input_size, emb_size, n_cores_list, ranks_list))
print("")
```

    No of parameters (1/512)
           1      2      3      4      5
    2  20480  38656  56832  75008  93184
    3   4768   8896  14688  22144  31264
    4   3472   5472   8304  11968  16464

# Evaluating no of parameters in linear layer using low-rank factorisation.

```python
def low_rank_params(in_features, out_features, rank, bias):
    return ((in_features + int(bias)) * rank) + ((rank + int(bias)) * out_features)

in_size = 256
out_size = 256
ranks = 2 ** np.arange(1,7)

print("No of parameters (uncompressed):", 256 * 256)
print("Rank\tNum of Parms")
for rank in ranks:
    print("{}\t{}".format(rank, low_rank_params(in_size, out_size, rank, True)))
```

    No of parameters (uncompressed): 65536
    Rank	Num of Parms
    2	1282
    4	2308
    8	4360
    16	8464
    32	16672
    64	33088

# Plot experiment result

```python
# dict with "key: n-cores", "value: dict of (key: rank, value: test_eer)"
# experiments_result_dict = {2: {1: 0, 2: 0, 4: 0},
#                           3: {1: 0, 2: 0, 4: 0},
#                           4: {1: 0, 2: 0, 4: 0}}
experiments_result_dict = {2: {1: 0, 2: 0, 3:0, 4: 0},
                          3: {1: 0, 2: 0, 3:0, 4: 0},
                          4: {1: 0, 2: 0, 3:0, 4: 0}}
exp_seed = 11

for exp in experiments:
    compression = exp.get_parameters_summary("compression")["valueCurrent"]
    if not compression == 'tt' : continue

    try:
        n_cores = int(safely_get_parameter(exp, "n_cores"))
        tt_rank = int(safely_get_parameter(exp, "rank"))
        seed = int(safely_get_parameter(exp, "seed"))
        best_val_eer = float(exp.get_metrics("test_EER")[-1]['metricValue'])
    except Exception as e:
        print(e)
        continue

    if n_cores not in experiments_result_dict \
            or tt_rank not in experiments_result_dict[n_cores] \
                or seed != exp_seed:
        print("Skipping experiment: compression={}, n_cores={}, tt_rank={}, seed={}".format(compression, n_cores, tt_rank, seed))
        continue
    experiments_result_dict[n_cores][tt_rank] = best_val_eer
print(experiments_result_dict)

```

    Skipping experiment: compression=tt, n_cores=2, tt_rank=5, seed=11
    Skipping experiment: compression=tt, n_cores=4, tt_rank=5, seed=11
    Skipping experiment: compression=tt, n_cores=3, tt_rank=5, seed=11
    {2: {1: 0.047115384616034435, 2: 0.04344512195077787, 3: 0.06874999999999927, 4: 0.06213942307541739}, 3: {1: 0.06093749999999991, 2: 0.053125000000000026, 3: 0.05468749999999987, 4: 0.053846153846863}, 4: {1: 0.05337284482744496, 2: 0.05576923076880846, 3: 0.05312499999999996, 4: 0.04278846153918008}}

```python
%matplotlib qt

plt.figure(dpi=100)
input_size = 40
emb_size = 256
hidden_size = 768
num_layers = 1

# plot uncompressed experiment.
plt.plot(2682112, 0.07333 * 100, "D", label="LSTM", color='black', markersize=8)

# # plot low-rank experiments
# lr_param_counts = [low_rank_params(in_size, out_size, rank, True) for rank in [8, 16, 32, 64]]
# print("LR params: ", lr_param_counts)
# plt.plot(lr_param_counts, [.0647, 0.0587, 0.0433, 0.0511], "-.D", label="low-rank")

# plot tt experiments.
for k, v in experiments_result_dict.items():
    param_counts = []
    for rank in v.keys():
        num_lstm_layers = tt_lstm_num_params_with_concattrick(num_layers, input_size, hidden_size, k, rank, bias=True)
        num_linear_layer = tt_linear_num_params(hidden_size, emb_size, k, rank, bias=True)

        param_counts.append(num_lstm_layers + num_linear_layer)
    print("TT-{}cores params: {}".format(k, param_counts))

    # plot all rank values for the current core, k.
    shape_str = " $x$ ".join([str(s) for s in auto_shape(hidden_size, k)])
    values_in_perc = [val * 100 for val in v.values()]
    plt.plot(param_counts, values_in_perc, "-o", label="TT-LSTM " + shape_str, linewidth=3, markersize=8)
plt.xscale('log')
plt.xlabel('# parameters', fontsize=14)
plt.ylabel('EER (%)', fontsize=14)
plt.ylim([0,20])
plt.xticks(fontsize=13)
plt.yticks(fontsize=13)
plt.legend(fontsize=13, loc='upper center')
plt.show()
```

```python
# plot the val EER for each rank.
ranks_result_dict = {'tt-cores4-rank2':[], 'tt-cores4-rank3':[], 'tt-cores4-rank4':[], 'tt-cores4-rank5':[],
                     'tt-cores4-rank6':[], 'tt-cores4-rank8':[]}
ranks_avg_result_dict = fetch_val_eer(experiments, ranks_result_dict)
plt.figure()
plt.plot(np.arange(2,8), ranks_avg_result_dict.values(), '-o')
plt.xlabel('rank')
plt.ylabel('val EER')
plt.title('no of cores = 4')
plt.show()
```

# Visualising saved spectograms.

```python
frame_fpath = r'D:\Users\Charley\Documents\Esperanza\repos\_experiments\speech-model-compression\speaker-verification\_librispeech_train-clean-100_tisv\LibriSpeech_train-clean-100_103\1240_103-1240-0000.npy'
frame_fpath = r'D:\Users\Charley\Documents\Esperanza\repos\_experiments\speech-model-compression\speaker-verification\_librispeech_train-clean-100_tisv\LibriSpeech_train-clean-100_1183\128659_1183-128659-0000.npy'
frames = np.load(frame_fpath).transpose()
frames.shape
plt.stem(y)
plt.show()

# import librosa
# import IPython.display as ipd
# wav_path = '777-126732-0006.wav'
# y, sr = librosa.load(wav_path)
# D = librosa.amplitude_to_db(np.abs(librosa.stft(y)), ref=np.max)
# librosa.display.specshow(D, y_axis='linear')
# plt.colorbar(format='%+2.0f dB')
```

# Learning curves and Data efficiency

```python
from scipy.ndimage import median_filter

def get_learning_curve(exp_id, metric_name, step_limit=20000):
    myexp = comet_api.get_experiment_by_id(exp_id)

    train_eer = []
    train_step = []
    for dic in myexp.get_metrics(metric_name):
        cur_step = int(dic['step'])
        cur_train_eer = float(dic['metricValue'])
        if cur_step > step_limit:
            break
        train_eer.append(cur_train_eer)
        train_step.append(cur_step)

    return train_step, train_eer

def smooth_array(my_array, size=100):
    s_array = median_filter(my_array,  size=size)
    return s_array

lstm_train_steps, lstm_train_eer = get_learning_curve("445596754f414da8bef0377f7f0906d3", "train_EER")
lstm_val_steps, lstm_val_eer = get_learning_curve("445596754f414da8bef0377f7f0906d3", "val_EER")
ttlstm_train_steps, ttlstm_train_eer = get_learning_curve("d6530fa27f11416298efa377b89b22ed", "train_EER")
ttlstm_val_steps, ttlstm_val_eer = get_learning_curve("d6530fa27f11416298efa377b89b22ed", "val_EER")

# smooth curves
lstm_train_eer, lstm_val_eer = smooth_array(lstm_train_eer, 150), smooth_array(lstm_val_eer, 10)
ttlstm_train_eer, ttlstm_val_eer = smooth_array(ttlstm_train_eer, 150), smooth_array(ttlstm_val_eer, 10)

%matplotlib qt
plt.figure(figsize=(20,6))

plt.subplot(1,2,1)
plt.plot(np.array(lstm_train_steps)/1000, np.array(lstm_train_eer) * 100, 'm-', label='LSTM (Train)' )
plt.plot(np.array(lstm_val_steps)/1000, np.array(lstm_val_eer) * 100, 'm--', label='LSTM (Validation)' )
plt.plot(np.array(ttlstm_train_steps)/1000, np.array(ttlstm_train_eer) * 100, 'b-', label='TT-LSTM (Train)' )
plt.plot(np.array(ttlstm_val_steps)/1000, np.array(ttlstm_val_eer) * 100, 'b--', label='TT-LSTM (Validation)' )
plt.ylabel('EER (%)', fontsize=14)
plt.xlabel('Training steps (K)', fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.yscale('log')
plt.legend(fontsize=14)

# Results of data efficency experiment.
de_percs = np.array([0.2, 0.4, 0.6, 0.8, 1.0]) * 100
de_lstm = np.array([0.1289, 0.1027, .0856, .0791, .0733]) * 100
de_ttlstm = np.array([.1047, .0717, .0504, .0654, .0434]) * 100

plt.subplot(1,2,2)
plt.plot(de_percs, de_lstm, 'md-', label='LSTM')
plt.plot(de_percs, de_ttlstm, 'bd-', label='TT-LSTM')
plt.xlabel('% of training data used', fontsize=14)
plt.ylabel('Test EER (%)', fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.legend(fontsize=14)
plt.tight_layout(pad=3.0, w_pad=20.0)
plt.show()
```

# Gradients distribution

```python
experiments = comet_api.get(workspace="tensorized-rnn", project_name="speaker-verification-loggrad")
n_steps = 1000

def fetch_grads(exp_id, metric_name, step_limit=1000):
    myexp = comet_api.get_experiment_by_id(exp_id)

    gradients = []
    for dic in myexp.get_metrics(metric_name):
        if len(gradients) >= step_limit:
            break
        gradients.append(float(dic['metricValue']))

    return gradients

nott_grads = np.array(fetch_grads("3545bf372e404dcf94fc57c9b9df3edf", "grad_norm"))
tt_grads = np.array(fetch_grads("3c9ada04c04c47fb91bd45c8fe7c7112", "grad_norm"))
tt_grads[tt_grads > 8] = 8

# nottweights = np.ones_like(nott_grads) / len(nott_grads)
# ttweights = np.ones_like(tt_grads) / len(tt_grads)

fig, ax1 = plt.subplots()

color = 'orchid'
ax1.set_xlabel('Gradient Value', fontsize=14)
ax1.set_ylabel('Counts - LSTM', fontsize=14)
ax1.tick_params(axis='y')
ax1.hist(nott_grads, bins=50, alpha=0.7, density=False, color=color, label='LSTM')
ax1.legend(loc='upper left', fontsize=14)

ax2 = ax1.twinx()
color = 'tab:blue'
ax2.hist(tt_grads, bins=100, alpha=0.7, density=False, label='TT-LSTM')
ax2.set_ylabel('Counts - TT-LSTM', fontsize=14)
ax2.legend(fontsize=14)
plt.show()
```

```python

```

```python

```


$$V(X) = -w (X) + \frac{w L}{2}$$

Where:

* $X = x-x_0$
* $x$ - is relative to the end of the beam
* $x_0$ - is the absolute position

$$V(X) = -w X + \frac{w L}{2}$$

$$M(X) = -\frac{w}{2} X^2 + \frac{w L}{2} X$$

$$M(X) = \frac{w}{2}\left(-X^2  + L X\right)$$

$$\theta(X) = \frac{-w}{2 E I}\left(-\frac{X^3}{3}  +L \frac{X^2}{2} - C\right)$$

$$\theta(X) = \frac{w}{2 E I}\left(\frac{X^3}{3} - L \frac{X^2}{2} + C\right)$$

$$\Delta(X) = \frac{w}{2 E I}\left(\frac{X^4}{12} - L \frac{X^3}{6} +C X + D\right)$$

$$\Delta(0) = \frac{w}{2 E I}\left(\frac{0^4}{12} - L \frac{0^3}{6} +C\cdot 0 + D\right) = y_0$$

$$\frac{w}{2 E I} D = y_0$$

$$\text{therefore:}\quad D = \frac{ y_0 2 E  I}{w}$$

$$\Delta(L) = \frac{w}{2 E I}\left(\frac{L^4}{12} - L \frac{L^3}{6} +C \cdot L + \frac{2 y_0 E  I}{w}\right) = y_0$$

$$\frac{w}{2 E I}\left(\frac{L^4}{12} - L \frac{L^3}{6} +C \cdot L\right) + y_0 = y_0$$

$$ \frac{w}{2 E I}\left(\frac{L^4}{12} - L \frac{L^3}{6} +C \cdot L \right) = 0$$

$$ \frac{L^4}{12} - L \frac{L^3}{6} +C \cdot L  = 0$$

$$ \frac{L^3}{12} -  \frac{L^3}{6} +C   = 0$$

$$C = \frac{L^3}{6} - \frac{L^3}{12}$$

$$C = \frac{L^3}{12}$$

$$\Delta(X) = \frac{w}{2 E I}\left(\frac{X^4}{12} - L \frac{X^3}{6} +\frac{L^3}{12} X + \frac{2 y_0 E  I}{w}\right)$$

$$X = x-x_0$$

$$\Delta(x) = \frac{w}{2 E I}\left(\frac{(x-x_0)^4}{12} - L \frac{(x-x_0)^3}{6} +\frac{L^3}{12} (x-x_0) + \frac{2 y_0 E  I}{w}\right)$$

```python
from sympy import symbols, collect, expand, latex, simplify
D, x, x_0, E, I, w, y_0, L, y, t = symbols('Delta x x_0 E I w y_0 L y t')
from sympy import init_printing
init_printing(use_unicode=True)

D = w/(2*E*I)*((x-x_0)**4/12-L*(x-x_0)**3/6+L**3/12*(x-x_0)+2*y_0*E*I/w)
D
```

```python
E = expand(D)
E
```

```python
F = collect(E,x)
F
```

```python
latex(F)
```

    'x^{3} \\left(- \\frac{L w}{12 E I} - \\frac{w x_{0}}{6 E I}\\right) + x^{2} \\left(\\frac{L w x_{0}}{4 E I} + \\frac{w x_{0}^{2}}{4 E I}\\right) + x \\left(\\frac{L^{3} w}{24 E I} - \\frac{L w x_{0}^{2}}{4 E I} - \\frac{w x_{0}^{3}}{6 E I}\\right) + y_{0} - \\frac{L^{3} w x_{0}}{24 E I} + \\frac{L w x_{0}^{3}}{12 E I} + \\frac{w x^{4}}{24 E I} + \\frac{w x_{0}^{4}}{24 E I}'

$$\Delta(x) = x^{3} \left(- \frac{L w}{12 E I} - \frac{w x_{0}}{6 E I}\right) + x^{2} \left(\frac{L w x_{0}}{4 E I} + \frac{w x_{0}^{2}}{4 E I}\right) + x \left(\frac{L^{3} w}{24 E I} - \frac{L w x_{0}^{2}}{4 E I} - \frac{w x_{0}^{3}}{6 E I}\right) + y_{0} - \frac{L^{3} w x_{0}}{24 E I} + \frac{L w x_{0}^{3}}{12 E I} + \frac{w x^{4}}{24 E I} + \frac{w x_{0}^{4}}{24 E I}$$

$$\Delta(x) = \frac{w}{2 EI}\left(\frac{x^{4}}{12} - \left(\frac{L}{6 } + \frac{x_{0}}{3}\right)x^{3}  + \left(\frac{L  x_{0}}{2} + \frac{x_{0}^{2}}{2}\right) x^{2}  + \left(\frac{L^{3}}{12} - \frac{L x_{0}^{2}}{2} - \frac{x_{0}^{3}}{3}\right) x \right)  +\frac{w}{2 E I}\left(- \frac{L^{3}  x_{0}}{12} + \frac{L x_{0}^{3}}{6}  + \frac{ x_{0}^{4}}{12}\right)+ y_{0}$$

$$\Delta(x) = \frac{w}{2 EI}\left(\frac{x^{4}}{12} - \frac{1}{3}\left(\frac{L}{2} + x_{0}\right)x^{3}  + \frac{1}{2}\left(L  x_{0} + x_{0}^{2}\right) x^{2}  + \left(\frac{L^{3}}{12} - \frac{L x_{0}^{2}}{2} - \frac{x_{0}^{3}}{3}\right) x \right)  +\frac{w}{2 E I}\frac{1}{6}\left(- \frac{L^{3}  x_{0}}{2} + L x_{0}^{3}  + \frac{ x_{0}^{4}}{2}\right)+ y_{0}$$

##The following was not used due to another approach being found.

The following calculations were done in an effort to determine what the loacation of the supports needed to be if they did not move in the drawing when the beam deflects.  This is necessary due to the simplifiaction made where the ends of the beam centerline do not move when the bem deflects.

```python
G = simplify(F)
G
```

$$\Delta(x)=\frac{1}{24 E I} \left(24 E I y_{0} - L^{3} w x_{0} + 2 L w x_{0}^{3} + w x^{4} - 2 w x^{3} \left(L + 2 x_{0}\right) + 6 w x^{2} x_{0} \left(L + x_{0}\right) - w x \left(- L^{3} + 6 L x_{0}^{2} + 4 x_{0}^{3}\right) + w x_{0}^{4}\right)$$

$$\theta(x) =\frac{1}{24 E I} \left(4 w x^{3} - 6 w x^{2} \left(L + 2 x_{0}\right) + 12 w x x_{0} \left(L + x_{0}\right) - w  \left(- L^{3} + 6 L x_{0}^{2} + 4 x_{0}^{3}\right) \right)$$

$$\theta(x) = \frac{1}{24 E I} \left(4 w x^{3} - 6 w x^{2} \left(L + 2 x_{0}\right) + 12 w x x_{0} \left(L + x_{0}\right) - w  \left(- L^{3} + 6 L x_{0}^{2} + 4 x_{0}^{3}\right) \right)$$

$$y_s = y_0 +(y - y_0) +\sqrt{\frac{t^2}{4} - (x-x_0)^2}$$

$$y_s = \frac{-1}{\theta(x)}(x_0-x) + y$$

$$ 0 = y_0 +(y - y_0) +\sqrt{\frac{t^2}{4} - (x-x_0)^2} - \left( \frac{-1}{\theta(x)}(x_0-x) + y \right)$$

$$ 0 = y_0 +y - y_0 +\sqrt{\frac{t^2}{4} - (x-x_0)^2} + \frac{1}{\theta(x)}(x_0-x) - y $$

$$ 0 = \sqrt{\frac{t^2}{4} - (x-x_0)^2} + \frac{x_0-x}{\theta(x)} $$

```python
from sympy import diff,sqrt, roots
T = diff(G,x)
T
```

```python
H = sqrt(t**2/4-(x-x_0)**2)*T #+ (x-x_0)
J = H**2
```

```python
K = collect(expand((J - (x_0-x)**2)/(w**2/(12*E**2*I**2))),x)
K
```

```python
c = -12*E**2/w**2*I**2*x_0**2 + L**6*t**2/192 - L**6*x_0**2/48 -L**4*t**2/16*x_0**2\
    +L**4*x_0**4/4 - L**3*t**2/24*x_0**3 + L**3*x_0**5/6 + 3*L**2/16*t**2*x_0**4 \
    -3*L**2/4*x_0**6 + L*t**2/4*x_0**5 - L*x_0**7 + t**2*x_0**6/12 - x_0**8/3
K-c
```


# Introduction to $\LaTeX$ for Jupyter notebooks

*(Need to update this for discrete math. Especially, $\LaTex$ symbols for logic.)

```python
# Standard workhorses
import numpy as np
import matplotlib.pyplot as plt

# Seaborn, useful for graphics
import seaborn as sns

# Magic function to make matplotlib inline; other style specs must come AFTER
%matplotlib inline

# This enables SVG graphics inline (only use with static plots (non-Bokeh))
%config InlineBackend.figure_formats = {'svg',}

# JB's favorite Seaborn settings for notebooks
rc = {'lines.linewidth': 2,
      'axes.labelsize': 18,
      'axes.titlesize': 18,
      'axes.facecolor': 'DFDFE5'}
sns.set_context('notebook', rc=rc)
sns.set_style('darkgrid', rc=rc)
```

In this tutorial, you will learn some of the basics on how to use $\LaTeX$ to display equations in Jupyter notebooks.  For looking up symbols you may need, you can use any of the many cheat sheets you can find by asking Google.

## Basic inline Latex
To embed Latex within text, simply encapsulate the Latex portions in dollar signs (`$`).  MathJax takes care of the rest.  As an example, consider the sentence below and the markdown/Latex code to render it.

>Einstein told us that $E = mc^2$.
>
>`Einstein told us that $E = mc^2$.`

Notice how the equation is properly rendered, with mathematical variables in italics.  Not also how `^2` was used to exponentiate.  If the exponential has more than one character in it, it should be enclosed in braces (`{}`).  In fact, braces are used to generally group symbols in Latex.

>Euler told us that $\mathrm{e}^{i \pi} - 1 = 0$.
>
>`Euler told us that $\mathrm{e}^{i \pi} - 1 = 0$.`

Aside from the grouping braces, there are several other syntactical items of note.  First, notice that I made the special character $\pi$ with `\pi`.  In general, a forward slash precedes special symbols or commands in Latex.  If we want another Greek letter, like $\theta$, we use `\theta`.  Now, also note that I used "`\mathrm{e}`" for the base of the natural logarithm.  I was signaling to Latex that I wanted the character written in Roman font, and not italics, so I used `\mathrm`.  Anything in the braces following the function `\mathrm` is rendered in Roman font.  Note the difference.

>This is $e$.  This is $\mathrm{e}$.
>
>`This is $e$.  This is $\mathrm{e}$.`

Now, back to grouping things in braces.  We can do similar groupings using braces with with subscripts.

>The dot product of two $n$-vectors is $\mathbf{a} \cdot \mathbf{b} = \sum_{i=1}^n a_i b_i$.
>
>`The dot product of two $n$-vectors is $\mathbf{a} \cdot \mathbf{b} = \sum_{i=1}^n a_i b_i$.`

Here, I have used `$\mathbf{a}$` to make the character `a` boldface, denoting a vector.  Note that we denote subscripts with an underscore.  Notice also that the bounds of the sum use the same underscore and caret notation as for subscripts and superscripts.

## Displaying equations on separate lines
The bounds on the summation in the above example may look a little funny to you because they are not above and below the summation symbol.  This is because this particular equation is written inline.  If we had separated it from the text, it renders differently.

>We can make an equation appear centered on a new line, like
>
\begin{align}
\mathbf{a} \cdot \mathbf{b} = \sum_{i=1}^n a_i b_i.
\end{align}
>
>`We can make an equation appear centered on a new line, like `
>
>`\begin{align}
\mathbf{a} \cdot \mathbf{b} = \sum_{i=1}^n a_i b_i.
\end{align}`

The `align` environment in Latex specifies that you want centered equations, separated from the text.  It is called `align` because it allows you to align the equations.  You separate lines in the equations with a double backslash (`//`).  Insert an ampersand (`&`) in each line at the alignment point.  All equations will be aligned at the location of the ampersand symbols (and, of course, the ampersands will not appear in the rendered equations).

>For a three-vector consisting of $x$, $y$, and $z$ components,
>
\begin{align}
\mathbf{a} \cdot \mathbf{b} &= \sum_{i=1}^n a_i b_i \\
&= a_x b_x + a_y b_y + a_z b_z.
\end{align}
>
>`For a three-vector consisting of $x$, $y$, and $z$ components,`
>
>`\begin{align}
\mathbf{a} \cdot \mathbf{b} &= \sum_{i=1}^n a_i b_i \\
&= a_x b_x + a_y b_y + a_z b_z.
\end{align}`

Note that I always put an extra blank line before the `\begin{align}` statement.  This is not necessary, but I think things look better with the extra space.

## Fractions (and an example of fine-tuning)
To display fractional quantities, we use the `\frac{}{}` command.  `\frac` is always followed by two sets of braces; the numerator is contained in the first, and the denominator is contained in the second.  As an example, we can write an equation you will become intimately familiar with,

>\begin{align}
P(A \mid B) = \frac{P(B \mid A) \, P(A)}{P(B)}
\end{align}
>
>`\begin{align}
P(A \mid B) = \frac{P(B \mid A) \, P(A)}{P(B)}
\end{align}`

The right hand side has a nicely-formatted fraction.  I did a little extra fine-tuning in this equation.  I'll show the equation again without the fine-tuning, which used the `\mid` and `\,` commands.

>\begin{align}
P(A | B) = \frac{P(B | A) P(A)}{P(B)}.
\end{align}
>
>`\begin{align}
P(A | B) = \frac{P(B | A) P(A)}{P(B)}.
\end{align}`

First, the `\mid` command should be used in conditional probabilities.  Just using a vertical bar (`|`) results in crowding.  Similarly, I used the `\,` command to insert a little extra space between the two probabilities in the numerator.  This makes the equation a bit easier to read.  This `\,` operator is especially important when defining integrals.  We can put a little space between the $\mathrm{d}x$ and the integrand.

>\begin{align}
\text{good: } &\int_0^{2\pi} \mathrm{d}x \, \sin x. \\[1em]
\text{bad: } &\int_0^{2\pi} \mathrm{d}x \sin x.
\end{align}
>
`\begin{align}
\text{good: } &\int_0^{2\pi} \mathrm{d}x \, \sin x. \\[1em]
\text{bad: } &\int_0^{2\pi} \mathrm{d}x \sin x.
\end{align}`

Note that I inserted extra space after the new line.  Specifically, `\\[1em]` instructs Latex to insert a space equation to the width of an M character between the equations.  I often do this to keep things clear.

It is also very important to note that I used $\sin$ and not $sin$.  Mathematical functions should be in Roman font and are invoked with a backslash.  Otherwise, the characters are interpreted as separate variables.  To be clear:

>\begin{align}
\text{good: } &\sin x. \\[1em]
\text{bad: } & sin x.
\end{align}
>
`\begin{align}
\text{good: } &\sin x. \\[1em]
\text{bad: } & sin x.
\end{align}`

Finally, notice that I was able to put text in the equation like this: `\text{good: }`.

## Grouping operators (and more fine-tuning)
Compare the following equations.

>\begin{align}
\text{good: } &\sum_{i=1}^n i^3
               = \left(\sum_{i=1}^n i\right)^2. \\[1em]
\text{bad: }  &\sum_{i=1}^n i^3 = (\sum_{i=1}^n i)^2.
\end{align}
>
>`\begin{align}
\text{good: } &\sum_{i=1}^n i^3
               = \left(\sum_{i=1}^n i\right)^2. \\[1em]
\text{bad: }  &\sum_{i=1}^n i^3 = (\sum_{i=1}^n i)^2.
\end{align}`

In the second equation, I did not use the `\left(` and `\right)` construction for parentheses and the result looks pretty awful.  In Latex, the height of anything that is encapsulated by `\left(` and `\right)` scales the parentheses appropriately.  You can use `\left` and `\right` with many symbols.  An important example is `\left\{`.  Note that to display braces in an equation, you have to use `\{` because just a plain brace (`{`) has a different meaning.

(By the way, that equation is true, and pretty amazing.  It says that the sum of the first $n$ *cubes* of integers is equal to the sum of the first $n$ integers *squared*!)

Finally, if you use `\left.` or `\right.`, Latex will simply scale the opposite symbol to match the height of the text, but will suppress printing the other.  For example,

>\begin{align}
\left. \frac{1}{x + 2} \right|_0^2 = -\frac{1}{4}.
\end{align}
>
>`\begin{align}
\left. \frac{1}{x + 2} \right|_0^2 = -\frac{1}{4}.
\end{align}`

This is also useful if you are going to use `/` for a division operation.  Compare the following.

>\begin{align}
\text{good: } & \left. x^2 \middle/ y^2 \right. \\[1em]
\text{bad: } & x^2 / y^2
\end{align}
>
>`\begin{align}
\text{good: } & \left. x^2 \middle/ y^2 \right. \\[1em]
\text{bad: } & x^2 / y^2
\end{align}`

Here, we used the `\middle` operator to scale the length of the division sign.

## Matrices and arrays
On occasion, you'll need to express matrices.  This is most easily done using the `pmatrix` environment.  For example, a covariance matrix for two variables might be written as

>\begin{align}
\sigma^2 = \begin{pmatrix}
\sigma_1^2 & \sigma_{12}^2 \\
\sigma_{12}^2 & \sigma_2^2
\end{pmatrix}.
\end{align}
>
>`\begin{align}
\sigma^2 = \begin{pmatrix}
\sigma_1^2 & \sigma_{12}^2 \\
\sigma_{12}^2 & \sigma_2^2
\end{pmatrix}.
\end{align}`

Once in the `pmatrix` environment, each row has entries separated by an ampersand.  The row ends with a `\\`.  Each row must have the same number of entries.

You may also need to represent an values stacked on top of each other.  For example, we might specify a Jeffreys prior like this.

>\begin{align}
P(\sigma) = \left\{
\begin{array}{cl}
\sigma^{-1} & \sigma > 0 \\
0 & \sigma \le 0.
\end{array}
\right.
\end{align}
>
>`\begin{align}
P(\sigma) = \left\{
\begin{array}{cl}
\sigma^{-1} & \sigma > 0 \\
0 & \sigma \le 0.
\end{array}
\right.
\end{align}`

The `array` environment allow arrays of text.  The `{cl}` after `\begin{array}` indicates that two columns are wanted, with the first column being centered and the second being left-aligned.  If we chose instead `{lr}`, the first column is left-aligned and the second column is right-aligned.

## Using Latex in figures
Fortunately, Matplotlib allows use of Latex in axis labels and other text in figures.  You simply need to include the dollar signs in your string.

```python
# Make a plot of JB's favorite function
x = np.linspace(0.0, 2.0 * np.pi, 200)
y = np.exp(np.sin(x))
plt.plot(x, y)
plt.xlabel('$x$')
plt.ylabel('$\mathrm{e}^{\sin x}$')
```

Unfortunately, Bokeh does not yet have Latex support, but will in the near future.  So, for now, you will have to avoid using Latex in the text in figures if you want them to be interactive.

## Useful Latex symbols
Following is a list of some symbols you may find useful in this class.

Latex | symbol
--- | :---:
`\approx` | $\approx$
`\sim` | $\sim$
`\propto` | $\propto$
`\le` | $\le$
`ge` | $\ge$
`\pm` | $\pm$
`\in` | $\in$
`\ln` | $\ln$
`\exp` | $\exp$
`\prod_{i\in D}` | ${\displaystyle \prod_{i\in D}}$
`\sum_{i\in D}` | ${\displaystyle \sum_{i\in D}}$
`\frac{\partial f}{\partial x}` | ${\displaystyle \frac{\partial f}{\partial x}}$
`\sqrt{x}` | $\sqrt{x}$
`\bar{x}` | $\bar{x}$
`\hat{x}` | $\hat{x}$
`\langle x \rangle` | $\langle x \rangle$
`\left\langle \frac{x}{y} \right\rangle` | $\left\langle \frac{x}{y} \right\rangle$

```python

```