# Martingales and the characteristic functions of absorption time on bipartite graphs

### Travis Monk, International Centre for Neuromorphic Systems, The MARCS Institute, Western Sydney University

This iPython notebook is freely available for all commercial and noncommercial use.  It accompanies the manuscript of the same title.  It reproduces all of the figures in the paper and saves them as pdf files.

One of those figures plots simulation results and compares them with theoretical analysis.  This notebook also runs simulations of the Moran process on the bipartite graph and saves them as .npz files to be loaded and plotted later.  If the .npz files exist in the same folder as this notebook, then the simulations do not run to expedite execution of this notebook.  To run simulations, simply delete or move the .npz files to another folder.

The preamble loads standard requisite packages and defines functions that will be called later.

The function 'fg' calculates the functions f(h) and g(h) that yield a product martingale.  The function 'psi' calculates the conditional characteristic functions of the number of mutant population size changes.  The function 'sims' runs simulations of the Moran process on the complete bipartite graph and saves the results as a .npz file.

```python
import numpy as np
from matplotlib import pyplot as plt
from sympy import *
from sympy.solvers.solveset import linsolve
from matplotlib.lines import Line2D
from os import path

plt.rcParams.update({"text.usetex": True,'font.size': 16})

def fg(A,B,r,h):

    AA = (r/A + B/A**2)
    BB = -(r/A + 1/B)*(1./h + B/(A*r*h)) - B*h/A**2 + h/B
    CC = (r/A + 1/B)*B/(A*r)

    f1 = (-BB + np.sqrt(BB**2 - 4*AA*CC))/(2*AA)
    f2 = (-BB - np.sqrt(BB**2 - 4*AA*CC))/(2*AA)

    g1 = (r/B + 1/A - 1/A*h*f1**(-1))*B/(r*h)
    g2 = (r/B + 1/A - 1/A*h*f2**(-1))*B/(r*h)

    return f1, f2, g1, g2

def psi(A,B,r,Sa0,Sb0,h):

    #calculate fixation probability.  get roots when g = 1:
    f0 = (A + B*r)/(r*(A*r + B)); g0 = (A*r + B)/(r*(A + B*r))

    #fixation probability is:
    p = (f0**Sa0*g0**Sb0 - 1)/(f0**A*g0**B - 1)

    #get the roots for general h:
    f1, f2, g1, g2 = fg(A,B,r,h)

    #calculate fixation CFs
    psia = (f1**Sa0*g1**Sb0 - f2**Sa0*g2**Sb0)/(p*(f1**A*g1**B - f2**A*g2**B))
    psib = (f2**Sa0*g2**Sb0*f1**A*g1**B - f1**Sa0*g1**Sb0*f2**A*g2**B)/((1-p)*(f1**A*g1**B - f2**A*g2**B))

    return psia, psib, p

def sims(A,B,r,Sa0,Sb0,tau,trials):

    #store results
    S_fix = 0; C_fix = []; S_ext = 0; C_ext = [];  T_fix = []; T_ext = []

    #define absorbing barriers
    fix = np.array([A,B]); ext = np.array([0,0])

    for i in range(trials):

        #reset initial conditions
        S = np.array([Sa0,Sb0]); C = 0; T = 0

        while (not np.array_equal(S,fix) and not np.array_equal(S,ext)):

            #draw a random number
            draw = np.random.uniform()

            #assemble probabilities that translate the random number to change in graph
            #total fitness
            F = r*(S[0] + S[1]) + (A-S[0]) + (B-S[1])

            #transition probabilities
            pbu = r*S[0]*(B-S[1])/(F*B); pbd = (A-S[0])*S[1]/(F*B)
            pau = r*S[1]*(A-S[0])/(F*A); pad = (B-S[1])*S[0]/(F*A)
            trans = np.cumsum([pbu,pbd,pau,pad,1-pbu-pbd-pau-pad])

            #find which transition you picked
            j = next(x[0] for x in enumerate(trans) if x[1] > draw)

            #change in graph and update clocks
            Xa = 1*(j==2) - 1*(j==3)
            Xb = 1*(j==0) - 1*(j==1)
            C += (j != 4); T += 1

            #update graph
            S[0] += Xa; S[1] += Xb

        #store results
        if np.array_equal(S,fix):
            S_fix += 1; C_fix.append(C); T_fix.append(T)
        if np.array_equal(S,ext):
            S_ext += 1; C_ext.append(C); T_ext.append(T)

    #Fourier transforms of results
    if len(C_fix) > 0:
        cfasim = np.array([np.mean(np.exp(np.array(C_fix)*j)) for j in tau])
        cfaT = np.array([np.mean(np.exp(np.array(T_fix)*j)) for j in tau])
    else:
        cfasim = 0; cfaT = []
    if len(C_ext) > 0:
        cfbsim = np.array([np.mean(np.exp(np.array(C_ext)*j)) for j in tau])
        cfbT = np.array([np.mean(np.exp(np.array(T_ext)*j)) for j in tau])
    else:
        cfbsim = 0; cfbT = []

    return S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext
```

Fig. 1 is a schematic of our problem and notation.

```python
fig, ax = plt.subplots(figsize=(12,6))

x_plot = np.array([-.15,.05,.08,.3,.31,.54]); y_plot = np.array([.8,.3,.8,.8,.3,.8])
size = [.04,.06,.04,.06,.04,.04]; color = ['r','b','b','r','r','b']
ind = np.array([0,2,3,5])

x_fix = np.array([1.1,1.2,1.2,1.3,1.3,1.4]); y_fix = np.array([.8,.6,.8,.8,.6,.8])
y_ext = np.array([.4,.2,.4,.4,.2,.4])

#state change graph
[ax.add_artist(Line2D([i,.05],[.8,.3],color='k',zorder=0,linewidth = 8 if i==.3 else 1)) for i in x_plot[ind]];
[ax.add_artist(Line2D([i,.31],[.8,.3],color='k',zorder=0)) for i in x_plot[ind]];
[ax.add_artist(plt.Circle((i[0],i[1]), (i[2]), color=i[3])) for i in zip(x_plot,y_plot,size,color)];

#fixation graph
[ax.add_artist(Line2D([i,1.2],[.8,.6],color='k',zorder=0)) for i in x_fix[ind]];
[ax.add_artist(Line2D([i,1.3],[.8,.6],color='k',zorder=0)) for i in x_fix[ind]];
[ax.add_artist(plt.Circle((i[0],i[1]), (.04), color='r')) for i in zip(x_fix,y_fix)];

#extinction graph
[ax.add_artist(Line2D([i,1.2],[.4,.2],color='k',zorder=0)) for i in x_fix[ind]];
[ax.add_artist(Line2D([i,1.3],[.4,.2],color='k',zorder=0)) for i in x_fix[ind]];
[ax.add_artist(plt.Circle((i[0],i[1]), (.04), color='b')) for i in zip(x_fix,y_ext)];

tri = np.array([[.05, .32], [.05, .39], [.12, .35]])
triangle = plt.Polygon(tri,color='k',zorder=10)
plt.gca().add_patch(triangle)

plt.text(-.45,.76,r'$A=4$',fontsize=22)
plt.text(-.45,.28,r'$B=2$',fontsize=22)
plt.text(.55,.58,r'$\mathbf{S_{t-1}} = [2,1]$',fontsize=22)
plt.text(.56,.48,r'$\mathbf{X_{t}} = [0,1]$',fontsize=22)
plt.text(1.03,.9,r'$\mathbf{S_T} = \mathbf{a} = [A,B]$',fontsize=22)
plt.text(1.05,.07,r'$\mathbf{S_T} = \mathbf{b} = [0,0]$',fontsize=22)

ax.arrow(.9, .5, 0.15, 0.15, fc='gray', ec='gray', head_width=0.05, head_length=0.05)
ax.arrow(.9, .5, 0.15, -0.25, fc='gray', ec='gray', head_width=0.05, head_length=0.05)

ax.set_xlim([-.5,1.5]); ax.set_ylim([0,1])
ax.axis('off');

plt.savefig('schematic.pdf',bbox_inches='tight')
```

Fig. 2 plots the two pairs of functions $(f_1(h),g_1(h))$ and $(f_2(h),g_2(h))$ that yield a product martingale.  We substitute $h = e^\tau$, and plot the real and imaginary parts of all functions separately.

```python
plt.figure(figsize=(10,5))

tau = np.linspace(-1,1,10000)*1j; h = np.exp(tau)

A = 3; B = 6

i = 1
for r in [0.5,1.5]:

    #parameters as a function of tau
    f1, f2, g1, g2 = fg(A,B,r,h)

    #values of parameters at tau = 0
    f0 = (A + B*r)/(r*(A*r + B)); g0 = (A*r + B)/(r*(A + B*r))

    plt.subplot(2,2,i);
    plt.plot(np.imag(tau),np.real(f1),'r'); plt.plot(np.imag(tau),np.imag(f1),'k')
    plt.plot(np.imag(tau),np.real(f2),'r--'); plt.plot(np.imag(tau),np.imag(f2),'k--')
    plt.plot(0,f0,'ro',markersize=8); plt.plot(0,0,'ko',markersize=12); plt.plot(0,0,'o',color='gray',markersize=6)
    plt.plot(0,1,'o',color='pink',markersize=8)
    plt.xlim([-1,1])
    if i == 1:
        plt.xticks([]); plt.ylim([-1,2]); plt.yticks([-1,0,1,2])
        plt.title(r'$f(e^\tau)$')
        plt.ylabel(r'$r = .5$',rotation=0,labelpad=20)
    if i == 3:
        plt.xticks([-1,0,1],[r'$-i$',r'$0$',r'$i$'])
        plt.ylim([-.75,1.3]); plt.yticks([-.7,0,1,1.3])
        plt.xlabel(r'$\tau$'); plt.ylabel(r'$r = 1.5$',rotation=0,labelpad=20)

    i += 1
    plt.subplot(2,2,i);
    plt.plot(np.imag(tau),np.real(g1),'r',label='real, 1'); plt.plot(np.imag(tau),np.imag(g1),'k',label='imag, 1')
    plt.plot(np.imag(tau),np.real(g2),'r--',label='real, 2'); plt.plot(np.imag(tau),np.imag(g2),'k--',label='imag, 2')
    plt.plot(0,g0,'ro',markersize=8); plt.plot(0,0,'ko',markersize=12); plt.plot(0,0,'o',color='gray',markersize=6)
    plt.plot(0,1,'o',color='pink',markersize=8)
    plt.xlim([-1,1]); plt.xticks([])
    if i == 2:
        plt.title(r'$g(e^\tau)$')
        plt.ylim([-1,3]); plt.yticks([-1,0,3])
    if i == 4:
        plt.ylim([-.5,1.5]); plt.yticks([-.5,0,1,1.5])
        plt.legend(bbox_to_anchor=(1.05, 1))

    i += 1

plt.savefig('roots.pdf',bbox_inches='tight')
```

If the relevant simulation .npz files are not in the same directory as this notebook, then run the simulation function for the parameter values stated in the manuscript and save the results.

```python
if not path.exists("bipartite_sims_1.npz"):

    r = 3.; A = 10; B = 4; Sa0 = 1; Sb0 = 1; tau=np.linspace(-1,1,1000)*1j
    trials = 100000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_1.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext)

if not path.exists("bipartite_sims_2.npz"):

    r = .99; A = 10; B = 4; Sa0 = 1; Sb0 = 1; tau=np.linspace(-.25,.25,100)*1j
    trials = 100000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_2.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext)
```

Fig. 3 loads the simulation results and compares them with theoretical results for parameter values stated in the manuscript.

```python
with np.load("bipartite_sims_1.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']

h = np.exp(tau)

#calculate theoretical CCFs
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

#simulation first and second moments of C_T
EC_ext = np.mean(C_ext); EC_fix = np.mean(C_fix)
EC2_ext = np.mean([i**2 for i in C_ext]); EC2_fix = np.mean([i**2 for i in C_fix])

plt.figure(figsize=(12,5))
plt.subplot(2,2,1)
plt.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
plt.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
plt.plot(np.imag(tau),np.real(cfbsim),'r--',lw=2)
plt.plot(np.imag(tau),np.imag(cfbsim),'k--',lw=2)
plt.title(r'$\psi_{C_T|b}(\tau)$',fontsize=22)
plt.xticks([-1,0,1],[r'$-1i$',r'$0$',r'$1i$']); plt.xlim([-1,1])
plt.yticks([0,1]); plt.ylim([-.7,1.2])
plt.ylabel(r'$r = {}$'.format(r), rotation=0, labelpad = 30)
plt.plot(np.imag(tau),np.imag(tau)*EC_ext,'k-',lw=1)
plt.plot(np.imag(tau),-EC2_ext*np.imag(tau)**2+1,'r-',lw=1)
plt.text(-.9,.75 ,r'$E[C_T|b] = {}$'.format(round(EC_ext,1)),fontsize=14)
plt.text(.35,.75 ,r'$E[C_T^2|b] = {}$'.format(int(EC2_ext)),fontsize=14,color='r')

plt.subplot(2,2,2)
plt.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8)
plt.plot(np.imag(tau),np.imag(psia),color='gray',lw=8)
plt.plot(np.imag(tau),np.real(cfasim),'r--',lw=2)
plt.plot(np.imag(tau),np.imag(cfasim),'k--',lw=2)
plt.title(r'$\psi_{C_T|a}(\tau)$',fontsize=22)
plt.xticks([-.5,0,.5],[r'$-.5i$',r'$0$',r'$.5i$']); plt.xlim([-.5,.5])
plt.yticks([0,1]); plt.ylim([-1,1.2])
p_sim = float(S_fix)/(S_fix + S_ext)
plt.text(.05,-.87,r'$\alpha = {} \approx $'.format(round(p,3)) + ' ' + r'$ {}$'.format(round(p_sim,3)))
plt.plot(np.imag(tau),np.imag(tau)*EC_fix,'k-',lw=1)
plt.plot(np.imag(tau),-EC2_fix*np.imag(tau)**2+1,'r-',lw=1)
plt.text(-.36,.75 ,r'${}$'.format(round(EC_fix,1)),fontsize=14)
plt.text(.26,.75 ,r'${}$'.format(int(EC2_fix)),fontsize=14,color='r')

with np.load("bipartite_sims_2.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']

h = np.exp(tau)
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

#simulation first and second moments of C_T
EC_ext = np.mean(C_ext); EC_fix = np.mean(C_fix)
EC2_ext = np.mean([i**2 for i in C_ext]); EC2_fix = np.mean([i**2 for i in C_fix])

plt.subplot(2,2,3)
plt.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
plt.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
plt.plot(np.imag(tau),np.real(cfbsim),'r--',lw=2)
plt.plot(np.imag(tau),np.imag(cfbsim),'k--',lw=2)
plt.xticks([-.25,0,.25],[r'$-.25i$',r'$0$',r'$.25i$']); plt.xlim([-.25,.25])
plt.yticks([0,1]); plt.ylim([-.7,1.2])
plt.xlabel(r'$\tau$')
plt.ylabel(r'$r = {}$'.format(r), rotation=0, labelpad = 30)
plt.plot(np.imag(tau),np.imag(tau)*EC_ext,'k-',lw=1)
plt.plot(np.imag(tau),-EC2_ext*np.imag(tau)**2+1,'r-',lw=1)
plt.text(-.2,.75 ,r'${}$'.format(round(EC_ext,1)),fontsize=14)
plt.text(.15,.75 ,r'${}$'.format(int(EC2_ext)),fontsize=14,color='r')

plt.subplot(2,2,4)
plt.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8,label=r'Re($\psi(\tau)$)')
plt.plot(np.imag(tau),np.imag(psia),color='gray',lw=8,label=r'Im($\psi(\tau)$)')
plt.plot(np.imag(tau),np.real(cfasim),'r--',lw=2,label=r'Re(sims)')
plt.plot(np.imag(tau),np.imag(cfasim),'k--',lw=2,label=r'Im(sims)')
plt.xticks([-.2,0,.2],[r'$-.2i$',r'$0$',r'$.2i$']); plt.xlim([-.2,.2])
plt.yticks([0,1]); plt.ylim([-.8,1.2])
plt.legend(bbox_to_anchor=(1.25, .5),loc='center')
p_sim = float(S_fix)/(S_fix + S_ext)
plt.text(.02,-.67,r'$\alpha = {} \approx $'.format(round(p,3)) + ' ' + r'$ {}$'.format(round(p_sim,3)))
plt.plot(np.imag(tau),np.imag(tau)*EC_fix,'k-',lw=1)
plt.plot(np.imag(tau),-EC2_fix*np.imag(tau)**2+1,'r-',lw=1)
plt.text(-.15,.75 ,r'${}$'.format(round(EC_fix,1)),fontsize=14)
plt.text(.1,.75 ,r'${}$'.format(int(EC2_fix)),fontsize=14,color='r')

plt.tight_layout()
plt.savefig('CCFs.pdf',bbox_inches='tight')
```

Fig. 4 plots theoretical characteristic functions for varying population sizes.  We keep the proportion of individuals in each partition constant.

```python
tau_a = np.linspace(-.01,.01,1000)*1j; tau_b = np.linspace(-.05,.05,1000)*1j

r = 1.01; A = [9,18,36]; B = [3,6,12]; Sa0 = 1; Sb0 = 1; h_a = np.exp(tau_a); h_b = np.exp(tau_b)
psia = []; psib = []; ECb = []; EC2a = []

r_color = ['r','salmon','pink']; i_color = ['k','gray','silver']

for i in range(3):
    psia1,psib1,p = psi(A[i],B[i],r,Sa0,Sb0,h_a);
    psia.append(psia1);
    psia2,psib2,p = psi(A[i],B[i],r,Sa0,Sb0,h_b);
    psib.append(psib2)
    EC2a.append(-1*np.gradient(np.gradient(np.real(psia[-1]),np.imag(tau_a)),np.imag(tau_a))[round((len(tau_a)-2)/2)])
    ECb.append(np.gradient(np.imag(psib[-1]),np.imag(tau_b))[round((len(tau_b)-1)/2)])

plt.figure(figsize=(12,7))
plt.clf()
plt.subplot(2,2,1)
[plt.plot(np.imag(tau_b),np.real(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,r_color,A,B)]
[plt.plot(np.imag(tau_b),np.imag(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,i_color)]
[plt.plot(np.imag(tau_b),i[0]*np.imag(tau_b),color = i[1],lw=1,ls='--') for i in zip(ECb,i_color)]
plt.title(r'$\psi_{C_T|b}(\tau)$',fontsize=22)
plt.xticks([-.05,0,.05],[r'$-.05i$',r'$0$',r'$.05i$']); plt.xlim([-.05,.05])
plt.yticks([0,1]); plt.ylim([-.7,1.2])
[plt.text(.02,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='k') for i in zip([-.2,-.4,-.6],ECb)]
plt.text(.017,0,r'$E[C_T|b]$',fontsize=14)
plt.xlabel(r'$\tau$')

plt.subplot(2,2,2)
[plt.plot(np.imag(tau_a),np.real(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Re, $A, B = {}, {}$'.format(i[2],i[3])) for i in zip(psia,r_color,A,B)]

[plt.plot(np.imag(tau_a),np.imag(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Im, $A, B = {}, {}$'.format(i[2],i[3])) for i in zip(psia,i_color,A,B)]
[plt.plot(np.imag(tau_a),-i[0]*np.imag(tau_a)**2+1,color = i[1],lw=1,ls='--') for i in zip(EC2a,r_color)]
plt.title(r'$\psi_{C_T|a}(\tau)$',fontsize=22)
plt.xlim([-.01,.01])
plt.xticks([-.01,0,.01],[r'$-.01i$',r'$0$',r'$.01i$']);
plt.yticks([-1,0,1]);
plt.ylim([-1.2,1.2])
[plt.text(.0035,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='r') for i in zip([-.7,-.9,-1.1],EC2a)]
plt.text(.0035,-.45,r'$E[C_T^2|a]$',fontsize=14,color='r')
handles, labels = plt.gca().get_legend_handles_labels()
order = [0,3,1,4,2,5]
plt.legend([handles[idx] for idx in order],[labels[idx] for idx in order],bbox_to_anchor=(1.4, .5),loc='center')

plt.tight_layout()
plt.savefig('sizes.pdf',bbox_inches='tight')
```

Fig. 5 plots the theoretical characteristic functions for bipartite graphs with constant total population size ($A+B=12$), but we vary how many individuals are allocated to each partition.  Note that $(A,B) = (6,6)$ is an isothermal graph.  Its conditional characteristic functions of population size changes are equivalent to those of the fully-connected Moran process.

```python
tau_a = np.linspace(-.05,.05,1000)*1j; tau_b = np.linspace(-.5,.5,1000)*1j

r = 1.01; A = [1,3,6]; B = [11,9,6]; Sa0 = 0; Sb0 = 1; h_a = np.exp(tau_a); h_b = np.exp(tau_b)
psia = []; psib = []; ECb = []; EC2a = []

r_color = ['r','salmon','pink']; i_color = ['k','gray','silver']

for i in range(3):
    psia1,psib1,p = psi(A[i],B[i],r,Sa0,Sb0,h_a);
    psia.append(psia1);
    psia2,psib2,p = psi(A[i],B[i],r,Sa0,Sb0,h_b);
    psib.append(psib2)
    EC2a.append(-1*np.gradient(np.gradient(np.real(psia[-1]),np.imag(tau_a)),np.imag(tau_a))[round((len(tau_a)-2)/2)])
    ECb.append(np.gradient(np.imag(psib[-1]),np.imag(tau_b))[round((len(tau_b)-1)/2)])

plt.figure(figsize=(12,7))
plt.clf()
plt.subplot(2,2,1)
[plt.plot(np.imag(tau_b),np.real(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,r_color,A,B)]
[plt.plot(np.imag(tau_b),np.imag(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,i_color)]
[plt.plot(np.imag(tau_b),i[0]*np.imag(tau_b),color = i[1],lw=1,ls='--') for i in zip(ECb,i_color)]
plt.title(r'$\psi_{C_T|b}(\tau)$',fontsize=22)
plt.xticks([-.5,0,.5],[r'$-.5i$',r'$0$',r'$.5i$']); plt.xlim([-.5,.5])
plt.yticks([0,1]); plt.ylim([-.7,1.2])
[plt.text(.25,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='k') for i in zip([-.3,-.45,-.6],ECb)]
plt.text(.23,-.12,r'$E[C_T|b]$',fontsize=14)
plt.xlabel(r'$\tau$')

plt.subplot(2,2,2)
[plt.plot(np.imag(tau_a),np.real(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Re, $A, B = {}, {}$'.format(i[2],i[3])) for i in zip(psia,r_color,A,B)]

[plt.plot(np.imag(tau_a),np.imag(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Im, $A, B = {}, {}$'.format(i[2],i[3])) for i in zip(psia,i_color,A,B)]
[plt.plot(np.imag(tau_a),-i[0]*np.imag(tau_a)**2+1,color = i[1],lw=1,ls='--') for i in zip(EC2a,r_color)]
plt.title(r'$\psi_{C_T|a}(\tau)$',fontsize=22)
plt.xlim([-.05,.05])
plt.xticks([-.05,0,.05],[r'$-.05i$',r'$0$',r'$.05i$']);
plt.yticks([-1,0,1]);
plt.ylim([-1.2,1.2])
[plt.text(.03,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='r') for i in zip([-.7,-.9,-1.1],EC2a)]
plt.text(.03,-.45,r'$E[C_T^2|a]$',fontsize=14,color='r')
handles, labels = plt.gca().get_legend_handles_labels()
order = [0,3,1,4,2,5]
plt.legend([handles[idx] for idx in order],[labels[idx] for idx in order],bbox_to_anchor=(1.3, .5),loc='center')

plt.tight_layout()
plt.savefig('asymmetry.pdf',bbox_inches='tight')
```

Fig. 6 plots the theoretical characteristic functions for varying initial mutant population sizes.

```python
tau_a = np.linspace(-.1,.1,1000)*1j; tau_b = np.linspace(-.5,.5,1000)*1j

r = 2; A = 3; B = 10; Sa0 = [0,1,2]; Sb0 = [1,2,3]; h_a = np.exp(tau_a); h_b = np.exp(tau_b)
psia = []; psib = []; ECb = []; EC2a = []

r_color = ['r','salmon','pink']; i_color = ['k','gray','silver']

for i in range(3):
    psia1,psib1,p = psi(A,B,r,Sa0[i],Sb0[i],h_a);
    psia.append(psia1);
    psia2,psib2,p = psi(A,B,r,Sa0[i],Sb0[i],h_b);
    psib.append(psib2)
    EC2a.append(-1*np.gradient(np.gradient(np.real(psia[-1]),np.imag(tau_a)),np.imag(tau_a))[round((len(tau_a)-2)/2)])
    ECb.append(np.gradient(np.imag(psib[-1]),np.imag(tau_b))[round((len(tau_b)-1)/2)])

plt.figure(figsize=(12,7))
plt.subplot(2,2,1)
[plt.plot(np.imag(tau_b),np.real(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,r_color)]
[plt.plot(np.imag(tau_b),np.imag(i[0]),color = i[1],lw=2,ls='-') for i in zip(psib,i_color)]
[plt.plot(np.imag(tau_b),i[0]*np.imag(tau_b),color = i[1],lw=1,ls='--') for i in zip(ECb,i_color)]
plt.title(r'$\psi_{C_T|b}(\tau)$',fontsize=22)
[plt.text(.35,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='k') for i in zip([-.7,-.9,-1.1],ECb)]
plt.text(.07,-.85,r'$E[C_T|b]$',fontsize=14)
plt.xticks([-.5,0,.5],[r'$-.5i$',r'$0$',r'$.5i$']); plt.xlim([-.5,.5])
plt.yticks([0,1]); plt.ylim([-1.2,1.2])
plt.xlabel(r'$\tau$')

plt.subplot(2,2,2)

[plt.plot(np.imag(tau_a),np.real(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Re, $S_{{a,0}}, S_{{b,0}} = {}, {}$'.format(i[2],i[3])) for i in zip(psia,r_color,Sa0,Sb0)]

[plt.plot(np.imag(tau_a),np.imag(i[0]),color = i[1],lw=2,ls='-',\
         label=r'Im, $S_{{a,0}}, S_{{b,0}} = {}, {}$'.format(i[2],i[3])) for i in zip(psia,i_color,Sa0,Sb0)]

[plt.plot(np.imag(tau_a),-i[0]*np.imag(tau_a)**2+1,color = i[1],lw=1,ls='--') for i in zip(EC2a,r_color)]
plt.title(r'$\psi_{C_T|a}(\tau)$',fontsize=22)
[plt.text(.07,i[0],r'${}$'.format(round(i[1],1)),fontsize=14,color='r') for i in zip([-.7,-.9,-1.1],EC2a)]
plt.text(.03,-.85,r'$E[C_T^2|a]$',fontsize=14,color='r')
handles, labels = plt.gca().get_legend_handles_labels()
order = [0,3,1,4,2,5]
plt.legend([handles[idx] for idx in order],[labels[idx] for idx in order],bbox_to_anchor=(1.4, .5),loc='center')
plt.xlim([-.1,.1]); plt.xticks([-.1,0,.1],[r'$-.1i$',r'$0$',r'$.1i$']);
plt.yticks([-1,0,1]); plt.ylim([-1.2,1.2])

plt.tight_layout()
plt.savefig('start.pdf',bbox_inches='tight')
```

Fig. 7 compares our theoretical characteristic functions of the number of population size changes with those of absorption time.  We obtain the characteristic functions of absorption time from simulations.  If those simulations don't exist in the same directory as this notebook, then run them for the parameter values stated in the manuscript.

```python
if not path.exists("bipartite_sims_3.npz"):

    r = 1.01; A = 4; B = 6; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,1000)*1j
    trials = 200000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_3.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)

if not path.exists("bipartite_sims_4.npz"):

    r = 1.01; A = 9; B = 1; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,10000)*1j
    trials = 200000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_4.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)
```

Fig. 7 compares theoretical characteristic functions of $C_T$ with those of $T$ in the region of weak selection ($r=1.01$).

```python
with np.load("bipartite_sims_3.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']
    cfaT = data['cfaT']; cfbT = data['cfbT']

h = np.exp(tau)

#calculate theoretical CCFs
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

fig = plt.subplots(figsize=(10,6))
ax1 = plt.subplot(221); ax2 = plt.subplot(222); ax3 = plt.subplot(223); ax4 = plt.subplot(224)
ax1b = ax1.twiny(); ax2b = ax2.twiny(); ax3b = ax3.twiny(); ax4b = ax4.twiny()

ax1.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
ax1.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
ax1.set_xticks([-1,0,1]); ax1.set_xlim([-1,1])
ax1.set_title(r'$\psi_{C_T|b}(\tau), \quad \psi_{T|b}(\tau)$',fontsize=22)
ax1b.plot(np.imag(tau),np.real(cfbT),color='r',lw=2,ls='--')
ax1b.plot(np.imag(tau),np.imag(cfbT),color='k',lw=2,ls='--')
ax1b.set_xticks([-.3,.3]); ax1b.set_xlim([-.3,.3])
ax1.set_yticks([0,1])
ax1.text(.2,-.5,r'$A=4, B=6$')

ax2.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8)
ax2.plot(np.imag(tau),np.imag(psia),color='gray',lw=8)
ax2.set_title(r'$\psi_{C_T|a}(\tau), \quad \psi_{T|a}(\tau)$',fontsize=22)
ax2b.plot(np.imag(tau),np.real(cfaT),color='r',lw=2,ls='--')
ax2b.plot(np.imag(tau),np.imag(cfaT),color='k',lw=2,ls='--')
ax2b.set_xlim([-.35,.35]); ax2b.set_xticks([-.35,.35])
ax2.set_xlim([-1,1]); ax2.set_xticks([-1,0,1]); ax2.set_yticks([0,1])

with np.load("bipartite_sims_4.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']
    cfaT = data['cfaT']; cfbT = data['cfbT']

h = np.exp(tau)

#calculate theoretical CCFs
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

ax3.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
ax3.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
ax3.set_xticks([-1,0,1]); ax3.set_xlim([-1,1])
ax3b.plot(np.imag(tau),np.real(cfbT),color='r',lw=2,ls='--')
ax3b.plot(np.imag(tau),np.imag(cfbT),color='k',lw=2,ls='--')
ax3b.set_xticks([-.2,.2]); ax3b.set_xlim([-.2,.2])
ax3.set_yticks([0,1])
ax3.set_xlabel(r'$\tau$')
ax3.text(.2,-.35,r'$A=9,B=1$')

ax4.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8)
ax4.plot(np.imag(tau),np.imag(psia),color='gray',lw=8)
ax4b.plot(np.imag(tau),np.real(cfaT),color='r',lw=2,ls='--')
ax4b.plot(np.imag(tau),np.imag(cfaT),color='k',lw=2,ls='--')
ax4b.set_xlim([-.035,.035]); ax4b.set_xticks([-.035,.035])
ax4.set_xlim([-.1,.1]); ax4.set_xticks([-.1,0,.1]); ax4.set_yticks([0,1])

plt.tight_layout()
plt.savefig('time1.pdf',bbox_inches='tight')
```

Fig. 8 also compares our theoretical characteristic functions with simulation results, but for strong selection.  If simulation results don't exist in the current directory, then run the simulations for the parameter values stated in the manuscript.

```python
if not path.exists("bipartite_sims_5.npz"):

    r = 3.; A = 4; B = 6; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,1000)*1j
    trials = 200000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_5.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)

if not path.exists("bipartite_sims_6.npz"):

    r = 3.; A = 9; B = 1; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,10000)*1j
    trials = 200000
    S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
    np.savez("bipartite_sims_6.npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,C_ext=C_ext,\
             A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)
```

Fig. 8 is analogous to Fig. 7, but in the region of strong selection ($r = 3$).

```python
with np.load("bipartite_sims_5.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']
    cfaT = data['cfaT']; cfbT = data['cfbT']

h = np.exp(tau)

#calculate theoretical CCFs
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

fig = plt.subplots(figsize=(10,6))
ax1 = plt.subplot(221); ax2 = plt.subplot(222); ax3 = plt.subplot(223); ax4 = plt.subplot(224)
ax1b = ax1.twiny(); ax2b = ax2.twiny(); ax3b = ax3.twiny(); ax4b = ax4.twiny()

ax1.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
ax1.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
ax1.set_xticks([-1,0,1]); ax1.set_xlim([-1,1])
ax1.set_title(r'$\psi_{C_T|b}(\tau), \quad \psi_{T|b}(\tau)$',fontsize=22)
ax1b.plot(np.imag(tau),np.real(cfbT),color='r',lw=2,ls='--')
ax1b.plot(np.imag(tau),np.imag(cfbT),color='k',lw=2,ls='--')
ax1b.set_xticks([-.4,.4]); ax1b.set_xlim([-.4,.4])
ax1.set_yticks([0,1])
ax1.text(.2,-.6,r'$A=4,B=6$')

ax2.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8)
ax2.plot(np.imag(tau),np.imag(psia),color='gray',lw=8)
ax2.set_title(r'$\psi_{C_T|a}(\tau), \quad \psi_{T|a}(\tau)$',fontsize=22)
ax2b.plot(np.imag(tau),np.real(cfaT),color='r',lw=2,ls='--')
ax2b.plot(np.imag(tau),np.imag(cfaT),color='k',lw=2,ls='--')
ax2b.set_xlim([-.35,.35]); ax2b.set_xticks([-.35,.35])
ax2.set_xlim([-1,1]); ax2.set_xticks([-1,0,1]); ax2.set_yticks([0,1])

with np.load("bipartite_sims_6.npz") as data:
    S_fix = data['S_fix']; S_ext = data['S_ext']; cfasim = data['cfasim']; cfbsim = data['cfbsim'];
    C_fix = data['C_fix']; C_ext = data['C_ext'];
    A = data['A']; B = data['B']; r = data['r']; Sa0 = data['Sa0']; Sb0 = data['Sb0']; tau = data['tau']
    cfaT = data['cfaT']; cfbT = data['cfbT']

h = np.exp(tau)

#calculate theoretical CCFs
psia,psib,p = psi(A,B,r,Sa0,Sb0,h);

ax3.plot(np.imag(tau),np.real(psib),color = 'pink',lw=8)
ax3.plot(np.imag(tau),np.imag(psib),color='gray',lw=8)
ax3.set_xticks([-1,0,1]); ax3.set_xlim([-1,1])
ax3b.plot(np.imag(tau),np.real(cfbT),color='r',lw=2,ls='--')
ax3b.plot(np.imag(tau),np.imag(cfbT),color='k',lw=2,ls='--')
ax3b.set_xticks([-.35,.35]); ax3b.set_xlim([-.35,.35])
ax3.set_yticks([0,1])
ax3.set_xlabel(r'$\tau$')
ax3.text(.2,-.38,r'$A=9,B=1$')

ax4.plot(np.imag(tau),np.real(psia),color = 'pink',lw=8)
ax4.plot(np.imag(tau),np.imag(psia),color='gray',lw=8)
ax4b.plot(np.imag(tau),np.real(cfaT),color='r',lw=2,ls='--')
ax4b.plot(np.imag(tau),np.imag(cfaT),color='k',lw=2,ls='--')
ax4b.set_xlim([-.025,.025]); ax4b.set_xticks([-.025,.025])
ax4.set_xlim([-.1,.1]); ax4.set_xticks([-.1,0,.1]); ax4.set_yticks([0,1])

plt.tight_layout()
plt.savefig('time2.pdf',bbox_inches='tight')
```

Fig. 9 compares the conditional means of $C_T$ with those of $T$.

```python
r_means = (np.arange(1,6))

for i in r_means:
    if not path.exists("means_1_" + str(i) + ".npz"):

        r = i; A = 4; B = 6; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,1000)*1j
        trials = 100000
        S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
        np.savez("means_1_" + str(i) + ".npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,\
                 C_ext=C_ext,A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)

    if not path.exists("means_2_" + str(i) + ".npz"):
        r = i; A = 9; B = 1; Sa0 = 1; Sb0 = 0; tau=np.linspace(-1,1,10000)*1j
        trials = 100000
        S_fix, S_ext, cfasim, cfbsim, C_fix, C_ext, cfaT, cfbT, T_fix, T_ext = sims(A,B,r,Sa0,Sb0,tau,trials)
        np.savez("means_2_" + str(i) + ".npz",S_fix=S_fix,S_ext=S_ext,cfasim=cfasim,cfbsim=cfbsim,C_fix=C_fix,\
                 C_ext=C_ext,A=A,B=B,r=r,Sa0=Sa0,Sb0=Sb0,tau=tau,T_fix=T_fix,T_ext=T_ext,cfaT=cfaT,cfbT=cfbT)
```

```python
fig = plt.subplots(figsize=(10,6))
ax1 = plt.subplot(221); ax2 = plt.subplot(222); ax3 = plt.subplot(223); ax4 = plt.subplot(224)
ax1b = ax1.twinx(); ax2b = ax2.twinx(); ax3b = ax3.twinx(); ax4b = ax4.twinx()

ECfix1 = []; ECext1 = []; ETfix1 = []; EText1 = []; r = []

for i in r_means:
    with np.load("means_1_" + str(i) + ".npz") as data:
        ECfix1.append(np.mean(data['C_fix'])); ECext1.append(np.mean(data['C_ext'])); r.append(data['r'])
        ETfix1.append(np.mean(data['T_fix'])); EText1.append(np.mean(data['T_ext']));

ax1.plot(r,ECext1,'r.',markersize=24)
ax1b.plot(r,EText1,'bs')
ax2.plot(r,ECfix1,'r.',markersize=24)
ax2b.plot(r,ETfix1,'bs')
ax1.text(3.5,4.5,r'$A=4, B=6$')
ax1.set_title(r'$E[C_T|b], \quad E[T|b]$',fontsize=22)
ax1.set_xticks([])
ax2.set_xticks([])

ECfix2 = []; ECext2 = []; ETfix2 = []; EText2 = []; r = []

for i in r_means:
    with np.load("means_2_" + str(i) + ".npz") as data:
        ECfix2.append(np.mean(data['C_fix'])); ECext2.append(np.mean(data['C_ext'])); r.append(data['r'])
        ETfix2.append(np.mean(data['T_fix'])); EText2.append(np.mean(data['T_ext']));

ax3.plot(r,ECext2,'r.',markersize=24)
ax3b.plot(r,EText2,'bs')
ax4.plot(r,ECfix2,'r.',markersize=24)
ax4b.plot(r,ETfix2,'bs')
ax3.text(3.5,40,r'$A=9, B=1$')
ax2.set_title(r'$E[C_T|a], \quad E[T|a]$',fontsize=22)
ax3.set_xlabel(r'$r$',fontsize=18)
ax4.yaxis.label.set_color('red')

ax1.tick_params(axis='y', colors='blue')
ax1b.tick_params(axis='y', colors='red')
ax2.tick_params(axis='y', colors='blue')
ax2b.tick_params(axis='y', colors='red')
ax3.tick_params(axis='y', colors='blue')
ax3b.tick_params(axis='y', colors='red')
ax4.tick_params(axis='y', colors='blue')
ax4b.tick_params(axis='y', colors='red')

ax1.set_ylim(ymin=0); ax1b.set_ylim(ymin=0)
ax2.set_ylim(ymin=0); ax2b.set_ylim(ymin=0)
ax3.set_ylim(ymin=0); ax3b.set_ylim(ymin=0)
ax4.set_ylim(ymin=0); ax4b.set_ylim(ymin=0)

plt.tight_layout()
plt.savefig('means.pdf',bbox_inches='tight')
```

Martingales and the characteristic functions of absorption time on bipartite graphs.ipynb


# Feature & Target Engineering

Data pre-processing and engineering techniques generally refer to the addition, deletion, or transformation of data.  The time spent on identifying data engineering needs can be significant and requires you to spend substantial time understanding your data...or as Leo Breiman said "live with your data before you plunge into modeling" {cite}`breiman2001statistical`. Although this book primarily focuses on applying machine learning algorithms, feature engineering can make or break an algorithm’s predictive ability and deserves your continued focus and education.

We will not cover all the potential ways of implementing feature engineering; however, we'll cover several fundamental pre-processing tasks that has the potential to significantly improve modeling performance. Moreover, different models have different sensitivities to the type of target and feature values in the model and we will try to highlight some of these concerns. For more in depth coverage of feature engineering, please refer to {cite}`kuhn2019feature` and {cite}`zheng2018feature`.

# Learning objectives

By the end of this module you will know:

- When and how to transform the response variable ("target engineering").
- How to identify and deal with missing values.
- When to filter unnecessary features.
- Common ways to transform numeric and categorical features.
- How to apply dimension reduction.
- How to properly combine multiple pre-processing steps into the modeling process.

# Prerequisites

```python
# Helper packages
import missingno as msno
import modeldata
import numpy as np
import pandas as pd
from plotnine import *

# Modeling pre-processing with scikit-learn functionality
from sklearn.model_selection import train_test_split
from sklearn.compose import TransformedTargetRegressor
from sklearn.compose import ColumnTransformer
from sklearn.compose import make_column_selector as selector
from sklearn.preprocessing import PowerTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.impute import SimpleImputer
from sklearn.impute import KNNImputer
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA

# Modeling pre-processing with non-scikit-learn packages
from category_encoders.ordinal import OrdinalEncoder
from feature_engine.encoding import RareLabelEncoder

# Modeling
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import Pipeline
```

```python
# Ames housing data
ames = modeldata.load_dataset("ames")

# create train/test split
train, test = train_test_split(ames, train_size=0.7, random_state=123)

# separate features from labels and only use numeric features
X_train = train.drop("Sale_Price", axis=1)
y_train = train[["Sale_Price"]]
```

# Target engineering

Although not always a requirement, transforming the response variable can lead to predictive improvement, especially with parametric models which require that certain assumptions about the model be met. For instance, ordinary linear regression models assume that the prediction errors (and hence the response) are normally distributed. This is usually fine, except when the prediction target has heavy tails (i.e., _outliers_) or is skewed in one direction or the other. In these cases, the normality assumption likely does not hold. For example, as we saw in the data splitting section in the last module, the response variable for the Ames housing data (`Sale_Price`) is right (or positively) skewed ranging from \$12,789 to \$755,000. A simple linear model, say $\text{Sale_Price}=\beta_{0} + \beta_{1} \text{Year_Built} + \epsilon$, often assumes the error term $\epsilon$ (and hence `Sale_Price`) is normally distributed; fortunately, a simple log (or similar) transformation of the response can often help alleviate this concern as the below plot illustrates.

Furthermore, using a log (or other) transformation to minimize the response skewness can be used for shaping the business problem as well.  For example, in the [House Prices: Advanced Regression Techniques Kaggle competition](https://www.kaggle.com/c/house-prices-advanced-regression-techniques), which used the Ames housing data, the competition focused on using a log transformed Sale Price response  because _"...taking logs means that errors in predicting expensive houses and cheap houses will affect the result equally."_ This would be an alternative to using the root mean squared logarithmic error (RMSLE) loss function as discussed in the last module.

There are three common approaches to help correct for positively skewed target variables:

1. Normalize with a log transformation. This will transform most right skewed distributions to be approximately normal.

2. If your response has negative values or zeros then a log transformation will produce `NaN`s and `-Inf`s, respectively (you cannot take the logarithm of a negative number).  If the nonpositive response values are small (say between -0.99 and 0) then you can apply a small offset such as in `numpy.log1p()` which adds 1 to the value prior to applying a log transformation. If your data consists of values $\le -1$, use the Yeo-Johnson transformation instead.

3. (__Preferred__) Use a _Box Cox transformation_. A Box Cox transformation is more flexible than (but also includes as a special case) the log transformation and will find an appropriate transformation from a family of power transforms that will transform the variable as close as possible to a normal distribution {cite}`box1964analysis, carroll1981prediction`. At the core of the Box Cox transformation is an exponent, lambda ($\lambda$), which varies from -5 to 5. All values of $\lambda$ are considered and the optimal value for the given data is estimated from the training data; The "optimal value" is the one which results in the best transformation to an approximate normal distribution. The transformation of the response $Y$ has the form:

$$
 \begin{equation}
 y(\lambda) =
\begin{cases}
   \frac{Y^\lambda-1}{\lambda}, & \text{if}\ \lambda \neq 0 \\
   \log\left(Y\right), & \text{if}\ \lambda = 0.
\end{cases}
\end{equation}
$$

```{tip}
If your response has negative values, the Yeo-Johnson transformation is very similar to the Box-Cox but does not require the input variables to be strictly positive.
```

Below illustrates that the log transformation and Box Cox transformation both do about equally well in transforming `Sale_Price` to look more normally distributed.

In Python we use `TransformedTargetRegressor()` to build a plan for target engineering. This will not return the actual log/box-cox transformed values but, rather, a blueprint to be applied later. In this example we are simply building an object that will apply a Box-Cox transformation to the target variable when we fit our model.

```{tip}
There is a `sklearn.preprocessing.power_transform` function that can be applied to immediately transform an array. However, later in this module we'll discuss the idea of data leakage and how important it is to create isolated pre-processing steps.
```

```python
tt = TransformedTargetRegressor(transformer=PowerTransformer(method='box-cox'))
tt
```

    TransformedTargetRegressor(transformer=PowerTransformer(method='box-cox'))

# Dealing with missingness

Data quality is an important issue for any project involving analyzing data. Data quality issues deserve an entire book in their own right, and a good reference is [The Quartz guide to bad data](https://github.com/Quartz/bad-data-guide). One of the most common data quality concerns you will run into is missing values.

Data can be missing for many different reasons; however, these reasons are usually lumped into two categories: _informative missingness_ {cite}`apm` and _missingness at random_ {cite}`little2014statistical`. Informative missingness implies a structural cause for the missing value that can provide insight in its own right; whether this be deficiencies in how the data was collected or abnormalities in the observational environment.  Missingness at random implies that missing values occur independent of the data collection process[^missing_at_random].

[^missing_at_random]: {cite}`little2014statistical` discuss two different kinds of missingness at random; however, we combine them for simplicity as their nuanced differences are distinguished between the two in practice.

The category that drives missing values will determine how you handle them.  For example, we may give values that are driven by informative missingness their own category (e.g., `"None"`) as their unique value may affect predictive performance.  Whereas values that are missing at random may deserve deletion[^missing_at_random2] or imputation.

[^missing_at_random2]: If your data set is large, deleting missing observations that have missing values at random rarely impacts predictive performance. However, as your data sets get smaller, preserving observations is critical and alternative solutions should be explored.

Furthermore, different machine learning models handle missingness differently.  Most algorithms cannot handle missingness (e.g., generalized linear models and their cousins, neural networks, and support vector machines) and, therefore, require them to be dealt with beforehand.  A few models (mainly tree-based), have built-in procedures to deal with missing values.  However, since the modeling process involves comparing and contrasting multiple models to identify the optimal one, you will want to handle missing values prior to applying any models so that your algorithms are based on the same data quality assumptions.

## Visualizing missing values

It is important to understand the distribution of missing values (i.e., `NA`) in any data set. So far, we have been using a pre-processed version of the Ames housing data set. However, if we use the [raw Ames housing data](https://rdrr.io/cran/AmesHousing/man/ames_raw.html), there are actually `r scales::comma(sum(is.na(AmesHousing::ames_raw)))` missing values---there is at least one missing value in each row of the original data! Good visualizations can help us understand patterns in the missing data.

```python
ames_raw = pd.read_csv("../data/ames_raw.csv")

# count total missing values
ames_raw.isnull().sum().sum()
```

    13997

```{tip}
Check out the [missingno](https://github.com/ResidentMario/missingno) package to see all the great ways to to visualize missing data!
```

```python
# can you identify patterns of missing data
# missingness is represented with white
msno.matrix(ames_raw, labels=True, filter="bottom", sort="ascending", n=50)
```

```python
# which features have most missing?
# this chart shows the number of observations so small bars (i.e. Pool QC)
# represent very few observed values (lots of missingness)
msno.bar(ames_raw, labels=True, filter="bottom", sort="ascending", n=50)
```

## Imputation

_Imputation_ is the process of replacing a missing value with a substituted, "best guess" value. Imputation should be one of the first feature engineering steps you take as it will affect any downstream pre-processing[^imputation1].

[^imputation1]: For example, standardizing numeric features will include the imputed numeric values in the calculation and one-hot encoding will include the imputed categorical value.

### Estimated statistic

An elementary approach to imputing missing values for a feature is to compute descriptive statistics such as the mean, median, or mode (for categorical) and use that value to replace `NA`s. Although computationally efficient, this approach does not consider any other attributes for a given observation when imputing (e.g., a female patient that is 63 inches tall may have her weight imputed as 175 lbs since that is the average weight across all observations which contains 65% males that have an average a height of 70 inches).

An alternative is to use grouped statistics to capture expected values for observations that fall into similar groups. However, this becomes infeasible for larger data sets.  Modeling imputation can automate this process for you and the two most common methods include K-nearest neighbor and tree-based imputation, which are discussed next.

However, it is important to remember that imputation should be performed __within the resampling process__ and as your data set gets larger, repeated model-based imputation can compound the computational demands.  Thus, you must weigh the pros and cons of the two approaches.

The following code snippet shows how to impute different features with the median value of a given feature.

In Python, the [`SimpleImputer`](https://scikit-learn.org/stable/modules/generated/sklearn.impute.SimpleImputer.html) can be used to apply an imputation to all features. However, if you only want to apply imputation to a subset of features then you would use [`ColumnTransformer`](https://scikit-learn.org/stable/modules/generated/sklearn.compose.ColumnTransformer.html) in addition to the `SimpleImputer` strategy.

```{warning}
These imputers do not yet perform imputation. At the end of this module we will show you how all these pieces of the feature engineering come together in our ML modeling pipeline.
```

```python
# median imputation to all features
a = SimpleImputer(strategy='median')

# median imputation to just numeric predictors
b = ColumnTransformer([("num_imp", a, selector(dtype_include="number"))])

# median imputation to 1 or more features
c = ColumnTransformer([("num_imp", a, selector("Gr_Liv_Area"))])
```

### K-nearest neighbor

_K_-nearest neighbor (KNN) imputes values by identifying observations with missing values, then identifying other observations that are most similar based on the other available features, and using the values from these nearest neighbor observations to impute missing values.

We discuss KNN for predictive modeling in a later module; the imputation application works in a similar manner.  In KNN imputation, the missing value for a given observation is treated as the targeted response and is predicted based on the average (for quantitative values) or the mode (for qualitative values) of the _k_ nearest neighbors.

As discussed in the KNN modeling module, if all features are quantitative then standard Euclidean distance is commonly used as the distance metric to identify the _k_ neighbors and when there is a mixture of quantitative and qualitative features then Gower's distance {cite}`gower1971general` can be used. KNN imputation is best used on small to moderate sized data sets as it becomes computationally burdensome with larger data sets {cite}`kuhn2019feature`.

```{tip}
As we saw in the last module, **k** is a tunable hyperparameter. Suggested values for imputation are 5--10 {cite}`kuhn2019feature`.
```

```python
knn_imp = KNNImputer(n_neighbors=6)
```

```{tip}
Read more about Scikit-Learn's imputation options [here](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.impute).
```

The plot below illustrates the differences between mean and KNN-based imputation on the raw Ames housing data. The red points represent actual values which were removed and made missing and the blue points represent the imputed values. Estimated statistic imputation methods (i.e. mean, median) merely predict the same value for each observation and can reduce the signal between a feature and the response; whereas KNN imputation procedures tend to maintain the feature distribution and relationship.

```python
from matplotlib import gridspec
from mizani import formatters
import random
import warnings

warnings.filterwarnings('ignore', module='plotnine')

random.seed(123)
sample_index = random.sample(list(ames.index), 50)

# identify actuals
actuals = ames.iloc[sample_index].copy()
p1 = (
    ggplot()
    + geom_point(ames, aes(x="Gr_Liv_Area", y="Sale_Price"), alpha=0.1)
    + geom_point(actuals, aes(x="Gr_Liv_Area", y="Sale_Price"), color='red')
    + scale_x_log10(limits=(500, 4000))
    + scale_y_log10(
        limits=(50000, 600000),
        labels=formatters.currency_format(digits=0, big_mark=',')
        )
    + ggtitle("Actual values")
)

# mean imputed
mean_impute = actuals.copy()
mean_impute['Gr_Liv_Area'] = actuals['Gr_Liv_Area'].mean()
p2 = (
    ggplot()
    + geom_point(actuals, aes(x="Gr_Liv_Area", y="Sale_Price"), color='red')
    + geom_point(mean_impute, aes(x="Gr_Liv_Area", y="Sale_Price"), color='blue')
    + scale_x_log10(limits=(500, 4000))
    + scale_y_log10(
        limits=(50000, 600000),
        labels=formatters.currency_format(digits=0, big_mark=',')
        )
    + ggtitle("Mean imputation")
)

# knn imputed
knn_imp = KNNImputer(n_neighbors=6)
ames_numeric = ames.select_dtypes(include=[np.number]).copy()
ames_numeric.loc[sample_index, 'Gr_Liv_Area'] = np.nan
knn_impute = knn_imp.fit_transform(ames_numeric)
knn_impute = pd.DataFrame(knn_impute, columns=ames_numeric.columns).iloc[sample_index]
p3 = (
    ggplot()
    + geom_point(actuals, aes(x="Gr_Liv_Area", y="Sale_Price"), color='red')
    + geom_point(knn_impute, aes(x="Gr_Liv_Area", y="Sale_Price"), color='blue')
    + scale_x_log10(limits=(500, 4000))
    + scale_y_log10(
        limits=(50000, 600000),
        labels=formatters.currency_format(digits=0, big_mark=',')
        )
    + ggtitle("KNN imputation")
)

# Empty plotnine figure to place the subplots on. Needs junk data (for backend "copy" reasons).

fig = (
    ggplot()
    + geom_blank(data=ames)
    + theme_void()
    + theme(figure_size=(15, 3))
    # such a hack it's embarrassing!
    + ggtitle((
        "    " +
        "(1) Actual values                                              " +
        "(2) Mean imputation                                         " +
        "(3) KNN imputation                          "
        ))
    ).draw()

# Create gridspec for adding subpanels to the blank figure
gs = gridspec.GridSpec(1, 3, wspace=0.3)
ax1 = fig.add_subplot(gs[0,0])
ax2 = fig.add_subplot(gs[0,1])
ax3 = fig.add_subplot(gs[0,2])

# Add subplots to the figure
_ = p1._draw_using_figure(fig, [ax1])
_ = p2._draw_using_figure(fig, [ax2])
_ = p3._draw_using_figure(fig, [ax3])

```

# Feature filtering

# Numeric feature engineering

# Categorical feature engineering

# Dimension reduction

# Proper implementation

# References

```{bibliography}
```


```python
# Importing packages
from __future__ import print_function
from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import numpy as np
import sympy as sm

```

# The Edgeworth Box

In this project we will analyse a small closed economy with the Edgeworth Box. The Edgeworth box can be used to analyse several economic problems. But in this analysis we will use it to describe how two agents: a farmer and a manufacturer of windmills will act in an economy where they have to share the stock capital and labor. The project will solve the problem from the point of view of the pork producer, so the solution would be how the pork producer should optimze the production as the industry is more labor abundant than the windmill producer, which is more capital abundant.

### Presenting the model:
Pork and windmill production are defined with a **Cobb-douglas production function**, with two inputs (capital and labor).

In the economy is the pork production has capital share ($\alpha$) of 30% and labor share ($\beta$) of 70 %. The relative price of the two goods is 0.5, which means that the price of producing pork is half the of producing windmills.

Pork production function: $K^{\alpha}*L^{1-\alpha}$

Windmill production functiong: $K^{\beta}*L^{1-\beta}$

The budget constraint and isoquant of the economy is presented as following:

Budget constraint: $(I/{p_2})-({p_1}/{p_2})*c_1$

Isoquant : $Q/(L^{1-\mu})^{1/\mu}$

```python
# Defining the model
ALPHA = 0.3   # Capital share in pork production
BETA = 0.7   # Labor share in pork production

KBAR = 100
LBAR = 100

p = 0.5      # =Pp/Pw  relative price of pork. Windmills will have the highest price.

# It is defined from the point of view of the pork producer.
# It is assumed that both goods are produced by a Cobb-Douglas production function:

# pork production function
def P(K,L,alpha=ALPHA):
    return (K**alpha)*(L**(1-alpha))

# Windmill production function
def W(K,L,beta=BETA):
    return (K**beta)*(L**(1-beta))

# budget constraint of the economy
def budgetc(c1, p1, p2, I):
    return (I/p2)-(p1/p2)*c1

# isoquant of the economy
def isoq(L, Q, mu):
    return (Q/(L**(1-mu)))**(1/mu)
```

# Efficiency locus
Now that we have defined the production functions and the budget constraints of the economy, it is time to define the efficiency locus, which is a line that represents all allocations where the two agents' isoquants are tangent and the economy is efficient.

```python
# defining the edgeworth locus
def edgeworth(L, Kbar=KBAR, Lbar=LBAR,alpha=ALPHA, beta=BETA):
    a = (1-alpha)/alpha
    b = (1-beta)/beta
    return b*L*Kbar/(a*(Lbar-L)+b*L)
```

# Using sympy
To solve the model symbolically we have to define the symbols first:

```python
L = sm.symbols('L')
K = sm.symbols('K')
alpha = sm.symbols('alpha')
beta = sm.symbols('beta')
MU_pl = sm.symbols('MU_pl')
MU_pk = sm.symbols('MU_pk')
MU_wl = sm.symbols('MU_wl')
MU_wk = sm.symbols('MU_wk')
MRS_p = sm.symbols('MRS_p')
MRS_w = sm.symbols('MRS_w')
pr = sm.symbols('pr')

sm.init_printing(use_unicode=True)
```

First, we find the MP of labor and capital for each good, by using sympys diff function:

```python
#pork
MUpl = sm.diff((K**alpha)*L**(1-alpha),L)
MUpk = sm.diff((K**alpha)*L**(1-alpha),K)
sm.Eq(MUpl/MUpk,MRS_p)
```

```python
#Windmills
MUwl = sm.diff((K**beta)*L**(1-beta),L)
MUwk = sm.diff((K**beta)*L**(1-beta),K)
sm.Eq(MUwl/MUwk,MRS_w)
```

Now, we should be able to the equations, by setting the MRS's equal to the price relationship. But unfortunately, we are having some issues with solving systems of equations.

```python
sm.solve(sm.Eq(MUwl/MUwk/MUpl/MUpk,pr),pr=0.5)
```

# Production Possibility frontier (PPF)
The PPF's are functions which describe the maximum possible production with given levels of capital and labor. It's the PPF's that the efficiency locus are running through. Below are plotted a PPF for pork production, which will use a huge amount of labor relative to capital.

```python
def ppf(LP,Kbar=KBAR, Lbar=LBAR,alpha=ALPHA,beta=BETA):

    KP = edgeworth(LP, Kbar, Lbar,alpha, beta)
    RTS = (alpha/(1-alpha))*(KP/LP)
    QP = P(KP,LP,alpha)
    QW = W(Kbar-KP,Lbar-LP,beta)
    ax.scatter(QP,QW)
    Lp = np.arange(0,Lbar)
    Kp = edgeworth(Lp, Kbar, Lbar,alpha, beta)
    Qp = P(Kp,Lp,alpha)
    Qw = W(Kbar-Kp,Lbar-Lp,beta)
    ax.set_xlim(0, Lbar)
    ax.set_ylim(0, Kbar)
    ax.plot(Qp, Qw,'k--')
    ax.set_xlabel(r'$Q_{Alpha}$',fontsize=12)
    ax.set_ylabel(r'$Q_{Beta}$',fontsize=12)
    plt.show()
```

```python
# Plotting the production possibility frontier
fig, ax = plt.subplots(figsize=(7,6))
ppf(30,alpha =0.3, beta=0.7)
```

# Modelling the Edgeworth Box
The Edgeworth Box is illustrated, using the economy we defined above.

```python
def edgeplot(LP, Kbar=KBAR, Lbar=LBAR, alpha=ALPHA, beta=BETA):
    LP = 60
    KP = edgeworth(LP, Kbar, Lbar,alpha, beta)
    RTS = (alpha/(1-alpha))*(KP/LP)
    QP = P(KP,LP,alpha)
    QW = W(Kbar-KP,Lbar-LP,beta)
    print("(LP,KP)=({:4.1f}, {:4.1f})  (QP, QW)=({:4.1f}, {:4.1f})  RTS={:4.1f}"
          .format(LP,KP,QP,QW,RTS))
    Lp = np.arange(1,Lbar)
    fig, ax = plt.subplots(figsize=(12,8))
    ax.set_xlim(0, Lbar)
    ax.set_ylim(0, Kbar)
    ax.plot(Lp, edgeworth(Lp,Kbar,Lbar,alpha,beta),'k-')
    ax.plot(Lp, isoq(Lp, QP, alpha),'b-')
    ax.plot(Lp, Kbar-isoq(Lbar-Lp, QW, beta),'r-')
    ax.plot(LP, KP,'ob')
    ax.vlines(LP,0,KP, linestyles="dashed")
    ax.hlines(KP,0,LP, linestyles="dashed")
    ax.text(-6,-6,r'$O_P$',fontsize=12)
    ax.text(Lbar,Kbar,r'$O_W$',fontsize=12)
    ax.set_xlabel(r'$Labor$', fontsize=12)
    ax.set_ylabel('$Capital$', fontsize=12)
    plt.show()
```

```python
edgeplot(50)
```

# Analysis
The model shows, as expected, that the less capital intense industry will use a smaller share of the capital and more of the labor.

```python
# Interactive edgeworth box
LP = 50
interact(edgeplot, LP=(10, LBAR-10,1),
         Kbar=fixed(KBAR), Lbar=fixed(LBAR),
         alpha= (0.1,0.9,0.1), beta=(0.1,0.9,0.1));
```

    interactive(children=(IntSlider(value=50, description='LP', max=90, min=10), FloatSlider(value=0.3, descriptio…

# Interactive model
In this interactive version of the Edgeworth Box, the reader can adjust the labor intensity and the $\alpha$ and $\beta$ values of pork production and see how windmill manufacturing reacts in the competitive economy.

# Conclusion
This Edgeworth Box is an excelent model to show how firms react to changes in other firms behavior. It showed that the firms use their comparative advantage so that a capital intensive industry will use the greatest share of capital and vice versa.

```python

```


# **Practice 2: Four Fundamental Subspaces, SVD, Manipulability, and Identification**

---

Goals for today is to recall what SVD and apply it to particular problems.

Are goals for today:
*  Recall the SVD.
*  Apply SVD to find manipulability ellipsoids.
*  Study internal force in cable driven manipulator.

## **Singular Value Decomposition**
The singular value decomposition of an $m\times n$ real or complex matrix $\mathbf{A}$  is a factorization of the form $\mathbf{U S V^{*}}$, where $\mathbf {U}$  is an $m\times m$ real or complex unitary matrix, $\mathbf{S}$ is an ${m\times n}$ rectangular diagonal matrix with non-negative real numbers on the diagonal, and $\mathbf {V}$ is an $n\times n$ real or complex unitary matrix.

The diagonal entries $\sigma_{i}=\mathbf{S}_{ii}$ are known as the singular values of $\mathbf{A}$. The number of non-zero singular values is equal to the rank of $\mathbf{A}$. . Let us for now stick in to a real domain where SVD can be written as:

---

\begin{equation}
\mathbf{A} = \mathbf{U}\mathbf{S}\mathbf{V}^T
\end{equation}

---

with matrices $\mathbf{U},\mathbf{S},\mathbf{V}$ above obeing following usefull properties:
*   Rank $r$ of matrix $\mathbf{A}$ is number of non zero singular values $\sigma_i$ ($\text{dim}\{\mathbf{S}_r\}$)
* Singular values $\sigma$ of $\mathbf{A}$ and eigenvalues $\lambda$ of $\mathbf{A^TA}$ (or $\mathbf{AA^T}$) are related as $\sigma_i = \lambda_i^2$
* The columns of $\mathbf{V}$ are eigenvectors of $A^TA$ called right singular vectors of $\mathbf{A}$.
* The columns of $\mathbf{U}$ are eigenvectors of $AA^T$ called left singular vecotrs of $\mathbf{A}$.
* Determinant is equal to product of eigenvalues $\det\{\mathbf{A}\} = \prod_{i=1}^r\sigma_i$

>**HW EXERCISE**:
* Proof the statements above

### **SVD $\rightarrow$Four Fundamental Subspaces**
Outstanding is that SVD directly provides all **four fundamental subspaces** at once.

---

\begin{equation}
\mathbf{A} = \mathbf{U}\mathbf{S}\mathbf{V}^T = \begin{bmatrix}\underset{m \times r}{\mathbf{U}_r} & \underset{m \times m - r}{\mathbf{U}_n}
\end{bmatrix}
\begin{bmatrix}
\underset{r \times r}{\mathbf{S}_r} & \underset{r \times n - r}{\mathbf{0}} \\
\underset{m - r \times r}{\mathbf{0}} &
\underset{m - r \times n - r}{\mathbf{0}}
\end{bmatrix}
\begin{bmatrix}\underset{n \times r}{\mathbf{V}_r} & \underset{n \times n -r}{\mathbf{V}_n}
\end{bmatrix}^T
= \mathbf{U}_r \mathbf{S}_r \mathbf{V}^T_r
\end{equation}

---

*   **Column space** $\mathcal{C}(\mathbf{A})$is spanned by first $r$ vectors in $\mathbf{U}_r$
*  **Left null space** $\mathcal{N}(\mathbf{A}^T)$ is spanned by $m-r$ vectors in $\mathbf{U}_n$
*   **Row space** $\mathcal{R}(\mathbf{A}^T)$is spanned by first $r$ right singular vectors in $\mathbf{V}_r$
*  **Null space** $\mathcal{N}(\mathbf{A})$ is spanned by $n-r$ vectors in $\mathbf{V}_n$

---

## **Geometrical Representation**
A Singular Value Decomposition allow the intuitive geometrical interpretation.

## **Example: Manipulability Ellipsoids**

in the previous practices we have studied how to describe force and velocity mapping between task and joint spaces, particularly we focus on the null-space motion. Today we will quantify the nonlinear mapping between manipulator spaces by mean of specific indices. It is maybe intuitively done via geometric interpretation of the Jacobian SVD. Consider case of velocity bounded by it's euclidean norm:
\begin{equation}
\|\dot{\mathbf{q}}\|_2^2 = \dot{\mathbf{q}}^T \dot{\mathbf{q}} \leq 1
\end{equation}
Now think about velocity constraints defined by mapping of the joint speeds which is nothing but transformation from the sphere in $\mathbb{R}^n$ to the ellipsoid in $\mathbb{R}^m$.

For example Let us consider the 2 dof mainpulator:

The $i$-th column of Jacobian calculated as:
\begin{equation}
\mathbf{J}_i =
\frac{\partial \boldsymbol{x}}{\partial q_i}
=
\sum^n_{j=i}
\ell_j
\begin{bmatrix}
 -\sin \sum^j_{k=1} q_k\\
 \cos \sum^j_{k=1} q_k
\end{bmatrix}
\end{equation}

```
from numpy import cos, sin, zeros, sum, pi, dot, array

def jacobian_column(q, l, i):
    jac_i = zeros(2)
    for j in range(i,len(q)):
      jac_i+=l[j]*array([-sin(sum(q[:j+1])), cos(sum(q[:j+1]))])
    return jac_i

def jacobian(q, l):
    jac = zeros((2,len(q)))
    for i in range(len(q)):
      jac[:,i] = jacobian_column(q, l, i)
    return jac
```

Let's plot the manipulablity ellipsoid in different configurations:

```
from numpy import linspace
# Define a circle in joint space
n = 100
phi = linspace(0, 1,n)
dq_circle = cos(2*pi*phi), sin(2*pi*phi)

# transform a circle by jacobian in the particular posture
q0 = pi/2,-pi/3
l = 1, 1
J = jacobian(q0, l)
dx_transformed = dot(J,dq_circle)
from matplotlib.pyplot import *
figure(figsize=(6,6))
plot(dq_circle[0], dq_circle[1], color = 'blue')
plot(dx_transformed[0], dx_transformed[1], color = 'red')
ylim([-2.5,2.5])
xlim([-2.5,2.5])
grid(color='black', linestyle='--', linewidth=1.0, alpha = 0.7)
ylabel(r'$\dot{q}_1$ (rad/s)')
xlabel(r'$\dot{q}_2$ (rad/s)')
show()

```

Now we can use SVD to characterize such transformation:

```
from numpy.linalg import svd
from numpy import transpose
from numpy import diag
U,S,VT = svd(J)

print(f' Left singular vectors U:\n {U}\n\n Singular values  S:\n {S}\n\n Right singular vectors V:\n {VT.transpose()}\n\n ')

figure(figsize=(6,6))
plot([0, S[0]*U[0,0]], [0, S[0]*U[0,1]], color = 'red', marker = 'o')
plot([0, S[1]*U[1,0]], [0, S[1]*U[1,1]], color = 'red', marker = 'o')
plot(dq_circle[0], dq_circle[1], color = 'blue')
plot(dx_transformed[0], dx_transformed[1], color = 'red')
ylim([-2.0,2.0])
xlim([-2.0,2.0])
grid(color='black', linestyle='--', linewidth=1.0, alpha = 0.7)
# ylabel(r'$\dot{q}_1$ (rad/s)')
# xlabel(r'$\dot{q}_2$ (rad/s)')
show()

print(f' Elipsoid semi-axes U*S:\n {dot(U,diag(S))}\n')

```

>**QUESTION**:
What will happen in case of singular Jacobian? How you can interpret corresponding SVD physically and geometrically?

It's fairly intuitive to use this concept with $n \times n$ square matrices (not redundant manipulator), however let us now move a bit further to the general case, to do so we will perform similar analysis for the planar 3-DoF manipulator from the first practice:

```
from numpy import ones

q0 = [pi/2, -pi/4, 0]
l = ones(3)
J = jacobian(q0,l)
U,S,VT = svd(J)
print(f' Left singular vectors U:\n {U}\n\n  Singular values  S:\n {S}\n\n Right singular vectors V:\n {VT.transpose()}\n ')

print(f' Elipsoid semi-axes U*S:\n {dot(U,diag(S))}')
```

     Left singular vectors U:
     [[-0.80868274  0.58824504]
     [ 0.58824504  0.80868274]]

      Singular values  S:
     [3.55394055 0.44489738]

     Right singular vectors V:
     [[ 7.83422488e-01 -6.21489505e-01  2.54873123e-17]
     [ 5.55877112e-01  7.00714376e-01 -4.47213595e-01]
     [ 2.77938556e-01  3.50357188e-01  8.94427191e-01]]

     Elipsoid semi-axes U*S:
     [[-2.87401039  0.26170867]
     [ 2.0905879   0.35978083]]

Here a left singular vectors multiplied by a singular values denote a semi axis of ellipsoid, and last right singular vector lies in $\mathcal{N}(\mathbf{J})$

>**HW EXERCISE**:
>*   Visualize a corresponding transformation from the 3D sphere to ellipse

>**BONUS**:
>*   Animate a motion of manipulator for some trajectory, and display transformation of the ellipsoid.

## **Condition Number**
In order to characterize anisotropy of resulting transformation one can denote the following criteria:
\begin{equation}
\kappa (\mathbf{A})={\frac {\sigma _{\text{max}}(\mathbf{A})}{\sigma _{\text{min}}(\mathbf{A})}}
\end{equation}

In general a condition number allow you to characterize how much the output value of the function can change for a small change in the input argument. A definition above is valid for $\ell^2$ norm:
\begin{equation}
\kappa (\mathbf{A})= {\frac {\left\|A^{-1}e\right\|}{\left\|A^{-1}b\right\|}}/{\frac {\|e\|}{\|b\|}}
\end{equation}

```
from numpy.linalg import cond

A = [[1.5,0.2],[0,1]]
B = [[2,1],[1,2]]
figure(figsize=(6,6))
plot(dq_circle[0], dq_circle[1], color = 'blue')
for matrix in A, B:
  x = dot(matrix,dq_circle)
  plot(x[0], x[1])
  ylim([-2.5,2.5])
  xlim([-2.5,2.5])
  print(f'Condition number c {cond(matrix)}')
grid(color='black', linestyle='--', linewidth=1.0, alpha = 0.7)
ylabel(r'$\dot{x}_1$')
xlabel(r'$\dot{x}_2$')
show()

```

In application to the manipulators condition number provides measure of the velocity homogeneity in task space. As closer $\kappa$ to one as velocity is more homogeneus in different directions.

```
q0 = pi/3,-pi/2
l = 1, 1
J = jacobian(q0, l)
dx_transformed = dot(J,dq_circle)

figure(figsize=(6,6))
plot(dq_circle[0], dq_circle[1], color = 'blue')
plot(dx_transformed[0], dx_transformed[1], color = 'red')
ylim([-2.5,2.5])
xlim([-2.5,2.5])
grid(color='black', linestyle='--', linewidth=1.0, alpha = 0.7)
ylabel(r'$\dot{q}_1$ (rad/s)')
xlabel(r'$\dot{q}_2$ (rad/s)')
show()

print(f'Condition number c: {cond(J)}')
```

>**HW EXERCISE**:
>*  Plot condition number for 2 DoF manipulator as function of joint variables (3d plot). Which posture is more homogeneus in terms of speeds?

## **Example: Cable Driven Manipulator, Vector Closure**
Consider the cable driven manipulator with three cables that we discussed previously:

with relationships between cable tensions $\boldsymbol{\tau}\in \mathbb{R}^n$ and force acting on end effector $\mathbf{F}\in \mathbb{R}^m$ given by:

\begin{equation}
\mathbf{F} = \mathbf{W}\boldsymbol{\tau}
\end{equation}

For simplest case $\mathbf{W}$ containt columns of unit vectors  $\mathbf{e}_i$ pointing along direction of applied force $\boldsymbol{\tau}_i$.
\begin{equation}
\mathbf{W} = [\mathbf{e}_1, \mathbf{e}_2, \dots, \mathbf{e}_n]
\end{equation}

```
from numpy.linalg import norm

e1 = [1,1]
e2 = [-1,1]
e3 = [0,-1]
W = array([e1/norm(e1),e2/norm(e2),e3/norm(e3)]).transpose()
print(W)
```

    [[ 0.70710678 -0.70710678  0.        ]
     [ 0.70710678  0.70710678 -1.        ]]

A problem to define tensions $\boldsymbol{\tau_i}$ that corresponds to zero end effector force and also being positive is defined as:
\begin{equation}
\mathbf{W}\boldsymbol{\tau} = \sum_i \mathbf{e}_i\boldsymbol{\tau}_i = \mathbf{0}
\end{equation}

For $\boldsymbol{\tau}_i >0$, and with each set of $m$ from the $\min(n) = m+1$ vectors $\mathbf{e}_i$  in $\mathbf{W}$ being linearly independent.

This criteria is known as [**vector closure**](https://ieeexplore.ieee.org/document/1088014) since it define a set of closed vectors and is nothing but positive subset of the nullspace of $\boldsymbol{W}$. Thus to find a positive forces that do not produce any force we just need to find $\mathcal{N}(\mathbf{W})$ with all positive elements. So in order to check that system of vectors is vector closure we can perform SVD of $\mathbf{W}$

```
U, S, VT = svd(W)
print(f' Left singular vectors U:\n {U}\n\n  Singular values  S:\n {S}\n\n Right singular vectors V:\n {VT.transpose()[:n]}\n ')

print(f' Elipsoid semi-axes U*S:\n {dot(U,diag(S))}')
```

     Left singular vectors U:
     [[0. 1.]
     [1. 0.]]

      Singular values  S:
     [1.41421356 1.        ]

     Right singular vectors V:
     [[ 0.5         0.70710678  0.5       ]
     [ 0.5        -0.70710678  0.5       ]
     [-0.70710678  0.          0.70710678]]

     Elipsoid semi-axes U*S:
     [[0.         1.        ]
     [1.41421356 0.        ]]

>**QUESTION**:
What is physical interpretation of matrix $\mathbf{U}\mathbf{S}$

>**HW EXERCISE**:
>
>  Formulate complete vector closure condition in terms of SVD. To do so answer a following questions:
>*  What we can say about linear dependency of vectors $\mathbf{e}$ by SVD?
>*  Why should we have a minimal $m+1$ cables to perform task in $\mathbb{R}^m$, how to describe this in terms of SVD?

---

>**HW EXERCISE**:
> Given mechanical system with dynamics written linear to parameters ([revise here](https://colab.research.google.com/drive/1UDpCx13PppfQe4u3g0vdq7tpYxaaTujz#scrollTo=exLMblug8v92) ):
\begin{equation}
\mathbf{Q} =
 \mathbf{Y}(\mathbf{q},\dot{\mathbf{q}},\ddot{\mathbf{q}}) \mathbf{p}
\end{equation}
>where:
>* $\mathbf{Q} \in \mathbb{R}^n $ - generalized forces corresponding to
>* $\mathbf{q} \in \mathbb{R}^{n}$ - vector of generilized coordinates
>* $\mathbf{Y} \in \mathbb{R}^{n\times p}$ is so called **regressor matrix**
>* $\mathbf{p} \in \mathbb{R}^{p}$ - is full vector of parameters.
generalized coordinates
>
>Try to intepretate SVD of regressor $\mathbf{Y}$ in terms of phisycal quantities ($\mathbf{Q},\mathbf{p}$).
>
>
>Suppose that you have $N$ measurements $\dot{q}(t)_i,\ddot{q}(t)_i$ and calculate augmented regressor defined as:
>
>\begin{equation}
\bar{\mathbf{Y}} =
\begin{bmatrix}
\mathbf{Y}(t_1) \\
\mathbf{Y}(t_2) \\
\cdots \\
\mathbf{Y}(t_N)
\end{bmatrix}=
\begin{bmatrix}
\mathbf{Y}(\mathbf{q}_1,\dot{\mathbf{q}}_1,\ddot{\mathbf{q}}_1) \\
\mathbf{Y}(\mathbf{q}_2,\dot{\mathbf{q}}_2,\ddot{\mathbf{q}}_2) \\
\cdots \\
\mathbf{Y}(\mathbf{q}_N,\dot{\mathbf{q}}_N,\ddot{\mathbf{q}}_N)
\end{bmatrix}
\end{equation}
>
>* What will be the optimal values of parameter estimates $\hat{\mathbf{p}}$ that minimize: $\| \bar{\mathbf{Q}}^* - \bar{\mathbf{Y}}\mathbf{p} \|$?
>* Does one can always calculate such optimal solution (identify) $\hat{\mathbf{p}}$? Answer in terms of SVD. What condition on trajectory $\mathbf{q}$ implies identifiability, propose the measure of this condition?

---

Google Colab notebooks/Practice Fundumental Subspaces 2020 part 2/practice_02_on_the_svd.ipynb


 INF-285 - Computación Científica / ILI-285 - Computación Científica I

 Floating Point Arithmetic

  [S]cientific [C]omputing [T]eam

 Version: 1.17

## Table of Contents
* [Introduction](#intro)
* [The nature of floating point numbers](#nature)
* [Visualization of floating point numbers](#visualization)
* [What is the first integer that is not representable in double precision?](#firstinteger)
* [Loss of significance](#loss)
* [Loss of significance in funcion evaluation](#func)
* [Another analysis (example from textbook)](#another)
* [Acknowledgements](#acknowledgements)

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

## Introduction

Hello! This notebook is an introduction to how our computers handle the representation of real numbers using double-presicion floating-point format. To understand the contents of this notebook you should have at least a basic notion of how binary numbers work.

The aforementioned format occupies 64 bits which are divided as follows:

* 1 bit for the sign
* 11 bits for the exponent
* 52 bits for the mantissa

This means that the very next representable number after $1$ is $1 + 2^{-52}$, and their difference, $2^{-52}$, is the $\epsilon_{mach}$.

Additionally, if you'd like to quickly go from a base-2 integer to a base-10 integer and viceversa, Python has some functions that can help you with that.

```python
int('0b11', 2)
```

    3

```python
bin(9)
```

    '0b1001'

```python
bin(2**53)
```

    '0b100000000000000000000000000000000000000000000000000000'

## The nature of floating point numbers

As we know until now, float representations of real numbers are just a finite and bounded represetation of them. But another interesting thing, is that these floating numbers are distributed across the real numbers.

To see that, it's really important to keep in mind the following property:

\begin{equation} \left|\frac{\text{fl}(x)-x}{x}\right| \leq \frac{1}{2} \epsilon_{\text{mach}} \end{equation}

where $\text{fl}(x)$ means the float representation of $x \in R$. What it says is that **the relative error in representing any non-zero real number x, is bounded by a quantity that depends on the system representation** ($\epsilon_{\text{mach}}$).

Maybe now you're thinking: what relationship does this have with the distribution of floating point numbers? So if we rewrite the previous property like this:

\begin{equation} |\text{fl}(x)-x| \leq \frac{1}{2} \epsilon_{\text{mach}} |x| \end{equation}

it's clearer: **The absolute error (distance) between a real number and its floating point representation is proportional to the real number's magnitude.**

Intuitively speaking, if the representation error of a number increases as its magnitude increases, then it's quite natural that **the distance between a floating point number and the next representable floating point number will increase as the magnitude of such number increases (and conversely)**. Could you prove that?  For now we will prove it experimentally.

We will use a library named **bitstring** to handle different number representations. You can install it with:
```
pip install bitstring
```

```python
import bitstring as bs
```

The next two functions are self-explanatory:

1. `next_float(f)` computes the next representable float number.
2. `epsilon(f)` computes the difference between f and the next representable float number

```python
def next_float(f):
    #packing double-precision foat
    b = bs.pack('>d', f)

    #extracting mantisa as unsigned int
    #and adding up 1
    m = b[12:].uint
    m += 1

    #putting the result in his place
    b[12:] = m

    return b.float

def epsilon(f):
    next_f = next_float(f)
    return next_f - f
```

So if we compute `epsilon(1)` we should get the epsilon machine number. Let's try it:

```python
epsilon(1)
```

    2.220446049250313e-16

In order to prove our hypotesis, we will create an array of values: [1e-32, 1e-31, ..., 1e31, 1e32] and compute their corresponding epsilon.

```python
#values between 10**-32 and 10**+32
values = np.array([10**i for i in range(-32,32)]).astype(float)

#corresponding epsilons
vepsilon = np.vectorize(epsilon)
eps = vepsilon(values)
```

We include now a comparison between a linear scale plot and a loglog scale plot. Which one is more useful here?

```python
fig = plt.figure(figsize=(10,5))

plt.subplot(121)
plt.plot(values, eps,'.',markersize=20)
plt.xlabel('Values')
plt.ylabel('Corresponding Epsilons')
plt.title('Epsilons v/s Values')
plt.grid(True)

plt.subplot(122)
plt.loglog(values, eps,'.')
plt.xlabel('Values')
plt.ylabel('Corresponding Epsilons')
plt.title('Epsilons v/s Values')
plt.grid(True)

fig.tight_layout()
plt.show()
```

As you can see, the hypotesis was right. In other words: Floating point numbers are not linearly distributed across the real numbers, and the distance between them is proportional to their magnitude. **Tiny numbers (~ 0) are closer between each other than bigger numbers are.**

## Visualization of floating point numbers

With the help of `bitstring` library we could write a function to visualize floating point numbers in his binary representation

```python
def to_binary(f):
    b = bs.pack('>d', f)
    b = b.bin
    #show sign + exponent + mantisa
    print(b[0]+' '+b[1:12]+ ' '+b[12:])
```

Let's see some intereseting examples

```python
to_binary(1.)
```

    0 01111111111 0000000000000000000000000000000000000000000000000000

```python
int('0b01111111111', 2)
```

    1023

```python
to_binary(1.+epsilon(1.))
```

    0 01111111111 0000000000000000000000000000000000000000000000000001

```python
to_binary(+0.)
```

    0 00000000000 0000000000000000000000000000000000000000000000000000

```python
to_binary(-0.)
```

    1 00000000000 0000000000000000000000000000000000000000000000000000

```python
to_binary(np.inf)
```

    0 11111111111 0000000000000000000000000000000000000000000000000000

```python
to_binary(-np.inf)
```

    1 11111111111 0000000000000000000000000000000000000000000000000000

```python
to_binary(np.nan)
```

    0 11111111111 1000000000000000000000000000000000000000000000000000

```python
to_binary(-np.nan)
```

    1 11111111111 1000000000000000000000000000000000000000000000000000

```python
to_binary(2.**-1074)
```

    0 00000000000 0000000000000000000000000000000000000000000000000001

```python
print(2.**-1074)
```

    5e-324

```python
to_binary(2.**-1075)
```

    0 00000000000 0000000000000000000000000000000000000000000000000000

```python
print(2.**-1075)
```

    0.0

```python
to_binary(9.4)
```

    0 10000000010 0010110011001100110011001100110011001100110011001101

## What is the first integer that is not representable in double precision?

Recall that $\epsilon_{\text{mach}}=2^{-52}$ in double precision.

```python
to_binary(1)
to_binary(1+2**-52)
```

    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000001

This means that if we want to store any number in the interval $[1,1+\epsilon_{\text{mach}}]$, only the numbers $1$ and $1+\epsilon_{\text{mach}}$ will be stored. For example, compare the exponent and the mantissa in the previous cell with the following outputs:

```python
for i in np.arange(11):
    to_binary(1+i*2**-55)
```

    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111111111 0000000000000000000000000000000000000000000000000001

Now, we can scale this difference such that the scaling factor multiplied but $\epsilon_{\text{mach}}$ is one. The factor will be $2^{52}$. This means $2^{52}\,\epsilon_{\text{mach}}=1$. Repeating the same example as before but with the scaling factor we obtain:

```python
for i in np.arange(11):
    to_binary((1+i*2**-55)*2**52)
```

    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000001
    0 10000110011 0000000000000000000000000000000000000000000000000001
    0 10000110011 0000000000000000000000000000000000000000000000000001
    0 10000110011 0000000000000000000000000000000000000000000000000001
    0 10000110011 0000000000000000000000000000000000000000000000000001
    0 10000110011 0000000000000000000000000000000000000000000000000001

Which means we can only store exactly the numbers:

```python
to_binary(2**52)
to_binary(2**52+1)
```

    0 10000110011 0000000000000000000000000000000000000000000000000000
    0 10000110011 0000000000000000000000000000000000000000000000000001

This means, the distance now from $2^{52}$ and the following number representable is $1$ !!!! So, what would happend if I can to store $2^{53}+1$?

```python
to_binary(2**53)
to_binary(2**53+1)
```

    0 10000110100 0000000000000000000000000000000000000000000000000000
    0 10000110100 0000000000000000000000000000000000000000000000000000

I can't stored the **Integer** $2^{53}+1$! Thus, the first integer not representable is $2^{53}+1$.

## Loss of significance

As we mentioned, there's a small leap between 1 and the next representable number, which means that if you want to represent a number between those two, you won't be able to do so; that number is nonexistent as it is for the computer, so it'll have to round it to a representable number before storing it in memory.

```python
a = 1.
b = 2.**(-52) #emach
result_1 = a + b     # arithmetic result is 1.0000000000000002220446049250313080847263336181640625
result_1b = result_1-1.0
print("{0:.1000}".format(result_1))
print(result_1b)
print(b)
```

    1.0000000000000002220446049250313080847263336181640625
    2.220446049250313e-16
    2.220446049250313e-16

```python
c = 2.**(-53)
result_2 = a + c     # arithmetic result is 1.00000000000000011102230246251565404236316680908203125
np.set_printoptions(precision=16)
print("{0:.1000}".format(result_2))
print(result_2-a)
```

    1.0
    0.0

```python
to_binary(result_2)
to_binary(result_2-a)
```

    0 01111111111 0000000000000000000000000000000000000000000000000000
    0 00000000000 0000000000000000000000000000000000000000000000000000

```python
d = 2.**(-53) + 2.**(-54)

result_3 = a + d     # arithmetic result is 1.000000000000000166533453693773481063544750213623046875
print("{0:.1000}".format(result_3))
to_binary(result_3)
to_binary(d)
```

    1.0000000000000002220446049250313080847263336181640625
    0 01111111111 0000000000000000000000000000000000000000000000000001
    0 01111001010 1000000000000000000000000000000000000000000000000000

As you can see, if you try to save a number between $1$ and $1 + \epsilon _{mach}$, it will have to be rounded (according to some criteria) to a representable number before being stored, thus creating a difference between the real number and the stored number. This situation is an example of loss of significance.

Does that mean that the "leap" between representable numbers is always going to be $\epsilon _{mach}$? Of course not! Some numbers will require smaller leaps, and some others will require bigger leaps.

In any interval of the form $[2^n,2^{n+1}]$ for $n\in \mathbb{Z}$ and representables, is constant. For example, all the numbers between $2^{-1}$ and $2^0$ (but excluding $2^0$) have a distance of $\epsilon _{mach}/2$ between them. All the numbers between $2^0$ and $2^1$ (excluding $2^1$) have a distance of $\epsilon _{mach}$ between them. Those between $2^1$ and $2^2$ (not including $2^2$) have a distance of $2\,\epsilon _{mach}$ between them, and so on and so forth.

```python
e = 2.**(-1)
f = b/2. # emach/2

result_4 = e + f     # 0.50000000000000011102230246251565404236316680908203125
print("{0:.1000}".format(result_4))

result_5 = e + b     # 0.5000000000000002220446049250313080847263336181640625
print("{0:.1000}".format(result_5))
```

    0.50000000000000011102230246251565404236316680908203125
    0.5000000000000002220446049250313080847263336181640625

```python
g = b/4.

result_5 = e + g     # 0.500000000000000055511151231257827021181583404541015625
print("{0:.1000}".format(result_5))
```

    0.5

We'll let the students find some representable numbers and some non-representable numbers. It's important to note that loss significance can occur in many more operations and functions other that the simple addition of two numbers.

```python
num_1 = a
num_2 = b
result = a + b
print("{0:.1000}".format(result))
```

    1.0000000000000002220446049250313080847263336181640625

## Loss of significance in function evaluation

Loss of Significance is present too in the representation of **functions**. A classical example (which you can see in the guide book), is the next function:

\begin{equation}f(x)= \frac{1 - \cos x}{\sin^{2}x} \end{equation}

Applying trigonometric identities, we can obtain the 'equivalent' function:

\begin{equation}f(x)= \frac{1}{1 + \cos x} \end{equation}

Both of these functions are apparently equals. Nevertheless, its graphics say to us another thing when $x$ is equal to zero.

```python
x = np.arange(-10,10,0.1)
y = (1-np.cos(x))/(np.sin(x)**2)
plt.figure()
plt.plot(x,y,'.')
plt.grid(True)
plt.show()
```

```python
x = np.arange(-10,10,0.1)
y = 1/(1+np.cos(x))
plt.figure()
plt.plot(x,y,'.')
plt.grid(True)
plt.show()
```

```python
x = np.arange(-1,1,0.01)
y = (1-np.cos(x))/np.sin(x)**2
plt.figure()
plt.plot(x,y,'.',markersize=10)
plt.grid(True)
plt.show()
```

```python
y = 1/(1+np.cos(x))
plt.figure()
plt.plot(x,y,'.',markersize=10)
plt.grid(True)
plt.show()
```

This happens because when $x$ is equal to zero, the first function has an indetermination, but previously, the computer makes a subtraction between numbers that are almost equals. This generates a loss of significance, turning the expression close to this point to be zero. However, modifying this expression to make the second function, eliminates this substraction, fixing the error in its calculation when $x=0$.

In conclusion, for us, two representations of a function can be equals, but for the computer they can be different!

## Another analysis (example from textbook)

```python
f1 = lambda x: (1.-np.cos(x))/(np.sin(x)**2)
f2 = lambda x: 1./(1+np.cos(x))
f3 = lambda x: (1.-np.cos(x))
f4 = lambda x: (np.sin(x)**2)
x = np.logspace(-19,0,20)[-1:0:-1]
o1 = f1(x)
o2 = f2(x)
o3 = f3(x)
o4 = f4(x)

print("x,            f1(x),        f2(x),        f3(x),                       f4(x)")
for i in np.arange(len(x)):
    print("%1.10f, %1.10f, %1.10f, %1.25f, %1.25f" % (x[i],o1[i],o2[i],o3[i],o4[i]))
```

    x,            f1(x),        f2(x),        f3(x),                       f4(x)
    1.0000000000, 0.6492232052, 0.6492232052, 0.4596976941318602349895173, 0.7080734182735711756961905
    0.1000000000, 0.5012520863, 0.5012520863, 0.0049958347219742904599116, 0.0099667110793791851425238
    0.0100000000, 0.5000125002, 0.5000125002, 0.0000499995833347366414046, 0.0000999966667111107946004
    0.0010000000, 0.5000001250, 0.5000001250, 0.0000004999999583255032576, 0.0000009999996666667110814
    0.0001000000, 0.4999999986, 0.5000000012, 0.0000000049999999696126451, 0.0000000099999999666666680
    0.0000100000, 0.5000000414, 0.5000000000, 0.0000000000500000041370185, 0.0000000000999999999966667
    0.0000010000, 0.5000444503, 0.5000000000, 0.0000000000005000444502912, 0.0000000000009999999999997
    0.0000001000, 0.4996003611, 0.5000000000, 0.0000000000000049960036108, 0.0000000000000100000000000
    0.0000000100, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000001000000000
    0.0000000010, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000010000000
    0.0000000001, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000100000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000001000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000010
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000
    0.0000000000, 0.0000000000, 0.5000000000, 0.0000000000000000000000000, 0.0000000000000000000000000

## Libraries
### Please make sure you make all of them your BFF!!

* Numpy - IEEE 754 Floating Point Special Values: https://docs.scipy.org/doc/numpy-1.10.0/user/misc.html
* Matplotlib: http://matplotlib.org/examples/pylab_examples/simple_plot.html
* Nice Trick: https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/

# Acknowledgements
* _Material originally created by professor Claudio Torres_ (`ctorres@inf.utfsm.cl`) _and assistants: Laura Bermeo, Alvaro Salinas, Axel Simonsen and Martín Villanueva. v.1.1. DI UTFSM. March 2016._
* _Update April 2020 - v1.14 - C.Torres_ : Fixing some issues.
* _Update April 2020 - v1.15 - C.Torres_ : Adding subplot.
* _Update April 2020 - v1.16 - C.Torres_ : Adding value of numerator and denominator in example of f1 = lambda x: (1.-np.cos(x))/(np.sin(x)** 2).
* _Update April 2020 - v1.17 - C.Torres_ : Adding section "What is the first integer that is not representable in double precision?"

```python

```


# Section 4.6 $\quad$ Basis and Dimension

## Definition of Basis

>The vectors $\mathbf{v}_1, \mathbf{v}_2, \cdots, \mathbf{v}_k$ in a vector space $V$ are said to form a $\underline{basis}$ for $V$ if
-

-

**Remark**

### Example 1

In $\mathbb{R}^3$, the vectors
$\left[\begin{array}{c} 1 \\ 0 \\ 0 \\ \end{array} \right]$,
$\left[\begin{array}{c} 0 \\ 1 \\ 0 \\ \end{array} \right]$,
$\left[\begin{array}{c} 0 \\ 0 \\ 1 \\ \end{array} \right]$,
form a basis.

**Remark**

### Example 2

Show that the set $S = \{t^2+1,t-1,2t+2\}$ is a basis for the vector space $\mathcal{P}_2$.

```python
from sympy import *

A = Matrix([[1, 0, 1], [0, 1, -1], [0, 2, 2]]);

A.rank()
```

    3

### Example 3

Find a basis for the subspace $V$ of $\mathcal{P}_2$, consisting of all vectors of the form $at^2+bt+c$, where $c=a-b$.

>**Theorem** If $S = \{\mathbf{v}_1, \mathbf{v}_2, \cdots, \mathbf{v}_k\}$ is a basis for $V$,

**Proof**

>**Theorem** If $S = \{\mathbf{v}_1, \mathbf{v}_2, \cdots, \mathbf{v}_k\}$, and $T = \{\mathbf{w}_1, \mathbf{w}_2, \cdots, \mathbf{w}_l\}$ are bases for $V$, then

### Example 4

Let
\begin{equation*}
  S = \left\{
  \left[\begin{array}{cc} 1 & 1\\ 0 & 0 \\ \end{array} \right],
  \left[\begin{array}{cc} 0 & 0\\ 1 & 1 \\ \end{array} \right],
  \left[\begin{array}{cc} 1 & 0\\ 0 & 1 \\ \end{array} \right],
  \left[\begin{array}{cc} 0 & 1\\ 1 & 1 \\ \end{array} \right]
  \right\},~~~~
  T = \left\{
  \left[\begin{array}{cc} 1 & 1\\ 0 & 0 \\ \end{array} \right],
  \left[\begin{array}{cc} 0 & 0\\ 1 & 1 \\ \end{array} \right],
  \left[\begin{array}{cc} 1 & 0\\ 0 & 1 \\ \end{array} \right]
  \right\}.
\end{equation*}
Is $S$ a basis of $M_{22}$? Is $T$ a basis of $M_{22}$?

```python
from sympy import *

S = Matrix([[1, 1, 0, 0], [0, 0, 1, 1], [1, 0, 0, 1], [0, 1, 1, 1]]);

S.rank()
```

    4

### Example 5

Let $V=\mathbb{R}^3$, $s=\{\mathbf{v}_1, \mathbf{v}_2, \mathbf{v}_3, \mathbf{v}_4, \mathbf{v}_5\}$, where $\mathbf{v}_1 = \left[\begin{array}{c} 1 \\ 0 \\ 1 \\ \end{array} \right]$,
$\mathbf{v}_2 = \left[\begin{array}{c} 0 \\ 1 \\ 1 \\ \end{array} \right]$,
$\mathbf{v}_3 = \left[\begin{array}{c} 1 \\ 1 \\ 2 \\ \end{array} \right]$,
$\mathbf{v}_4 = \left[\begin{array}{c} 1 \\ 2 \\ 1 \\ \end{array} \right]$,
$\mathbf{v}_5 = \left[\begin{array}{c} -1 \\ 1 \\ -2 \\ \end{array} \right]$.
Find a subset of $S$ that is a basis for $\mathbb{R}^3$.

```python
from sympy import *

A = Matrix([[1, 0, 1], [0, 1, 1], [1, 2, 1]]);

A.rank()
```

    3