```python
 %matplotlib inline
import networkx as nx
import itertools
from itertools import combinations
import random
import numpy as np
import matplotlib.pyplot as plt
import time
from scipy.special import comb
from collections import Counter
from itertools import chain
import math
import pandas as pd
import seaborn as sns
import pickle
from google.colab import drive
drive.mount('/content/gdrive')
import seaborn as sns
from sympy.functions.combinatorial.numbers import stirling, bell
from scipy.special import factorial
from sympy import symbols, exp
from scipy.optimize import fsolve
```

    Mounted at /content/gdrive

```python
b = 11
x = 3
R = 10
```

```python
G = nx.Graph()
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
G.add_nodes_from(allPerms)
start_time = time.time()
Source = allPerms[random.randint(0,2**b-1)]
Dest = allPerms[random.randint(0,2**b-1)]
twoLocs_Source = random.sample(range(b),x)
twoLocs_Dest = random.sample(range(b),x)
allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
```

```python
G = nx.Graph()
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
G.add_nodes_from(allPerms)
start_time = time.time()
for i in range(R):
    Source = allPerms[random.randint(0,2**b-1)]
    Dest = allPerms[random.randint(0,2**b-1)]
    twoLocs_Source = random.sample(range(b),x)
    twoLocs_Dest = random.sample(range(b),x)
    allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
    allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
    for perms_source in range(2**x):
        for perms_dest in range(2**x):
            s_twos = list(allPerms_lengthX_Source[perms_source])
            d_twos = list(allPerms_lengthX_Dest[perms_dest])
            G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))

time.time() - start_time
```

    0.004911661148071289

# Save Numcomp, etc

```python
b = 8
xRange = range(1,8,1)
repeats = 30;
minR = 2
maxR = 160
```

```python
maxLinks = 0.5*2**b*(2**b-1)+2**b
rangeToUse = np.unique([int(i) for i in np.rint(np.logspace(np.log10(minR),np.log10(maxR),50))])
densities = [[] for i in xRange]
conn_comp = [[] for i in xRange]
LCC = [[] for i in xRange]
selfmoment = [[] for i in xRange]
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
```

```python
for j,x in enumerate(list(xRange)):
    print('j = ' + str(j))
    for R in rangeToUse:
        print('R = ' + str(R))
        densities_local = []
        conn_comp_local = []
        LCC_local = []
        selfmoment_local = []
        conn_crit_local = []

        for reps in range(repeats):
            G = nx.Graph()
            G.add_nodes_from(allPerms)
            for i in range(R):
                Source = allPerms[random.randint(0,2**b-1)]
                Dest = allPerms[random.randint(0,2**b-1)]
                twoLocs_Source = random.sample(range(b),x)
                twoLocs_Dest = random.sample(range(b),x)
                allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                for perms_source in range(2**x):
                    for perms_dest in range(2**x):
                        s_twos = list(allPerms_lengthX_Source[perms_source])
                        d_twos = list(allPerms_lengthX_Dest[perms_dest])
                        G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))
            conn_comp_G = list(nx.connected_components(G))
            conn_comp_local.append(len(conn_comp_G))
            LCC_local.append(len(max(conn_comp_G, key=len)))
            densities_local.append(G.number_of_edges()/maxLinks)
            degrees = list(dict(G.degree()).values())
            DegCounts = Counter(degrees)
            total = sum(DegCounts.values(), 0.0)
            for key in DegCounts:
                DegCounts[key] /= total
            k_net = sum(list(DegCounts.values())*np.array(list(DegCounts.keys()))**2)
            k = G.number_of_edges()/maxLinks*2**b
            selfmoment_local.append(k_net/(k*(k+1)))
            degrees = G.degree()
            to_keep = [n[0] for n in degrees if n[1] != 0]
            conn_comp_G = list(nx.connected_components(G.subgraph(to_keep)))
            conn_crit_local.append(len(conn_comp_G))
        conn_comp_crit[j].append(np.mean(conn_crit_local))
        selfmoment[j].append(np.mean(selfmoment_local))
        conn_comp[j].append(np.mean(conn_comp_local))
        LCC[j].append(np.mean(LCC_local))
        densities[j].append(np.mean(densities_local))

with open('/content/gdrive/My Drive/b8phasePlots', 'wb') as f:
  pickle.dump([selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit], f)
```

    j = 0
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 1
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 2
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 3
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 4
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 5
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160
    j = 6
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160

```python
with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 11
```

```python
with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
b = 10
```

```python
R_crit_density = []
R_crit_ruleNum = []
for j,x in enumerate(list(range(1,len(xRange)+2,1))):
  R_crit_density.append(np.log(1-2**(-1*b))/(2*np.log(1-2**(2*(x-b)))))
  R_crit_ruleNum.append(2**((b-x)))
```

```python
ax = plt.axes(projection='3d')
X,Y = np.meshgrid(rangeToUse,xRange)
ax.plot_surface(X, Y, np.array(densities))
plt.xlabel('Number of Rules')
plt.ylabel('Number of Xs')
ax.set_zlabel('Density')
```

```python
b=11
v2 = []
for e,v in enumerate(LCC):
  nodesPerBC = 2**(e+2)
  v2.append(np.array(v)/nodesPerBC)
```

```python
#plt.imshow(densities)
X,Y = np.meshgrid(np.insert(rangeToUse,0,0),np.insert(xRange,0,0))
#X,Y = np.meshgrid(rangeToUse,xRange)
plt.pcolor(X,Y,np.array(densities))
plt.colorbar()
#plt.ylim(-0.5,7.5)
plt.step(R_crit_ruleNum,np.array(range(len(xRange)+1)),c = 'r')
plt.step(R_crit_density,np.array(range(len(xRange)+1)),c = 'r')
#plt.xticks(list(range(4,40,5)),rangeToUse[4:41:5]);
#plt.yticks(list(range(0,8,3)),list(range(1,9,3)));
plt.xlim((0,160))
plt.ylim((0,max(xRange)))
#plt.xticks(range(8))
plt.show()
#plt.savefig('/content/gdrive/My Drive/degree_5_70.pdf')
```

```python
ax = plt.axes(projection='3d')
X,Y = np.meshgrid(rangeToUse,xRange)
ax.plot_surface(X, Y, np.array(LCC))
plt.xlabel('Number of Rules')
plt.ylabel('Number of Xs')
ax.set_zlabel('LCC')
```

```python
ax = plt.axes(projection='3d')
X,Y = np.meshgrid(rangeToUse,xRange)
ax.plot_surface(X, Y, np.array(conn_comp))
plt.xlabel('Number of Rules')
plt.ylabel('Number of Xs')
ax.set_zlabel('Connected Components')
```

```python
#plt.imshow(densities)
X,Y = np.meshgrid(np.insert(rangeToUse,0,0),np.insert(xRange,0,0))
#X,Y = np.meshgrid(rangeToUse,xRange)
plt.pcolor(X,Y,np.array(LCC))
plt.colorbar()
#plt.ylim(-0.5,7.5)
plt.step(R_crit_ruleNum,np.array(range(len(xRange)+1)),c = 'r')
plt.step(R_crit_density,np.array(range(len(xRange)+1)),c = 'r')
#plt.xticks(list(range(4,40,5)),rangeToUse[4:41:5]);
#plt.yticks(list(range(0,8,3)),list(range(1,9,3)));
plt.xlim((0,160))
plt.ylim((0,max(xRange)))
#plt.xticks(range(8))
#plt.show()
plt.savefig('/content/gdrive/My Drive/b11LCC.pdf')
```

```python
#plt.imshow(densities)
X,Y = np.meshgrid(np.insert(rangeToUse,0,0),np.insert(xRange,0,0))
#X,Y = np.meshgrid(rangeToUse,xRange)
plt.pcolor(X,Y,np.array(conn_comp))
plt.colorbar()
#plt.ylim(-0.5,7.5)
plt.step(R_crit_ruleNum,np.array(range(len(xRange)+1)),c = 'r')
plt.step(R_crit_density,np.array(range(len(xRange)+1)),c = 'r')
#plt.xticks(list(range(4,40,5)),rangeToUse[4:41:5]);
#plt.yticks(list(range(0,8,3)),list(range(1,9,3)));
plt.xlim((0,160))
plt.ylim((0,max(xRange)))
#plt.xticks(range(8))
#plt.show()
plt.savefig('/content/gdrive/My Drive/nCCb11.pdf')
```

```python
#plt.imshow(densities)
X,Y = np.meshgrid(np.insert(rangeToUse,0,0),np.insert(xRange,0,0))
#X,Y = np.meshgrid(rangeToUse,xRange)
plt.pcolor(X,Y,np.array(conn_comp_crit))
plt.colorbar()
#plt.ylim(-0.5,7.5)
plt.step(R_crit_ruleNum,np.array(range(len(xRange)+1)),c = 'r')
plt.step(R_crit_density,np.array(range(len(xRange)+1)),c = 'r')
#plt.xticks(list(range(4,40,5)),rangeToUse[4:41:5]);
#plt.yticks(list(range(0,8,3)),list(range(1,9,3)));
plt.xlim((0,160))
plt.ylim((0,max(xRange)))
#plt.xticks(range(8))
#plt.show()
plt.set_cmap('PuBuGn')
#plt.savefig('/content/gdrive/My Drive/nCCCb11.pdf')
```

```python
#plt.imshow(densities)
X,Y = np.meshgrid(np.insert(rangeToUse,0,0),np.insert(xRange,0,0))
#X,Y = np.meshgrid(rangeToUse,xRange)
plt.pcolor(X,Y,np.array(conn_comp_crit))
plt.colorbar()
#plt.ylim(-0.5,7.5)
plt.step(R_crit_ruleNum,np.array(range(len(xRange)+1)),c = 'r')
plt.step(R_crit_density,np.array(range(len(xRange)+1)),c = 'r')
#plt.xticks(list(range(4,40,5)),rangeToUse[4:41:5]);
#plt.yticks(list(range(0,8,3)),list(range(1,9,3)));
plt.xlim((0,160))
plt.ylim((0,max(xRange)))
#plt.xticks(range(8))
plt.show()
```

## Scaling

```python
R = 36
# 23 # R = 38
# 31 # R = 78
# 36 # R = 122

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 11
densities = np.transpose(np.array(densities))
plt.scatter(b-np.array(xRange),densities[R],c='b',label='b=11')

with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 10
densities = np.transpose(np.array(densities))
plt.scatter(b-np.array(xRange),densities[R], facecolors='none',edgecolors='m',label='b=10')

with open('/content/gdrive/My Drive/b9phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 9
densities = np.transpose(np.array(densities))
plt.scatter(b-np.array(xRange),densities[R],marker='x',c='g',label='b=9')

with open('/content/gdrive/My Drive/b8phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 8
densities = np.transpose(np.array(densities))
plt.scatter(b-np.array(xRange),densities[R],marker='+',c='r',label='b=8')

plt.yscale('log')
#plt.plot(range(1,11),R/((2**(np.array(range(1,11)))-1)*2**(np.array(range(1,11)))))
E_poss = (2**(np.array(range(1,11)))-1)*2**(np.array(range(1,11)))/2.
plt.plot(range(1,11),1-((E_poss-1)/(E_poss))**(rangeToUse[R]),c='y',label='fit')
plt.ylabel('density')
plt.xlabel('b-x')
plt.legend()
plt.title(str(rangeToUse[R]))
```

```python
R = 36
# 23 # R = 38
# 31 # R = 78
# 36 # R = 122

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 11
LCC = np.transpose(np.array(LCC))
#plt.scatter(-b+np.array(xRange),LCC[R]/2**b)
plt.scatter(b-np.array(xRange),LCC[R]/2**b,c='b',label='b=11')
#plt.scatter(np.array(xRange)/b,LCC[R]/2**b)

with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 10
LCC = np.transpose(np.array(LCC))
plt.scatter(b-np.array(xRange),LCC[R]/2**b, facecolors='none',edgecolors='m',label='b=10')

with open('/content/gdrive/My Drive/b9phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 9
LCC = np.transpose(np.array(LCC))
plt.scatter(b-np.array(xRange),LCC[R]/2**b,marker='x',c='g',label='b=9')

with open('/content/gdrive/My Drive/b8phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 8
LCC = np.transpose(np.array(LCC))
plt.scatter(b-np.array(xRange),LCC[R]/2**b,marker='+',c='r',label='b=8')

#with open('/content/gdrive/My Drive/b7phasePlots', 'rb') as f:
#    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
#b = 7
#LCC = np.transpose(np.array(LCC))
#plt.scatter(b-np.array(xRange),LCC[R]/2**b)

plt.yscale('log')
#plt.plot(-1*np.array(range(1,11)),np.array(range(1,11))/(2**(2*np.array(range(1,11)))))
E_poss = (2**(np.array(range(1,11))))
plt.plot(range(1,11),1-((E_poss-1)/(E_poss))**(2*rangeToUse[R]),c='y',label='fit')
plt.ylabel('LCC/N')
plt.xlabel('b-x')
plt.title(str(rangeToUse[R]))
plt.legend()
```

```python
R = 36
# 23 # R = 38
# 31 # R = 78
# 36 # R = 122

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 11
LCC = np.transpose(np.array(conn_comp))
plt.scatter(b-np.array(xRange),LCC[R]/2**b,c='b',label='b=11')

#plt.scatter(np.array(xRange)/b,LCC[R]/2**b)

with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 10
LCC = np.transpose(np.array(conn_comp))
plt.scatter(b-np.array(xRange),LCC[R]/2**b, facecolors='none',edgecolors='m',label='b=10')

with open('/content/gdrive/My Drive/b9phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 9
LCC = np.transpose(np.array(conn_comp))
plt.scatter(b-np.array(xRange),LCC[R]/2**b,marker='x',c='g',label='b=9')

with open('/content/gdrive/My Drive/b8phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 8
LCC = np.transpose(np.array(conn_comp))
plt.scatter(b-np.array(xRange),LCC[R]/2**b,marker='+',c='r',label='b=8')

# with open('/content/gdrive/My Drive/b7phasePlots', 'rb') as f:
#    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
# b = 7
# LCC = np.transpose(np.array(conn_comp))
# plt.scatter(b-np.array(xRange),LCC[R]/2**b)

plt.yscale('log')
#plt.xscale('log')
#E_poss = (2**(1+np.array(range(0,11))))
#plt.plot(11-np.array(range(1,12)),1-((E_poss-1)/(E_poss))**(R))
minVal = 6
E_poss = 2**(np.array(range(minVal,11)))
plt.plot(range(minVal,11),((E_poss-1)/(E_poss))**(2*rangeToUse[R]),c='y',label='fit')
plt.ylabel('#CC/N')
plt.xlabel('b-x')
plt.title(str(rangeToUse[R]))
plt.legend()
```

```python
R = 10

# 23 # R = 38
# 31 # R = 78
# 36 # R = 122

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 11
LCC = np.transpose(np.array(conn_comp_crit))
plt.scatter(b-np.array(xRange),LCC[R],c='b',label='b=11')

#plt.scatter(np.array(xRange)/b,LCC[R]/2**b)

with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 10
LCC = np.transpose(np.array(conn_comp_crit))
plt.scatter(b-np.array(xRange),LCC[R], facecolors='none',edgecolors='m',label='b=10')

with open('/content/gdrive/My Drive/b9phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 9
LCC = np.transpose(np.array(conn_comp_crit))
plt.scatter(b-np.array(xRange),LCC[R],marker='x',c='g',label='b=9')

with open('/content/gdrive/My Drive/b8phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
b = 8
LCC = np.transpose(np.array(conn_comp_crit))
plt.scatter(b-np.array(xRange),LCC[R],marker='+',c='r',label='b=8')
#xCrit = -1*np.log(1-(1-2**(-1*b))**(1/(2*rangeToUse[R])))/2.*np.log(2.)
#plt.axvline(x=xCrit)

#with open('/content/gdrive/My Drive/b7phasePlots', 'rb') as f:
#    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit = pickle.load(f)
#b = 7
#LCC = np.transpose(np.array(conn_comp_crit))
#plt.scatter(b-np.array(xRange),LCC[R]/2**b)

plt.yscale('log')
#plt.xscale('log')
#E_poss = (2**(1+np.array(range(0,11))))
#plt.plot(11-np.array(range(1,12)),1-((E_poss-1)/(E_poss))**(R))
E_poss = 2**(np.array(range(3,11)))
#plt.plot(range(3,11),((E_poss-1)/(E_poss))**(2*R))
plt.ylabel('#CC')
plt.xlabel('b-x')
plt.legend()
```

```python
R = 10

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
b = 11
LCC = np.transpose(np.array(selfmoment))
plt.scatter(b-np.array(xRange),LCC[R])
#plt.scatter(np.array(xRange)/b,LCC[R]/2**b)

with open('/content/gdrive/My Drive/b10_phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
b = 10
LCC = np.transpose(np.array(selfmoment))
plt.scatter(b-np.array(xRange),LCC[R])

with open('/content/gdrive/My Drive/b9phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
b = 9
LCC = np.transpose(np.array(selfmoment))
plt.scatter(b-np.array(xRange),LCC[R])

with open('/content/gdrive/My Drive/b8phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
b = 8
LCC = np.transpose(np.array(selfmoment))
plt.scatter(b-np.array(xRange),LCC[R])

plt.yscale('log')
```

# Resave numComp

```python
b = 11
xRange_local = range(8,9,1)
repeats = 30;
minR = 2
maxR = 160
```

```python
with open('/content/gdrive/My Drive/b11phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
maxLinks = 0.5*2**b*(2**b-1)+2**b
rangeToUse = np.unique([int(i) for i in np.rint(np.logspace(np.log10(minR),np.log10(maxR),50))])
densities = [[] for i in xRange]
conn_comp = [[] for i in xRange]
LCC = [[] for i in xRange]
selfmoment = [[] for i in xRange]
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
```

```python
with open('/content/gdrive/My Drive/b11phasePlots', 'rb') as f:
    selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)

densities.append([])
conn_comp.append([])
LCC.append([])
selfmoment.append([])

for j,x in enumerate(list(xRange_local)):
    print('j = ' + str(j))
    for R in rangeToUse:
        print('R = ' + str(R))
        densities_local = []
        conn_comp_local = []
        LCC_local = []
        selfmoment_local = []

        for reps in range(repeats):
            G = nx.Graph()
            G.add_nodes_from(allPerms)
            for i in range(R):
                Source = allPerms[random.randint(0,2**b-1)]
                Dest = allPerms[random.randint(0,2**b-1)]
                twoLocs_Source = random.sample(range(b),x)
                twoLocs_Dest = random.sample(range(b),x)
                allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                for perms_source in range(2**x):
                    for perms_dest in range(2**x):
                        s_twos = list(allPerms_lengthX_Source[perms_source])
                        d_twos = list(allPerms_lengthX_Dest[perms_dest])
                        G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))
            conn_comp_G = list(nx.connected_components(G))
            conn_comp_local.append(len(conn_comp_G))
            LCC_local.append(len(max(conn_comp_G, key=len)))
            densities_local.append(G.number_of_edges()/maxLinks)
            degrees = list(dict(G.degree()).values())
            DegCounts = Counter(degrees)
            total = sum(DegCounts.values(), 0.0)
            for key in DegCounts:
                DegCounts[key] /= total
            k_net = sum(list(DegCounts.values())*np.array(list(DegCounts.keys()))**2)
            k = G.number_of_edges()/maxLinks*2**b
            selfmoment_local.append(k_net/(k*(k+1)))
        selfmoment[x-1].append(np.mean(selfmoment_local))
        conn_comp[x-1].append(np.mean(conn_comp_local))
        LCC[x-1].append(np.mean(LCC_local))
        densities[x-1].append(np.mean(densities_local))
xRange = range(1,9,1)
with open('/content/gdrive/My Drive/b11phasePlots_resave', 'wb') as f:
  pickle.dump([selfmoment,conn_comp,LCC,densities,xRange,rangeToUse], f)
```

    j = 0
    R = 2
    R = 3
    R = 4
    R = 5
    R = 6
    R = 7
    R = 8
    R = 9
    R = 10
    R = 11
    R = 12
    R = 13
    R = 14
    R = 16
    R = 17
    R = 19
    R = 20
    R = 22
    R = 24
    R = 27
    R = 29
    R = 32
    R = 35
    R = 38
    R = 42
    R = 46
    R = 50
    R = 55
    R = 60
    R = 65
    R = 72
    R = 78
    R = 86
    R = 94
    R = 102
    R = 112
    R = 122
    R = 134
    R = 146
    R = 160

# Save altered CC

```python
b = 11
xRange = range(1,8,1)
repeats = 30;
minR = 2
maxR = 160

maxLinks = 0.5*2**b*(2**b-1)+2**b
rangeToUse = np.unique([int(i) for i in np.rint(np.logspace(np.log10(minR),np.log10(maxR),50))])
conn_comp_crit = [[] for i in xRange]
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]

for j,x in enumerate(list(xRange)):
    print('j = ' + str(j))
    for R in rangeToUse:
        print('R = ' + str(R))
        densities_local = []
        conn_comp_local = []
        LCC_local = []
        selfmoment_local = []

        for reps in range(repeats):
            G = nx.Graph()
            G.add_nodes_from(allPerms)
            for i in range(R):
                Source = allPerms[random.randint(0,2**b-1)]
                Dest = allPerms[random.randint(0,2**b-1)]
                twoLocs_Source = random.sample(range(b),x)
                twoLocs_Dest = random.sample(range(b),x)
                allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                for perms_source in range(2**x):
                    for perms_dest in range(2**x):
                        s_twos = list(allPerms_lengthX_Source[perms_source])
                        d_twos = list(allPerms_lengthX_Dest[perms_dest])
                        G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))
            degrees = G.degree()
            to_keep = [n[0] for n in degrees if n[1] != 0]
            conn_comp_G = list(nx.connected_components(G.subgraph(to_keep)))
            conn_comp_local.append(len(conn_comp_G))
        conn_comp_crit[j].append(np.mean(conn_comp_local))

with open('/content/gdrive/My Drive/b11phasePlots_resave', 'rb') as f:
  selfmoment,conn_comp,LCC,densities,xRange,rangeToUse = pickle.load(f)
with open('/content/gdrive/My Drive/b11phasePlots_resave', 'wb') as f:
  pickle.dump([selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit], f)

b = 11
xRange_local = range(8,9,1)
repeats = 30;
minR = 2
maxR = 160

maxLinks = 0.5*2**b*(2**b-1)+2**b
rangeToUse = np.unique([int(i) for i in np.rint(np.logspace(np.log10(minR),np.log10(maxR),50))])
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]

for j,x in enumerate(list(xRange_local)):
    print('j = ' + str(j))
    for R in rangeToUse:
        print('R = ' + str(R))
        densities_local = []
        conn_comp_local = []
        LCC_local = []
        selfmoment_local = []

        for reps in range(repeats):
            G = nx.Graph()
            G.add_nodes_from(allPerms)
            for i in range(R):
                Source = allPerms[random.randint(0,2**b-1)]
                Dest = allPerms[random.randint(0,2**b-1)]
                twoLocs_Source = random.sample(range(b),x)
                twoLocs_Dest = random.sample(range(b),x)
                allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                for perms_source in range(2**x):
                    for perms_dest in range(2**x):
                        s_twos = list(allPerms_lengthX_Source[perms_source])
                        d_twos = list(allPerms_lengthX_Dest[perms_dest])
                        G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))
            degrees = G.degree()
            to_keep = [n[0] for n in degrees if n[1] != 0]
            conn_comp_G = list(nx.connected_components(G.subgraph(to_keep)))
            conn_comp_local.append(len(conn_comp_G))
        conn_comp_crit[j].append(np.mean(conn_comp_local))

xRange = range(1,9,1)
with open('/content/gdrive/My Drive/b11phasePlots_resave', 'wb') as f:
  pickle.dump([selfmoment,conn_comp,LCC,densities,xRange,rangeToUse,conn_comp_crit], f)
```

# Degree Distribution

```python
DegCounts = []
b = 11
x = 5
R = 70
del DegCounts
for rep in range(100):
  G = nx.Graph()
  allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
  G.add_nodes_from(allPerms)
  start_time = time.time()
  for i in range(R):
      Source = allPerms[random.randint(0,2**b-1)]
      Dest = allPerms[random.randint(0,2**b-1)]
      twoLocs_Source = random.sample(range(b),x)
      twoLocs_Dest = random.sample(range(b),x)
      allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      for perms_source in range(2**x):
          for perms_dest in range(2**x):
              s_twos = list(allPerms_lengthX_Source[perms_source])
              d_twos = list(allPerms_lengthX_Dest[perms_dest])
              G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))

  time.time() - start_time
  degrees = list(dict(G.degree()).values())
  if 'DegCounts' in locals():
    DegCounts =DegCounts+ Counter(degrees)
  else:
    DegCounts = Counter(degrees)
total = sum(DegCounts.values(), 0.0)
for key in DegCounts:
    DegCounts[key] /= total
```

```python
#slow
analysis_lim = 2**b
R_eff = 2*R
k_prob_analytics_slow = []
for k in range(max(DegCounts)):
    p_ri = []
    p_k_given_ri = []

    for R_i in range(1,R_eff+1):
        p_ri.append(np.exp(np.sum(np.log(np.arange(1,R_eff+1)))-sum(np.log(np.arange(1,R_i+1)))-sum(np.log(np.arange(1,(R_eff-R_i+1))))+R_i*np.log(2**(x-b))+(R_eff-R_i)*np.log(1-2**(x-b))))
        p_k_given_ri.append(np.exp(np.sum(np.log(np.arange(1,2**b+1)))-np.sum(np.log(np.arange(1,k+1)))-np.sum(np.log(np.arange(1,2**b-k+1)))+k*np.log(1-(1-2**(x-b))**(R_i))+(R_i*(2**b-k))*np.log(1-2**(x-b))))

    k_prob_analytics_slow.append(sum(np.array(p_ri)*np.array(p_k_given_ri)))
k_prob_analytics_slow[0] = (1-2**((x-b)))**(2*R)
```

```python
keys = sorted(DegCounts.keys(), reverse=False)
result = []
for key in keys:
    result.append(DegCounts[key])
ax = plt.subplot(1,1,1)
plt.scatter(keys,1-np.cumsum(result))

plt.plot(range(max(DegCounts)),1-np.cumsum(k_prob_analytics_slow),c='r')
#plt.plot(est_degs,1-np.cumsum(degrees_probs),c='r')
#ax.set(ylabel = 'p(k)', xlabel = 'k', ylim = [-0.05,1.4*max(DegCounts.values())])
#plt.xlim((-0.5,2**x))
ax.set(ylabel = 'p(k)', xlabel = 'k')
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
plt.savefig('/content/gdrive/My Drive/degree_5_70.pdf')
```

# Density

```python
b = 12
xRange = range(2,7,2)
repeats = 20;
minR = 100
maxR = 700
spacing = 40
```

```python
maxLinks = 0.5*2**b*(2**b-1)+2**b
rangeToUse = [int(i) for i in np.rint(np.logspace(np.log10(minR),np.log10(maxR),30))]
densities = [[] for i in xRange]
exp_densities = [[] for i in xRange]
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
```

```python
for j,x in enumerate(list(xRange)):
    print('j = ' + str(j))
    for R in rangeToUse:
        print('R = ' + str(R))
        densities_local = []
        exp_densities[j].append(1-(1-2**(2*(x-b)))**(2*R))

        for reps in range(repeats):
            G = nx.Graph()
            G.add_nodes_from(allPerms)
            for i in range(R):
                Source = allPerms[random.randint(0,2**b-1)]
                Dest = allPerms[random.randint(0,2**b-1)]
                twoLocs_Source = random.sample(range(b),x)
                twoLocs_Dest = random.sample(range(b),x)
                allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
                for perms_source in range(2**x):
                    for perms_dest in range(2**x):
                        s_twos = list(allPerms_lengthX_Source[perms_source])
                        d_twos = list(allPerms_lengthX_Dest[perms_dest])
                        G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))

            densities_local.append(G.number_of_edges()/maxLinks)
        densities[j].append(np.mean(densities_local))
```

    j = 0
    R = 100
    R = 107
    R = 114
    R = 122
    R = 131
    R = 140
    R = 150
    R = 160
    R = 171
    R = 183
    R = 196
    R = 209
    R = 224
    R = 239
    R = 256
    R = 274
    R = 293
    R = 313
    R = 335
    R = 358
    R = 383
    R = 409
    R = 438
    R = 468
    R = 500
    R = 535
    R = 572
    R = 612
    R = 655
    R = 700
    j = 1
    R = 100
    R = 107
    R = 114
    R = 122
    R = 131
    R = 140
    R = 150
    R = 160
    R = 171
    R = 183
    R = 196
    R = 209
    R = 224
    R = 239
    R = 256
    R = 274
    R = 293
    R = 313
    R = 335
    R = 358
    R = 383
    R = 409
    R = 438
    R = 468
    R = 500
    R = 535
    R = 572
    R = 612
    R = 655
    R = 700
    j = 2
    R = 100
    R = 107
    R = 114
    R = 122
    R = 131
    R = 140
    R = 150
    R = 160
    R = 171
    R = 183
    R = 196
    R = 209
    R = 224
    R = 239
    R = 256
    R = 274
    R = 293
    R = 313
    R = 335
    R = 358
    R = 383
    R = 409
    R = 438
    R = 468
    R = 500
    R = 535
    R = 572
    R = 612
    R = 655
    R = 700

```python
plt.scatter(list(rangeToUse),densities,label = 'x = 6')
plt.plot(list(rangeToUse),exp_densities,'r',linewidth = 2.5)
plt.ylabel('Density')
plt.legend(loc = 'upper left')
plt.xlabel('Number of Rules')
```

```python
ax = plt.subplot(111)

for j,x in enumerate(list(xRange)):
    ax.scatter(list(rangeToUse),densities[j],label = 'x = '+ str(x),LineWidth = 3)
    ax.plot(list(rangeToUse),exp_densities[j],'k',linewidth = 2.5)
ax.set(ylabel = 'Density', xlabel = 'Number of Rules',yscale = 'log',xscale = 'log')
#ax.ylabel('Density')
#plt.legend(loc = 'upper left')
#ax.xlabel('Number of Rules')
#ax.yscale('log')
#ax.xscale('log')
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
plt.savefig('Fix_X_Density.pdf')
```

# Analytic Density

```python
# Fix b and increase x and R

b = 12
XRange = range(0,13)
R_Range = range(1,10000)
G_DensStorer = [[] for i in range(len(XRange))]

for x in XRange:
    for R in R_Range:
        G_DensStorer[x-1].append(1-(1-2**(2*(x-b)))**(2*R))

for x in XRange:
    plt.plot(R_Range,G_DensStorer[x-1], linewidth=2, label='x = ' + str(x))
plt.xscale('log')
plt.yscale('log')
plt.xlabel('Number of Rules');
plt.ylabel('Density');
plt.legend()
```

# Robustness

## Random Attack

```python
# RANDOM ATTACKS

rand_giant_comps_tot = []
giant_comp_size_tot = []
graph_sizes = []

repnum = 100
z = plt.subplot(1,1,1)

for rep in range(repnum):
  start_time = time.time()

  # Redo Graphs at each time point?

  b = 11
  x = 3
  R = 15

  G = nx.Graph()
  allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
  #G.add_nodes_from(allPerms)
  start_time = time.time()
  for i in range(R):
      Source = allPerms[random.randint(0,2**b-1)]
      Dest = allPerms[random.randint(0,2**b-1)]
      twoLocs_Source = random.sample(range(b),x)
      twoLocs_Dest = random.sample(range(b),x)
      allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      for perms_source in range(2**x):
          for perms_dest in range(2**x):
              s_twos = list(allPerms_lengthX_Source[perms_source])
              d_twos = list(allPerms_lengthX_Dest[perms_dest])
              G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))

  time.time() - start_time
  G_rand = nx.gnm_random_graph(G.number_of_nodes(), G.number_of_edges())
  # End of experiment

  #rand_giant_comps = []
  #giant_comp_size = []
  rand_giant_comps = [0]*(2**b)
  giant_comp_size = [0]*(2**b)

  Gcc = sorted(nx.connected_components(G), key=len, reverse=True)
  G_breakdown = G.subgraph(Gcc[0]).copy()
  Gcc = sorted(nx.connected_components(G_rand), key=len, reverse=True)
  G_rand_breakdown = G_rand.subgraph(Gcc[0]).copy()

  init_size = G_breakdown.number_of_nodes()
  graph_sizes.append(init_size)

  rand_nodes_available = list(G_rand_breakdown.nodes())
  nodes_available = list(G_breakdown.nodes())

  initial_rand_nodes = len(rand_nodes_available)
  initial_nodes = len(nodes_available)

  #rand_giant_comps.append(len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/len(rand_nodes_available))
  #giant_comp_size.append(len(max(list(nx.connected_components(G_breakdown)), key=len))/len(nodes_available))
  rand_giant_comps[0] = len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/initial_rand_nodes
  giant_comp_size[0] = len(max(list(nx.connected_components(G_breakdown)), key=len))/initial_nodes

  for i in range(1,(init_size)):
    which_remove = random.choice(nodes_available)
    nodes_available.remove(which_remove)
    G_breakdown.remove_node(which_remove)
    which_remove = random.choice(rand_nodes_available)
    rand_nodes_available.remove(which_remove)
    G_rand_breakdown.remove_node(which_remove)
    if i%1 == 0:
      giant_comp_size[i] = len(max(list(nx.connected_components(G_breakdown)), key=len))/initial_nodes
      rand_giant_comps[i] = len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/initial_rand_nodes
      #giant_comp_size.append(len(max(list(nx.connected_components(G_breakdown)), key=len))/len(nodes_available))
      #rand_giant_comps.append(len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/len(rand_nodes_available))
  rand_giant_comps_tot.append(rand_giant_comps)
  giant_comp_size_tot.append(giant_comp_size)
  plt.plot(np.array(range(0,initial_rand_nodes,1))/initial_rand_nodes,np.array(rand_giant_comps_tot[rep][0:initial_rand_nodes]),color = (241./255,158./255,57./255)) # ER is Orange
  plt.plot(np.array(range(0,initial_nodes,1))/initial_nodes,np.array(giant_comp_size_tot[rep][0:initial_nodes]),color = (51./255,50./255,246./255)) # RG is blue
  if rep % 10 == 0:
    print(time.time() - start_time)

#z.set_yticklabels([])
z.set_xticklabels([])
z.set_yticks([0,0.5,1])
z.set_xticks([0,0.5,1])
z.spines['top'].set_visible(False)
z.spines['right'].set_visible(False)

plt.savefig('/content/gdrive/My Drive/SI_top_left.pdf')
```

## Targeted Attack

```python
# Targeted Attack

rand_giant_comps_tot = []
giant_comp_size_tot = []
graph_sizes = []
repnum = 100
z = plt.subplot(1,1,1)
for rep in range(repnum):
  start_time = time.time()

  # Test whether we need to reinitialize graph!
  b = 11
  x = 3
  R = 15

  G = nx.Graph()
  allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
  #G.add_nodes_from(allPerms)
  start_time = time.time()
  for i in range(R):
      Source = allPerms[random.randint(0,2**b-1)]
      Dest = allPerms[random.randint(0,2**b-1)]
      twoLocs_Source = random.sample(range(b),x)
      twoLocs_Dest = random.sample(range(b),x)
      allPerms_lengthX_Source = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      allPerms_lengthX_Dest = ["".join(seq) for seq in itertools.product("01", repeat=x)]
      for perms_source in range(2**x):
          for perms_dest in range(2**x):
              s_twos = list(allPerms_lengthX_Source[perms_source])
              d_twos = list(allPerms_lengthX_Dest[perms_dest])
              G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]))

  time.time() - start_time
  G_rand = nx.gnm_random_graph(G.number_of_nodes(), G.number_of_edges())
  # End of test

  rand_giant_comps = [0]*(2**b)
  giant_comp_size = [0]*(2**b)

  Gcc = sorted(nx.connected_components(G), key=len, reverse=True)
  G_breakdown = G.subgraph(Gcc[0]).copy()
  Gcc = sorted(nx.connected_components(G_rand), key=len, reverse=True)
  G_rand_breakdown = G_rand.subgraph(Gcc[0]).copy()

  init_size = G_breakdown.number_of_nodes()
  graph_sizes.append(init_size)

  rand_nodes_available = list(G_rand_breakdown.nodes())
  nodes_available = list(G_breakdown.nodes())

  initial_rand_nodes = len(rand_nodes_available)
  initial_nodes = len(nodes_available)

  #rand_giant_comps.append(len(max(list(nx.connected_components(G_rand_breakdown)), key=len)))
  #giant_comp_size.append(len(max(list(nx.connected_components(G_breakdown)), key=len)))
  rand_giant_comps[0] = len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/initial_rand_nodes
  giant_comp_size[0] = len(max(list(nx.connected_components(G_breakdown)), key=len))/initial_nodes

  for i in range(1,init_size):
    degree_dict = dict(G_breakdown.degree())
    max_val = max(degree_dict.values())
    which_remove = random.choice([key for key in degree_dict if degree_dict[key] == max_val])
    nodes_available.remove(which_remove)
    G_breakdown.remove_node(which_remove)
    degree_dict = dict(G_rand_breakdown.degree())
    max_val = max(degree_dict.values())
    which_remove = random.choice([key for key in degree_dict if degree_dict[key] == max_val])
    rand_nodes_available.remove(which_remove)
    G_rand_breakdown.remove_node(which_remove)
    if i%1 == 0:
      giant_comp_size[i] = len(max(list(nx.connected_components(G_breakdown)), key=len))/initial_nodes
      rand_giant_comps[i] = len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/initial_rand_nodes
      #giant_comp_size.append(len(max(list(nx.connected_components(G_breakdown)), key=len))/len(nodes_available))
      #rand_giant_comps.append(len(max(list(nx.connected_components(G_rand_breakdown)), key=len))/len(rand_nodes_available))
  rand_giant_comps_tot.append(rand_giant_comps)
  giant_comp_size_tot.append(giant_comp_size)
  plt.plot(np.array(range(0,initial_rand_nodes,1))/initial_rand_nodes,np.array(rand_giant_comps_tot[rep][0:initial_rand_nodes]),color = (241./255,158./255,57./255))
  plt.plot(np.array(range(0,initial_nodes,1))/initial_nodes,np.array(giant_comp_size_tot[rep][0:initial_nodes]),color = (51./255,50./255,246./255))
  if rep % 10 == 0:
    print(time.time() - start_time)

#z.set_yticklabels([])
#z.set_xticklabels([])
z.set_yticks([0,0.5,1])
z.set_xticks([0,0.5,1])

z.spines['top'].set_visible(False)
z.spines['right'].set_visible(False)

plt.savefig('/content/gdrive/My Drive/SI_bottom_left.pdf')
```

# Visualization for Overlap

```python
b = 7
x = 4
R = 2
```

```python
G = nx.Graph()
allPerms_lengthX = ["".join(seq) for seq in itertools.product("01", repeat=x)]
start_time = time.time()
Sources = ['1010000','1010000']
Source_ZeroLoc = [3,4,5,6]
Dests = ['010000000','010000000']
Dest_ZeroLoc = [[3,4,7,8],[5,6,7,8]]
for i in range(R):
    Source = Sources[i]
    Dest = Dests[i]
    twoLocs_Dest = Dest_ZeroLoc[i]
    for perms_source in range(2**x):
        for perms_dest in range(2**x):
            s_twos = list(allPerms_lengthX[perms_source])
            d_twos = list(allPerms_lengthX[perms_dest])
            G.add_edge(''.join([s_twos.pop() if index in Source_ZeroLoc else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]),label = str(i))

i = 2
twoLocs_Dest = [7,8]
allPerms_length2 = ["".join(seq) for seq in itertools.product("01", repeat=2)]
for perms_source in range(2**x):
        for perms_dest in range(2**2):
            s_twos = list(allPerms_lengthX[perms_source])
            d_twos = list(allPerms_length2[perms_dest])
            G.add_edge(''.join([s_twos.pop() if index in Source_ZeroLoc else bit for index,bit in enumerate(Source)]),''.join([d_twos.pop() if index in twoLocs_Dest else bit for index,bit in enumerate(Dest)]),label = str(i))

time.time() - start_time
```

    0.0037665367126464844

```python
perms_dest
```

    3

```python
len(G.nodes)
```

    44

```python
allPerms_lengthX[perms_dest]
```

    '0011'

```python
d = nx.get_edge_attributes(G,'label')
counts = Counter(chain.from_iterable(d[e] for e in d))
counts
```

    Counter({'0': 192, '1': 192, '2': 64})

```python
nx.write_edgelist(G,'/content/gdrive/My Drive/Overlap.csv')
```

# Homophily Heuristic

```python
b = 7
x = 1
G = nx.Graph()
allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
G.add_nodes_from(allPerms)
start_time = time.time()
ZeroLocs = list(itertools.combinations(list(range(b)),x))
allPerms_lengthX = ["".join(seq) for seq in itertools.product("01", repeat=x)]
allPerms_lengthBMinusX = ["".join(seq) for seq in itertools.product("01", repeat=b-x)]

for i in range(len(allPerms_lengthBMinusX)):
  for j in range(len(ZeroLocs)):
    Source = allPerms_lengthBMinusX[i]
    Dest = allPerms_lengthBMinusX[i]
    twoLocs_Source = ZeroLocs[j]
    twoLocs_Dest = ZeroLocs[j]
    allPerms_lengthX_Source = allPerms_lengthX
    allPerms_lengthX_Dest = allPerms_lengthX
    for perms_source in range(2**x):
        for perms_dest in range(2**x):
            s_twos = list(allPerms_lengthX_Source[perms_source])
            Source_temp = list(Source)
            d_twos = list(allPerms_lengthX_Dest[perms_dest])
            Dest_temp = list(Dest)
            G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else Source_temp.pop(0) for index,bit in enumerate(allPerms[0])]),''.join([d_twos.pop() if index in twoLocs_Dest else Dest_temp.pop(0) for index,bit in enumerate(allPerms[0])]))

time.time() - start_time

```

    0.014813423156738281

```python
Density_Meta = {}
Clustering_Meta = {}
Diameter_Meta = {}
Radius_Meta = {}
path_Meta = {}
for b in range(1,20):

  maxLinks = 0.5*2**b*(2**b-1)+2**b
  start_time = time.time()

  Densities = []
  Clustering = []
  Diameters = []
  avgPath = []
  Radii = []
  for x in range(b+1):
    G = nx.Graph()
    allPerms = ["".join(seq) for seq in itertools.product("01", repeat=b)]
    G.add_nodes_from(allPerms)

    ZeroLocs = list(itertools.combinations(list(range(b)),x))
    allPerms_lengthX = ["".join(seq) for seq in itertools.product("01", repeat=x)]
    allPerms_lengthBMinusX = ["".join(seq) for seq in itertools.product("01", repeat=b-x)]

    for i in range(len(allPerms_lengthBMinusX)):
      for j in range(len(ZeroLocs)):
        Source = allPerms_lengthBMinusX[i]
        Dest = allPerms_lengthBMinusX[i]
        twoLocs_Source = ZeroLocs[j]
        twoLocs_Dest = ZeroLocs[j]
        allPerms_lengthX_Source = allPerms_lengthX
        allPerms_lengthX_Dest = allPerms_lengthX
        for perms_source in range(2**x):
            for perms_dest in range(2**x):
                s_twos = list(allPerms_lengthX_Source[perms_source])
                Source_temp = list(Source)
                d_twos = list(allPerms_lengthX_Dest[perms_dest])
                Dest_temp = list(Dest)
                G.add_edge(''.join([s_twos.pop() if index in twoLocs_Source else Source_temp.pop(0) for index,bit in enumerate(allPerms[0])]),''.join([d_twos.pop() if index in twoLocs_Dest else Dest_temp.pop(0) for index,bit in enumerate(allPerms[0])]))
    if x > 0:
      #eccentricities = nx.eccentricity(G)
      #Diameters.append(nx.diameter(G,e = eccentricities))
      #Radii.append(nx.radius(G,e = eccentricities))
      avgPath.append(nx.average_shortest_path_length(G))
    Densities.append(G.number_of_edges()/maxLinks)
    Clustering.append(nx.average_clustering(G))
  print(time.time() - start_time)
  Density_Meta[b] = Densities
  Clustering_Meta[b] = Clustering
  #Diameter_Meta[b] = Diameters
  #Radius_Meta[b] = Radii
  path_Meta[b] = avgPath
  with open('/content/gdrive/My Drive/homiphilies2', 'wb') as f:
    pickle.dump([Density_Meta,Clustering_Meta,path_Meta], f)

```

    0.00047659873962402344
    0.0007736682891845703
    0.006624460220336914
    0.013876199722290039
    0.0829918384552002
    0.5804073810577393
    3.628187417984009
    26.228145837783813
    272.18391394615173
    2717.341167449951

```python
with open('/content/gdrive/My Drive/homiphilies', 'rb') as f:
    Density_Meta,Clustering_Meta = pickle.load(f)
```

```python
for k, v in Density_Meta.items():
  if k > 3:
    plt.plot(1-np.array(range(len(v)))/k,v,label = k)

plt.legend()
plt.legend()
plt.xlabel("o/b")
plt.ylabel("Density")
plt.savefig('/content/gdrive/My Drive/HomiphilyDensity.pdf')
```

```python
for k, v in Clustering_Meta.items():
  if k > 3:
    plt.plot(1-np.array(range(len(v)))/k,v,label = k)

plt.legend()
plt.legend()
plt.xlabel("o/b")
plt.ylabel("Average Clustering")
plt.savefig('/content/gdrive/My Drive/HomiphilyClustering.pdf')
```

```python
with open('/content/gdrive/My Drive/homiphilies2', 'rb') as f:
    Density_Meta,Clustering_Meta,path_Meta = pickle.load(f)

```

```python
for k, v in Density_Meta.items():
  if k > 3:
    plt.plot(1-np.array(range(len(v)))/k,v,label = k)

plt.legend()
plt.xlabel("o/b")
plt.ylabel("Density")
```

```python
for k, v in Clustering_Meta.items():
  if k > 3:
    plt.plot(1-np.array(range(len(v)))/k,v,label = k)

plt.legend()
plt.xlabel("o/b")
plt.ylabel("Average Clustering")
```

```python
for k, v in path_Meta.items():
    plt.plot(1-np.array(range(1,k+1))/k,np.array(v),label = k)

plt.legend()
plt.xlabel("o/b")
plt.ylabel("Average Path Length")
plt.savefig('/content/gdrive/My Drive/HomiphilyPL.pdf')
```

```python

```


# "FAVAR - BBE(2005) Replication in R"
> "Replication of Bernanke, Boivin and Eliasz (QJE, 2005) in R. This code is written to facilitate the undertanding of FAVAR rather than to be efficient"

- toc:true
- branch: master
- badges: true
- comments: false
- author: João B. Duarte
- categories: [Time Series, R, FAVAR]

The FAVAR model is represented by the following system:

$$
\begin{align}
X_t &= \Lambda^f F_t + \Lambda^y Y_t + e_t \\ \\
\begin{bmatrix} F_t \\ Y_t \end{bmatrix} &= \Psi(L)\begin{bmatrix} F_{t-1} \\ Y_{t-1} \end{bmatrix} + v_t
\end{align}
$$

where $F_t$ is the collection of variables that is unobservable, while $Y_t$ is is of the observable. For further description of the FAVAR model and its properties please see BBE (QJE, 2005)

```R
library(readxl)
library(boot)
library(tsDyn)
library(vars)
library(repr)
```

```R
# Get original large dataset of BBE (2005): 120 series
data = read_excel("./Data/bbe_data.xlsx")
```

```R
head(data)
```

Date	IPP	IPF	IPC	IPCD	IPCN	IPE	IPI	IPM	IPMD	...	PU85	PUC	PUCD	PUS	PUXF	PUXHS	PUXM	LEHCC	LEHM	HHSNTN

1959:01     	0.013396992 	0.008609752 	 0.007316259	0.005227654 	 0.009517142	0.013284200 	 0.018846549	 0.031175802	 0.045049989	...         	0.004728141 	 0.000000000	0.000000000 	0.004357305 	0.000000000 	0.000000000 	0.000000000 	 0.003478264	0.004618946 	95.8

1959:02     	0.006022800 	0.004916504 	 0.000000000	0.019405634 	-0.004747249	0.010731414 	 0.013885375	 0.025638933	 0.038650015	...         	0.004705891 	-0.003007521	0.005235614 	0.000000000 	0.000000000 	0.000000000 	0.000000000 	-0.003478264	0.009174376 	96.4

1959:03     	0.014342738 	0.014547535 	 0.015653159	0.006396823 	 0.016505699	0.025817364 	 0.015959619	 0.027254131	 0.029650367	...         	0.000000000 	 0.000000000	0.000000000 	0.004338402 	0.003454235 	0.000000000 	0.000000000 	 0.006944472	0.004555816 	96.9

1959:04     	0.008286750 	0.009562755 	 0.004755444	0.020121006 	 0.000000000	0.031893735 	 0.005638694	 0.025433102	 0.034050126	...         	0.004683849 	 0.003007521	0.002607563 	0.004319661 	0.003442344 	0.000000000 	0.000000000 	 0.006896579	0.000000000 	97.5

1959:05     	0.007036611 	0.007141339 	-0.004755444	0.007458365 	-0.007033631	0.023250403 	 0.003368017	-0.006587149	-0.007362857	...         	0.004662013 	 0.000000000	0.000000000 	0.000000000 	0.003430535 	0.003252035 	0.003372685 	 0.003430535	0.004535155 	97.2

1959:06     	0.001153906 	0.008234277 	 0.013062217	0.019610667 	 0.008209100	0.002193842 	-0.013540671	-0.061158362	-0.105490860	...         	0.000000000 	 0.000000000	0.002600782 	0.004301082 	0.000000000 	0.003241494 	0.003361348 	 0.003418807	0.000000000 	96.9

```R
# Standardizing data = all variables with mean 0 and standard deviation 1.
# This step is crucial in PC analysis
data_s = scale(data[,2:121], center = TRUE, scale = TRUE)
```

## Step 1: Extract principal componentes of all X (including Y)

```R
pc_all = prcomp(data_s, center=FALSE, scale.=FALSE, rank. = 3)
# summary(pc_all)
C = pc_all$x # saving the principal components
```

## Step 2: Extract principal componentes of Slow Variables

```R
# Slow Variables
slow_vars = c("IP", "LHUR", "PUNEW", "IPP", "IPF", "IPC", "IPCD", "IPCN", "IPE", "IPI", "IPM",
             "IPMD", "IPMND", "IPMFG", "IPD", "IPN", "IPMIN", "IPUT", "IPXMCA", "PMI", "PMP",
             "GMPYQ", "GMYXPQ", "LHEL", "LHELX", "LHEM", "LHNAG", "LHU680", "LHU5", "LHU14",
             "LHU15", "LHU26", "LPNAG", "LP", "LPGD", "LPMI", "LPCC", "LPEM", "LPED", "LPEN",
             "LPSP", "LPTU", "LPT", "LPFR", "LPS", "LPGOV", "LPHRM", "LPMOSA", "PMEMP", "GMCQ",
             "GMCDQ", "GMCNQ", "GMCSQ", "GMCANQ", "PWFSA", "PWFCSA", "PWIMSA", "PWCMSA", "PSM99Q",
             "PU83", "PU84", "PU85", "PUC", "PUCD", "PUS", "PUXF", "PUXHS", "PUXM", "LEHCC", "LEHM")
```

```R
data_slow = data_s[, slow_vars]
pc_slow = prcomp(data_slow, center=FALSE, scale.=FALSE, rank. = 3)
F_slow = pc_slow$x
```

## Step 3: Clean the PC from the effect of observed Y

```R
# Next clean the PC of all space from the observed Y
reg = lm(C ~ F_slow + data_s[,"FYFF"])
#summary(reg)
F_hat = C - data.matrix(data_s[,"FYFF"])%*%reg$coefficients[5,] # cleaning and saving F_hat
```

## Step 4: Estimate FAVAR and get IRFs

> Warning: The IRFs in BBE are reported in standard deviation units, which means they are reported in the scaled data. No need to scale the data back to original units if want to compare with their paper.

```R
data_var = data.frame(F_hat, "FYFF" = data_s[,"FYFF"])
var = VAR(data_var, p = 13)
#summary(var)

irf_point = irf(var, n.ahead = 48, impulse = "FYFF", response = "FYFF", boot = FALSE)

# Shock size of 25 basis points
impulse_sd = 0.25/sd(data$FYFF)
scale = impulse_sd/(irf_point$irf$FYFF[1]) # position of FYFF response at step 0

# Computing Loading Factors
reg_loadings = lm(data_s ~ 0 + F_hat + data_s[,"FYFF"])
loadings = reg_loadings$coefficients
# head(reg_loadings$coefficients)
#summary(reg_loadings)

#### BOOTSTRAPING ########

R = 500 # Number of simulations
nvars = 120 # Number of variables
nsteps = 49 # numbers of steps

IRFs = array(c(0,0,0), dim = c(nsteps,nvars,R))

var = lineVar(data_var, lag = 13, include = "const")
for(j in 1:R){
    data_boot = VAR.boot(var, boot.scheme ="resample")
    var_boot = VAR(data_boot, lag = 13)
    irf1 = irf(var_boot, n.ahead = 48, impulse = "FYFF", boot = FALSE)
    for(i in 1:nvars){
        IRFs[,i,j] = (irf1$irf$FYFF %*% matrix(loadings[1:4, i]))*scale
        }
} ## Boot simulations done

# Extract the quantiles of IRFs we are interested: 90% confidence intervals in BBE
Upper = array(c(0,0), dim = c(nsteps, nvars))
for(k in 1:nsteps){
    for(i in 1:nvars){
        Upper[k,i] = quantile(IRFs[k,i,], probs = c(0.95))[1]
        }
}
Lower = array(c(0,0), dim = c(nsteps, nvars))
for(k in 1:nsteps){
    for(i in 1:nvars){
        Lower[k,i] = quantile(IRFs[k,i,], probs = c(0.05))[1]
        }
}
IRF = array(c(0,0), dim = c(nsteps, nvars))
for(k in 1:nsteps){
    for(i in 1:nvars){
        IRF[k,i] = quantile(IRFs[k,i,], probs = c(0.5))[1]
        }
}
rm(var_boot)
rm(IRFs)
```

```R
# Select the Variables you are Interested in
# List of variables we are interested: FYFF, IP, CPI
variables = c(grep("^FYFF$", colnames(data_s)), grep("^IP$", colnames(data_s)), grep("^PUNEW$", colnames(data_s)),
            grep("^FYGM3$", colnames(data_s)), grep("^FYGT5$", colnames(data_s)), grep("^FMFBA$", colnames(data_s)),
            grep("^FM2$", colnames(data_s)), grep("^EXRJAN$", colnames(data_s)), grep("^PMCP$", colnames(data_s)),
            grep("^IPXMCA$", colnames(data_s)), grep("^GMCQ$", colnames(data_s)), grep("^GMCDQ$", colnames(data_s)),
            grep("^GMCNQ$", colnames(data_s)), grep("^LHUR$", colnames(data_s)), grep("^PMEMP$", colnames(data_s)),
            grep("^LEHM$", colnames(data_s)), grep("^HSFR$", colnames(data_s)), grep("^PMNO$", colnames(data_s)),
            grep("^FSDXP$", colnames(data_s)), grep("^HHSNTN$", colnames(data_s))
             )

transf_code = c(1, 5, 5,
             1, 1, 5,
             5, 5, 1,
             1, 5, 5,
             5, 1, 1,
             5, 1, 1,
             1, 1
               )

variable_names = c("Fed Funds Rate", "Industrial Production", "CPI",
                "3m Treasury Bills", "5y Treasury Bonds", "Monetary Base",
             "M2", "Exchange Rate Yen", "Commodity Price Index",
             "Capacity Util Rate", "Personal Consumption", "Durable Cons",
             "Nondurable Cons", "Unemployment", "Employment",
             "Avg Hourly Earnings", "Housing Starts", "New Orders",
             "Dividends", "Consumer Expectations"
               )
```

```R
# Replicating Figure II in BBE (2005) - 3 Factors and Y = FYFF
# Change plot size to 15 x 10
options(repr.plot.width=12, repr.plot.height=8)

par(mfrow=c(5,4),
   mar = c(2, 2, 2, 2))

for(i in variables){
    index = which(variables == i)
    if(transf_code[index] == 5){
        plot(cumsum(IRF[,i]), type ='l',lwd=2, main = variable_names[index],
             ylab= "", xlab="Steps", ylim=range(cumsum(Lower[,i]),cumsum(Upper[,i])),
             cex.main=1.8, cex.axis=1.3)
        lines(cumsum(Upper[,i]), lty=2, col="red")
        lines(cumsum(Lower[,i]), lty=2, col="red")
        abline(h=0)
    }
    else{
        plot(IRF[,i], type ='l',lwd=2, main = variable_names[index],
             ylab= "", xlab="Steps", ylim=range((Lower[,i]),(Upper[,i])),
             cex.main=1.8, cex.axis=1.3)
        lines((Upper[,i]), lty=2, col="red")
        lines((Lower[,i]), lty=2, col="red")
        abline(h=0)
    }
}
```

### How much of the variation is captured by the Factors and FYFF?

```R
options(repr.plot.width=12, repr.plot.height=6)

par(mfrow=c(1,2),
   mar = c(2, 2, 2, 2))

plot(data_s[, variables[2]], type = "l", ylab = "", main = "Industrial Production")
lines(fitted(reg_loadings)[,variables[2]], lty=2, col="red")
legend(300,6, legend=c("Data", "Predicted by PC"), lty = 1:2, col = c("black", "red"),
      box.lty=0)
plot(data_s[, variables[3]], type = "l", ylab = "", main = "CPI")
lines(fitted(reg_loadings)[,variables[3]], lty=2, col="red")
```

## Step 5: FEVD

One step-ahead Variance Decomposition of $X_t$:

$$
E_t[X_{t+1} - X_t] =  \Lambda^f E_t[F_{t+1} - F_t]  +  \Lambda^y E_t [Y_{t+1} - Y_t] +  e_{t+1}
$$

$$
F_t = \mu_f + L^1 \varepsilon_t + A_1 L^1 \varepsilon_t + ...
$$

$$
Y_t = \mu_y + L^2 \varepsilon_t + A_1 L^2 \varepsilon_t + ...
$$

where $L = \begin{bmatrix} L^1 \\ L^2 \end{bmatrix}$ is the lower triangle matrix from the Cholesky decomposition. Let $\Lambda = \begin{bmatrix} \Lambda^f \\ \Lambda^y \end{bmatrix}$, we have that:

$$
E_t[X_{t+1} - X_t] =  \Lambda L E_t[ \varepsilon_{t+1}]  +  E_t[e_{t+1}]
$$

$$
E_t[X_{t+1} - X_t] =  \Psi(0) E_t[ \varepsilon_{t+1}] +  E_t[e_{t+1}]
$$

$$
var(E_t[X_{t+1} - X_t]) =  \Psi(0) \Sigma_\varepsilon (\Psi(0)))'  +  \sigma^2_e
$$

where $\Sigma_\varepsilon = I $ because of the Cholesky decomposition, and $\Psi(0)$ is a 1x4 vector with the contemporaneous response of X_t to all structural shocks. Hence, the total variance of each observable variable $X_t$ is just the sum of the squared impulse response functions plus the variance of the measurement error.

At step-ahead $j$:

$$
var(E_t[X_{t+j} - X_t]) =  \sum_{i=0}^j \Psi(i)(\Psi(i)))'  +  \sigma^2_e
$$

The variance of $E_t[X_{t+j} - X_t]$ that comes from the monetary policy shock alone is given by:

$$
var(E_t[X_{t+j} - X_t|\varepsilon_F, e_t]) =  \sum_{i=0}^j \Psi_{MP}(i)(\Psi(i)_{MP}))'
$$

where $\Psi(i)_{MP} = \Psi(i)(1,4)$, i.e. the impulse response to monetary policy shock. Hence, the contribution of monetary policy shock to the $j$ step-ahead forecast error variance is given by:

$$
{\text{FEVD}}(j)_{MP}^X = \frac{var(E_t[X_{t+j} - X_t|\varepsilon_F, e_t])}{var(E_t[X_{t+j} - X_t])}
$$

```R
# Get the VAR point estimates
hor = 60
var = VAR(data_var, p = 13)
irf_point = irf(var, n.ahead = hor, boot = FALSE)
```

```R
# Get IRFs for all of X we are interested in, Dimensions: (hor, key_nvars)
# Find loadings
results = summary(reg_loadings) # the warning comes because of FYFF

key_nvars = length(variables)
irf_X_pc1 = array(c(0,0), dim=c(hor+1, key_nvars))
irf_X_pc2 = array(c(0,0), dim=c(hor+1, key_nvars))
irf_X_pc3 = array(c(0,0), dim=c(hor+1, key_nvars))
irf_X_fyff = array(c(0,0), dim=c(hor+1, key_nvars))

for(i in 1:key_nvars){
    irf_X_pc1[,i] = irf_point$irf$PC1 %*% matrix(loadings[1:4, variables[i]])
    irf_X_pc2[,i] = irf_point$irf$PC2 %*% matrix(loadings[1:4, variables[i]])
    irf_X_pc3[,i] = irf_point$irf$PC3 %*% matrix(loadings[1:4, variables[i]])
    irf_X_fyff[,i] = (irf_point$irf$FYFF) %*% matrix(loadings[1:4, variables[i]])
    }
```

    Warning message in summary.lm(object, ...):
    “essentially perfect fit: summary may be unreliable”

```R
# Get the IRFs squared and accumulate them
psi2_pc1 = array(0, dim = key_nvars)
psi2_pc2 = array(0, dim = key_nvars)
psi2_pc3 = array(0, dim = key_nvars)
psi2_fyff = array(0, dim = key_nvars)

for(i in 1:key_nvars){
    for(j in 1:hor){
        psi2_pc1[i] = psi2_pc1[i] + irf_X_pc1[j,i]^2
        psi2_pc2[i] = psi2_pc2[i] + irf_X_pc2[j,i]^2
        psi2_pc3[i] = psi2_pc3[i] + irf_X_pc3[j,i]^2
        psi2_fyff[i] = psi2_fyff[i] + irf_X_fyff[j,i]^2
    }
}
```

```R
var_total= array(0, dim = key_nvars)
var_fac= array(0, dim = key_nvars)
var_e= array(0, dim = key_nvars)

for(i in 1:key_nvars){
    var_fac[i] = psi2_pc1[i] + psi2_pc2[i] + psi2_pc3[i] + psi2_fyff[i]
    var_total[i] = psi2_pc1[i] + psi2_pc2[i] + psi2_pc3[i] + psi2_fyff[i] + results[[variables[i]]]$sigma^2
    var_e[i] = results[[variables[i]]]$sigma^2
}
```

```R
table = data.frame("PC1" = round((psi2_pc1),3), "PC2" = round((psi2_pc2),3), "PC3" = round((psi2_pc3),3), "FYFF" = round((psi2_fyff),3),
                  "Factor_Y_total" = round(var_fac,3) ,"e" = round((var_e),3), "Total" = round(var_total,3))
row.names(table) = variable_names
table
```

	PC1	PC2	PC3	FYFF	Factor_Y_total	e	Total

Fed Funds Rate	0.497	0.279	0.059	0.150	0.985	0.000	0.985

Industrial Production	0.550	0.037	0.054	0.147	0.787	0.255	1.042

CPI	0.155	0.701	0.019	0.117	0.991	0.138	1.129

3m Treasury Bills	0.456	0.281	0.048	0.130	0.914	0.025	0.939

5y Treasury Bonds	0.299	0.309	0.015	0.087	0.710	0.068	0.778

Monetary Base	0.017	0.056	0.010	0.005	0.087	0.900	0.988

M2	0.017	0.014	0.007	0.005	0.042	0.953	0.995

Exchange Rate Yen	0.016	0.003	0.005	0.005	0.028	0.981	1.010

Commodity Price Index	0.250	0.272	0.032	0.154	0.709	0.362	1.071

Capacity Util Rate	0.365	0.069	0.057	0.146	0.637	0.252	0.889

Personal Consumption	0.057	0.044	0.004	0.017	0.122	0.895	1.017

Durable Cons	0.039	0.013	0.004	0.012	0.068	0.942	1.010

Nondurable Cons	0.030	0.029	0.002	0.008	0.069	0.943	1.012

Unemployment	0.251	0.176	0.064	0.088	0.578	0.182	0.760

Employment	0.451	0.062	0.035	0.164	0.712	0.277	0.990

Avg Hourly Earnings	0.040	0.160	0.007	0.026	0.232	0.792	1.025

Housing Starts	0.280	0.033	0.023	0.093	0.428	0.600	1.028

New Orders	0.437	0.055	0.044	0.156	0.691	0.350	1.041

Dividends	0.109	0.346	0.007	0.040	0.502	0.448	0.950

Consumer Expectations	0.173	0.462	0.014	0.061	0.710	0.290	1.000

```R
# Replicating Table I in BBE (2005) - 3 Factors and Y = FYFF
r2 = array(0, dim = key_nvars)
for(i in 1:key_nvars){
    r2[i] = results[[variables[i]]]$r.squared
    }
```

```R
table2 = data.frame("Variables" = variable_names, "Contribution" = round((psi2_fyff/var_total),3), "R-squared" = round(r2,3))
table2
```

Variables	Contribution	R.squared

Fed Funds Rate       	0.152                	1.000

Industrial Production	0.141                	0.746

CPI                  	0.104                	0.863

3m Treasury Bills    	0.139                	0.975

5y Treasury Bonds    	0.112                	0.933

Monetary Base        	0.005                	0.105

M2                   	0.005                	0.052

Exchange Rate Yen    	0.005                	0.024

Commodity Price Index	0.144                	0.640

Capacity Util Rate   	0.165                	0.749

Personal Consumption 	0.017                	0.110

Durable Cons         	0.012                	0.063

Nondurable Cons      	0.008                	0.063

Unemployment         	0.116                	0.819

Employment           	0.166                	0.724

Avg Hourly Earnings  	0.025                	0.212

Housing Starts       	0.090                	0.404

New Orders           	0.150                	0.652

Dividends            	0.042                	0.555

Consumer Expectations	0.061                	0.712


#Chapter 6

This chapter focuses on the most discussed part of Bayesian methodologies: how to choose a prior distribution. We also present a very interesting connection between priors and linear regression.

## Getting our priorities straight

Up until now, we have been mostly ignored our priors. This is unfortunate as we can be very expressive with our priors, but we also must be careful about choosing them.

###Subjective vs Objective priors

Bayesian priors can be classified into two classes: *objective* priors, which aim to allow the data to speak the most, and *subjective* priors, which allow the practioner to express his or her views into the prior.

What is an example of a objective prior? We have seen a some already, including the *flat* prior (which is a uniform distribution over the entire range of the unknown). This implies we give each possible value an equal chance. Choosing this type of prior is invoking what is called "The Principle of Indifference."

Conversely, if we added more probability to certain areas of the prior, and less elsewhere, we are biasing our results towards the former area. This is an subjective, or *informative* prior. A subjective prior does not always reflect the practioner's opinion: more often the subjective prior was once a posterior to a previous problem, and know the practioner is updating this posterior with new data.

```
%pylab inline

import scipy.stats as stats
figsize(12.5,2.5)
colors = ["#348ABD", "#A60628", "#7A68A6", "#467821"]

x = np.linspace(0,1)
y1, y2 = stats.beta.pdf(x, 1,1), stats.beta.pdf(x, 10,10)

p = plt.plot( x, y1,
    label='An objective prior \n( uninformative, \n"Principle of Indifference" )'  )
plt.fill_between( x, 0, y1, color = p[0].get_color(), alpha = 0.3 )

p = plt.plot( x,y2 ,
     label = "A subjective prior \n(informative)" )
plt.fill_between( x, 0, y2, color = p[0].get_color(), alpha = 0.3 )

plt.ylim(0,4)

plt.ylim(0, 4)
leg = plt.legend()
leg.get_frame().set_alpha(0.4)
plt.title("Comparing objective vs. subjective priors" );
```

The choice, either *objective* or *subjective* mostly depend on the problem being solved, but there are a few cases where one is preferred over the other. In instances of scientific research, the choice of an objective prior is obvious. This eliminiates any biases in the results, and two researchers who might have differing prior opionions would feel an objective prior is fair. Consider an more extreme situation:

> A tobacco company publishes an report with a Bayesian methodology that retreated 60 years of medical research on tobacco use, would you believe them? No, probably because they choose a subjective prior that too strongly biased results in there favour.

Unfortunately, choosing a objective prior is not obvious, and the topic even today the problem is still not completely solved. Gelmen *et al.* [3] suggest that using a uniform distribution, with a large margin, is probably the best.

We must remember that choosing a prior, whether subjective or objective, is still part of the modelling process. To quote Gelman [3]

>...after the model has been ﬁt, one should look at the posterior distribution
and see if it makes sense. If the posterior distribution does not make sense, this implies
that additional prior knowledge is available that has not been included in the model,
and that contradicts the assumptions of the prior distribution that has been used. It is
then appropriate to go back and alter the prior distribution to be more consistent with
this external knowledge.

If the posterior does not make sense, then clearly one had an idea what the posterior *should* look like (not what one *hopes* it looks like), implying that the current prior does not contain all the prior information and should be updated. At this point, we can discard the objective prior and choose a more reflective one.

Although, one should be wary about using Uniform objective priors with large margins. Do you really think the unknown could be incredibly large? Often quantities are naturally biased towards 0. A Normal random variable with large variance (small precision) is often a better choice.

Either way, if using a subjective prior, **it is important to explain the rationale**, else you are no better than the tobacoo company's guilty parties.

We next introduce the Beta distribution.

### The Beta distribution

You may have seen the term `beta` in previous code in this book. Often, I was implementing a Beta distribution. The Beta distribution is very useful in Bayesian statistics. A random variable $X$ has a $\text{Beta}$ distribution, with paramters $(\alpha, \beta)$, if its density function is:

$$f_X(x | \; \alpha, \beta ) = \frac{ x^{(\alpha - 1)}(1-x)^{ (\beta - 1) } }{B(\alpha, \beta) }$$

where $B$ is the [Beta function](http://en.wikipedia.org/wiki/Beta_function) (hence the name). The random variable $X$ is only allowed in [0,1], making the Beta distribution a popular distribution for decimal values, probabilities and proportions. The values of $\alpha$ and $\beta$, both positive values, provide great flexibility in the shape of the distribution. Below we plot some distributions:

```
figsize( 12.5, 5 )
import scipy.stats as stats

params = [ (2,5), (1,1), (0.5, 0.5), ( 5, 5), (20, 4), (5, 1)]

x = np.linspace( 0.01, .99, 100 )
beta = stats.beta
for a, b in params:
    y = beta.pdf( x, a, b )
    lines = plt.plot( x, y, label = "(%.1f,%.1f)"%(a,b), lw = 3 )
    plt.fill_between( x, 0, y, alpha = 0.2, color = lines[0].get_color() )
    plt.autoscale(tight=True)
plt.ylim(0)
plt.legend(loc = 'upper left');
```

One thing I'd like the reader to notice is the flat distribution above, specified by parameters $(1,1)$. This is the Uniform distribution. Hence the Beta distribution is a generalization of the Uniform distribution, something we will revisit many times.

There is an interesting connection beween the Beta distribution and the Binomial distribution. Suppose we are interested in some unknown proportion or probability $p$. We assign a $\text{Beta}(\alpha, \beta)$ prior to $p$. We observe some data genereted by a Binomial process, say $X \sim \text{Binomial}(N, p)$. Then our posterior *is a Beta disitribution*, i.e. $p | X \sim \text{Beta}( \alpha + X, \beta + N -X )$. Succinctly, one can relate the two by "a Beta prior with Binomial observations creates a Beta posterior". This is a very useful property, both computationally and heuristically.

In light of the above two paragraphs, if we start with a $\text{Beta}(1,1)$ prior on $p$ (which is a Uniform), observe data $X \sim \text{Binomial}(N, p)$, then our posterior is $\text{Beta}(1 + X, 1 + N - X)$.

#####Example: Bayesian Multi-Armed Bandits
*Adapted from an example by Ted Dunning of MapR Technologies*

> Suppose you are faced with $N$ slot machines (colourfully called multi-armed bandits). Each bandit has an unknown probability of distributing a prize (assume for now the prizes are the same for each bandit, only the probabilities differ). Some bandits are very generous, others not so much. Of course, you don't know what these probabilities are. By only choosing one bandit per round, our task is devise a strategy to maximize our winnings.

Of course, if we knew the bandit with the largest probability, then always picking this bandit would yield the maximum winnings. So our task can be phrased as "Find the best bandit, and as quickly as possible".

The task is complicated by the stochastic nature of the bandits. A suboptimal bandit can return many winnings, purely by chance, which would make us believe that it is a very profitable bandit. Similarly, the best bandit can return many duds. Should we keep trying losers then, or give up?

A more troublesome problem is, if we have a found a bandit that returns *pretty good* results, do we keep drawing from it to maintain our *pretty good score*, or do we try other bandits in hopes of finding an *even-better* bandit? This is the exploration vs. exploitation dilemma.

### Applications

The Multi-Armed Bandit problem at first seems very artificial, something only a mathematician would love, but that is only before we address some applications:

1. Internet display advertising: companies have a suite of potential ads they can display to visitors, but the company is not sure which ad strategy to follow to maximize sales. This is similar to A/B testing, but has the added advantage of naturally minimizing strategies that do not work (and generalizes to A/B/C/D... strategies)
2. Ecology: animals have a finite amount of energy to expend, and following certain behaviours has uncertain rewards. How does the animal maximize its fitness?
3. Finance: which stock option gives the highest return, under time-varying return profiles.
4. Clinical trials: a researcher would like to find the best treatment, out of many possible treatment, while minimizing losses.
5. Psychology: how does punishment and reward effect our behaviour? How do humans' learn?

Many of these questions above a fundamental to the application's field.

It turns out the *optimal solution* is incredibly difficult, and it took decades for an overall solution to develop. There are also many approximately-optimal solutions which are quite good. The one I wish to discuss is one of the few solutions that can scale incredibly well. The solution is known as *Bayesian Bandits*.

### A Proposed Solution

Any proposed strategy is called an *online algorithm* (not in the internet sense, but in the continuously-being-updated sense), and more specifically a reinforcement learning algorithm. The algorithm starts in an ignorant state, where it knows nothing, and begins to acquire data by testing the system. As it acquires data and results, it learns what the best and worst behaviours are (in this case, it learns which bandit is the best). With this in mind, perhaps we can add an additional application of the Multi-Armed Bandit problem:

- Psychology: how does punishment and reward effect our behaviour? How do humans' learn?

The Bayesian solution begins by assuming priors on the probability of winning for each bandit. In our vignette we assumed complete ignorance of the these probabilities. So a very natural prior is the flat prior over 0 to 1. The algorithm proceeds as follows:

For each round:

1. Sample a random variable $X_b$ from the prior of bandit $b$, for all $b$.
2. Select the bandit with largest sample, i.e. select $B = \text{argmax}\;\; X_b$.
3. Observe the result of pulling bandit $B$, and update your prior on bandit $B$.
4. Return to 1.

That's it. Computationally, the algorithm involves sampling from $N$ distributions. Since the initial priors are $\text{Beta}(\alpha=1,\beta=1)$ (a uniform distribution), and the observed result $X$ (a win or loss, encoded 1 and 0 respectfully) is Binomial, the posterior is a $\text{Beta}(\alpha=1+X,\beta=1+1−X)$.

To answer our question from before, this algorithm suggests that we should not discard losers, but we should pick them at a decreasing rate as we gather confidence that there exist *better* bandits. This follows because there is always a non-zero chance that a loser will achieve the status of $B$, but the probability of this event decreases as we play more rounds (see figure below).

Below we implement Bayesian Bandits using two classes, `Bandits` that defines the slot machines, and `BayesianStrategy` which implements the above learning strategy.

```
from pymc import rbeta

rand = np.random.rand

class Bandits(object):
    """
    This class represents N bandits machines.

    parameters:
        p_array: a (n,) Numpy array of probabilities >0, <1.

    methods:
        pull( i ): return the results, 0 or 1, of pulling
                   the ith bandit.
    """
    def __init__(self, p_array):
        self.p = p_array
        self.optimal = np.argmax(p_array)

    def pull( self, i ):
        #i is which arm to pull
        return rand() < self.p[i]

    def __len__(self):
        return len(self.p)

class BayesianStrategy( object ):
    """
    Implements a online, learning strategy to solve
    the Multi-Armed Bandit problem.

    parameters:
        bandits: a Bandit class with .pull method

    methods:
        sample_bandits(n): sample and train on n pulls.

    attributes:
        N: the cumulative number of samples
        choices: the historical choices as a (N,) array
        bb_score: the historical score as a (N,) array

    """

    def __init__(self, bandits):

        self.bandits = bandits
        n_bandits = len( self.bandits )
        self.wins = np.zeros( n_bandits )
        self.trials = np.zeros(n_bandits )
        self.N = 0
        self.choices = []
        self.bb_score = []

    def sample_bandits( self, n=1 ):

        bb_score = np.zeros( n )
        choices = np.zeros( n )

        for k in range(n):
            #sample from the bandits's priors, and select the largest sample
            choice = np.argmax( rbeta( 1 + self.wins, 1 + self.trials - self.wins) )

            #sample the chosen bandit
            result = self.bandits.pull( choice )

            #update priors and score
            self.wins[ choice ] += result
            self.trials[ choice ] += 1
            bb_score[ k ] = result
            self.N += 1
            choices[ k ] = choice

        self.bb_score = np.r_[ self.bb_score, bb_score ]
        self.choices = np.r_[ self.choices, choices ]
        return
```

Below we visualize the learning of the Bayesian Bandit solution.

```
import scipy.stats as stats
figsize( 11.0, 10)

beta = stats.beta
x = np.linspace(0.001,.999,200)

def plot_priors(bayesian_strategy, prob, lw = 3):
    ## plotting function
    wins = bayesian_strategy.wins
    trials = bayesian_strategy.trials
    for i in range( prob.shape[0] ):
        y = beta( 1+wins[i], 1 + trials[i] - wins[i] )
        p = plt.plot( x, y.pdf(x), lw = lw )
        c = p[0].get_markeredgecolor()
        plt.fill_between(x,y.pdf(x),0, color = c, alpha = 0.2,
                         label="underlying probability: %.2f"%prob[i])
        plt.vlines( prob[i], 0, y.pdf(prob[i]) ,
                   colors = c, linestyles = "--", lw = 2 )
        plt.autoscale(tight = "True")
        plt.title("Posteriors After %d pull"%bayesian_strategy.N +\
                    "s"*(bayesian_strategy.N>1) )
        plt.autoscale(tight=True)
    return
```

```
hidden_prob = np.array([0.85, 0.60, 0.75] )
bandits = Bandits( hidden_prob )
bayesian_strat = BayesianStrategy( bandits )

draw_samples = [1, 1, 3, 10, 10, 25, 50, 100, 200, 600 ]

for j,i in enumerate(draw_samples):
    subplot( 5, 2, j+1)
    bayesian_strat.sample_bandits(i)
    plot_priors( bayesian_strat, hidden_prob )
    #plt.legend()
    plt.autoscale( tight = True )
plt.tight_layout()
```

Note that we don't real care how accurate we become about inference of the hidden probabilities -- for this problem we are more interested in choosing the best bandit (or more accurately, becoming *more confident* in choosing the best bandit). For this reason, the distribution of the red bandit is very wide (representing ignorance about what that hidden probability might be) but we are reasonably confident that it is not the best, so the algorithm chooses to ignore it.

### A Measure of *Good*

We need a metric to calculate how well we are doing. Recall the absolute *best* we can do is to always pick the bandit with the largest probability of winning. Denote this best bandit's probability of $w^*$. Our score should be relative to how well we would have done had we chosen the best bandit from the beginning. This motivates the *total regret* of a strategy, defined:

\begin{align}
R_T & = \sum_{i=1}^{T} \left( w^* - w_{B(i)} \right)\\\\
& = Tw^* - \sum_{i=1}^{T} \;  w_{B(i)}
\end{align}

where $w_{B(i)}$ is the probability of a prize of the chosen bandit in the $i$ round. A total regret of 0 means the strategy is matching the best possible score. This is likely not possible, as initially our algorithm will often make the wrong choice.  Ideally, a strategy's total regret should flatten as it learns the best bandit. (Mathematically we achieve $w_{B(i)}=w^*$ often)

Below we plot the total regret of this simulation, including the scores of some other strategies:

1. Random: randomly choose a bandit to pull. If you can't beat this, just stop and do anything but your current guess.
2. largest Bayesian credible bound: pick the strategy with the largest upper bound in it's credible region of the underlying probability. Think "flavour of the month"
2. smallest Bayesian credible bound: same as above, but choose lower credible bound. This a cautious player, one who would like to experiment before really choosing a winner.
3. Largest proportion: pick the bandit with the current largest observed proportion of winning.

The code for these are in the other_strats.py, where you can implement your own very easily.

```
figsize( 12.5, 5 )

from other_strats import *

#define a harder problem
hidden_prob = np.array([0.15, 0.2, 0.1, 0.05] )
bandits = Bandits( hidden_prob )

#define regret
def regret( probabilities, choices ):
    w_opt = probabilities.max()
    return ( w_opt - probabilities[choices.astype(int)] ).cumsum()

#create new strategies
names = [ "Max Mean", "Lower Bound", "Upper Bound", "Random", "Bayesian Bandits" ]
strategies = []

strategies.append( GeneralBanditStrat( bandits, max_mean ))
strategies.append( GeneralBanditStrat( bandits, lower_credible_choice ))
strategies.append( GeneralBanditStrat( bandits, upper_credible_choice ))
strategies.append( GeneralBanditStrat( bandits, random_choice ))
strategies.append( GeneralBanditStrat( bandits, bayesian_bandit_choice ))

#train 10000 times
for strat in strategies:
    strat.sample_bandits( 1000)

#test and plot
for i,strat in enumerate(strategies):
    _regret = regret( hidden_prob, strat.choices )
    plt.plot( _regret, label = names[i], lw = 3)

plt.title("Total Regret of Bayesian Bandits Strategy vs. Random guessing" )
plt.xlabel("Number of pulls")
plt.ylabel("Regret after $n$ pulls");
plt.legend(loc = "upper left")
```

Like we wanted, the graph starts to flatten, representing we are achieving  optimal choices. To be more scientific so as to remove any possible luck in the above simulation, we should instead look at the *expected total regret*:

$$\bar{R_T} = E[ R_T ] $$

It can be shown that any *sub-optimal* strategy's expected total regret is bounded below logarithmically. Formally,

$$ E[R_T] = \Omega \left( \;\log(T)\; \right)$$

Thus, any strategy that matches logarithmic-growing regret is said to "solve" the Multi-Armed Bandit problem [3].

Using the Law of Large Numbers, we can approximate Bayesian Bandit's expected total regret by performing the same experiment many times (25 times, to be fair):

```
trials = 50
expected_total_regret = np.zeros( ( 10000, 5 ) )
strategies= [upper_credible_choice, bayesian_bandit_choice,
            lower_credible_choice , max_mean,
            random_choice ]

for i_strat, strat in enumerate( strategies ):

    for i in range(trials):
        general_strat = GeneralBanditStrat( bandits, strat )
        general_strat.sample_bandits(10000)
        _regret =  regret( hidden_prob, general_strat.choices )
        expected_total_regret[:,i_strat] += _regret

    plot(expected_total_regret[:,i_strat]/trials, lw =3, label = strat.__name__  )

plt.title("Expected Total Regret of Mutlit-armed Bandit strategies" )
plt.xlabel("Number of pulls")
plt.ylabel("Exepected Total Regret \n after $n$ pulls");
plt.legend(loc = "upper left")
```

```
plt.figure()
[pl1, pl2] = plt.plot( expected_total_regret[:, [0,1]], lw = 3  )
plt.xscale("log")
plt.legend([ pl1, pl2], ["UBC", "Bayesian Bandit"])
plt.ylabel("Exepected Total Regret \n after $\log{n}$ pulls");
plt.title( "Comparing UBC vs. Bayesian Bandits" );
plt.ylabel("Exepected Total Regret \n after $\log{n}$ pulls");
```

### Extending the algorithm

Because of the algorithm's simplicity, it is easy to extend it. Some possibilities:

1. If interested in the *minimum* probability (eg: where prizes are are bad thing), simply choose $B = \text{argmin} \; X_b$ and proceed.

2. Adding learning rates: Suppose the underlying environment may change over time. Technically the standard Bayesian Bandit algorithm would self-update itself (awesome) by noting that what it thought was the best is starting to fail more often, we can motivate the algorithm to learn changing environments quicker. We simply need to add a *rate* term upon updating:

        self.wins[ choice ] = rate*self.wins[ choice ] + result
        self.trials[ choice ] = rate*self.trials[ choice ] + 1

   If `rate < 1`, the algorithm will *forget* its previous wins quicker and there will be a downward  pressure towards ignorance. Conversely, setting `rate > 1` implies your algorithm will act more risky, and bet on earlier winners more often and be more resistant to changing environments.

2. Hierarchical algorithms: We can setup a Bayesian Bandit algorithm on top of smaller bandit algorithms. Suppose we have $N$ Bayesian Bandit models, each varying in some behavior (for example  different `rate` parameters, representing varying sensitivity to changing environments). On top of these $N$ models is another Bayesian Bandit learner that will select a sub-Bayesian Bandit. This chosen Bayesian Bandit will then make an internal choice as to which machine to pull. The super-Bayesian Bandit updates itself depending on whether the sub-Bayesian Bandit was correct or not.

3.  Extending the rewards, denoted $y_a$ for bandit $a$, to random variables from a distribution $f_{y_a}(y)$ is straightforward. More generally, this problem can be rephrased as "Find the bandit with the largest expected value", as playing the bandit with the largest expected value is optimal. In the case above, $f_{y_a}$ was Bernoulli with probability $p_a$, hence the expected value for an bandit is equal to $p_a$, which is why it looks like we are aiming to maximize the probability of winning. If $f$ is not Bernoulli, and it is non-negative, which can be accomplished apriori by shifting the distribution (we assume we know $f$), then the algorithm behaves as before:

   For each round,

   1. Sample a random variable $X_b$ from the prior of bandit $b$, for all $b$.
   2. Select the bandit with largest sample, i.e. select bandit $B = \text{argmax}\;\; X_b$.
   3. Observe the result,$R \sim f_{y_a}$, of pulling bandit $B$, and update your prior on bandit $B$.
   4. Return to A

   The issue is in the sampling of $X_b$ drawing phase. With Beta priors and Bernoulli observations, we have a Beta posterior -- this is easy to sample from. But now, with arbitrary distributions $f$, we have a non-trivial posterior. Sampling from these can be difficult.

Just for fun, though the colors explode, we compare the learning of 15 difference options.

```
figsize( 12.0, 8)
beta = stats.beta
hidden_prob = beta.rvs(1,12, size = 12 )
print hidden_prob
bandits = Bandits( hidden_prob )
bayesian_strat = BayesianStrategy( bandits )

for j,i in enumerate([25, 150, 400, 1000 ]):
    subplot( 2, 2, j+1)
    bayesian_strat.sample_bandits(i)
    plot_priors( bayesian_strat, hidden_prob, lw = 1 )
    #plt.legend()
    plt.xlim(0, 0.5 )

```

##Effect of the prior as $N$ increases

In the first Chapter, I proposed that as the amount of observations, or data, that we posses, the less the prior matters. This is intuitive. After all, our prior is based on previous information, and eventually enough new information will shadow our previous information's value. The smothering of the prior by enough data is also helpful: if our prior is significantly wrong, then the self-correcting nature of the data will present to us a *less wrong*, and eventually *correct*, posterior.

We can see this mathematically. First, recall Bayes Theorem from Chapter 1 that relates the prior to the posterior. The following is a sample from [What is the relationship between sample size and the influence of prior on posterior?](http://stats.stackexchange.com/questions/30387/what-is-the-relationship-between-sample-size-and-the-influence-of-prior-on-poste)[1] on CrossValidated.

>The posterior distribution for a parameter $\theta$, given a data set ${\bf X}$ can be written as

\begin{equation}
p(\theta | {\bf X}) \propto \underbrace{p({\bf X} | \theta)}_{{\rm likelihood}} \cdot \underbrace{ p(\theta)}_{{\rm prior} }
\end{equation}

>or, as is more commonly displayed on the log scale,

$$ \log( p(\theta | {\bf X})  ) = c + L(\theta;{\bf X}) + \log(p(\theta)) $$

>The log-likelihood, $L(\theta;{\bf X}) = \log \left( p({\bf X}|\theta) \right)$, **scales with the sample size**, since it is a function of the data, while the prior density does not. Therefore, as the sample size increases, the absolute value of $L(\theta;{\bf X})$ is getting larger while $\log(p(\theta))$ stays fixed (for a fixed value of $\theta$), thus the sum $L(\theta;{\bf X}) + \log(p(\theta))$ becomes more heavily influenced by $L(\theta;{\bf X})$ as the sample size increases.

There is an interesting consequence not immediately apparent. For large sample sizes, the choice prior has less influence. Hence inference converges regardless of chosen prior, so long as areas of the prior assign non-zero probabilities are the same.

Below we visualize this. We examine the convergence of two posteriors of a Binomial's parameter $\theta$, one with a flat prior and the other with a biased prior towards 0. As the sample size increases, the posteriors, and hence the inference, converge.

```
figsize( 12.5, 15)
import scipy.stats as stats
import pymc as mc

p = 0.6
beta1_params = np.array( [1.,1.] )
beta2_params = np.array( [2,10] )
beta = stats.beta

x = np.linspace(0.00, 1, 125)
data = mc.rbernoulli(p, size=500)

figure()
for i,N in enumerate([0,4,8, 32,64, 128, 500]):
    s = data[:N].sum()
    subplot(8,1,i+1)
    params1 = beta1_params + np.array( [s, N-s] )
    params2 = beta2_params + np.array( [s, N-s] )
    y1,y2 = beta.pdf( x, *params1), beta.pdf( x, *params2)
    plt.plot( x,y1, label = r"flat prior", lw =3 )
    plt.plot( x, y2, label = "biased prior", lw= 3 )
    plt.fill_between( x, 0, y1, color ="#348ABD", alpha = 0.15)
    plt.fill_between( x, 0, y2, color ="#A60628", alpha = 0.15)
    plt.legend(title = "N=%d"%N)
    plt.vlines( p, 0.0, 7.5, linestyles = "--", linewidth=1)
    #plt.ylim( 0, 10)#

```

Keep in mind, not all posteriors will "forget" the prior this quickly. This example was just to show that *eventually* the prior is forgotten.

### Bayesian perspective of Penalized Linear Regressions

There is a very interesting relationship between the penalized least-squares regression and Bayesian priors. We will first describe the probablistic interpretation of Linear Regression. Denote our response variable $Y$, and features are contained in the data matrix $X$. The standard linear model is:

\begin{equation}
Y = X\beta + \epsilon
\end{equation}

where $\epsilon \sim \text{Normal}( {\bf 0}, \sigma{\bf I })$. Simpley, the observed $Y$ is a linear function of $X$ (with coeffcients $\beta$) plus some noise term. Our unknown to be learned is $\beta$. We use the following property of Normal random variables:

$$ \mu' + \text{Normal}( \mu, \sigma ) \sim \text{Normal}( \mu' + \mu , \sigma ) $$

to rewrite the above linear model as:

\begin{align}
& Y = X\beta + \text{Normal}( {\bf 0}, {\bf I }) \\\\
& Y = \text{Normal}( X\beta , {\bf I }) \\\\
\end{align}

In probabilitistic notation, denote $f_Y(y \; | \; \beta )$ the probabilitiy distribution of $Y$, and recall the density function for a Normal random variable:

$$ f_Y( Y \; |\; \beta, X) = L(\beta|\; X,Y)= \frac{1}{\sqrt{ 2\pi\sigma} } \exp \left( \frac{1}{2\sigma^2} (Y - X\beta)^T(Y - X\beta) \right) $$

This is the likelihood function for $\beta$ Taking the $\log$:

$$ \ell(\beta) = K - c(Y - X\beta)^T(Y - X\beta) $$

where $K$ and $c>0$ are constants. Maximum likelihood techniques wish to maximize this for $\beta$,

$$\hat{ \beta } = \text{argmax}_{\beta} \;\; - (Y - X\beta)^T(Y - X\beta) $$

Equivilantly we can minimize the negative of the above:

$$\hat{ \beta } = \text{argmin}_{\beta} \;\; (Y - X\beta)^T(Y - X\beta) $$

This is probably the familiar least-squares linear regression equation. We showed that the solution to this is the same as the maximum likelihood assuming Normal noise. Next we extend this to show how we can arrive at penalized linear regression by a suitable choice of priors on $\beta$.

#### Penalized least-squares

In the above, once we have the likelihood, we can include a prior distribution on $\beta$ to move to the equation for the posterior distribution:

$$P( \beta | Y, X ) = L(\beta|\;X,Y)p( \beta )$$

where $p(\beta)$ is a prior on the elements of $\beta$. What are some interesting priors?

1\. If we include *no explicit* prior term, we are actually including an uniformative prior, $P( \beta ) \propto 1$.

2\. If we have reason to believe the elements of $\beta$ are not too large, we can suppose that *apriori*:

$$ \beta \sim \text{Normal}({\bf 0 }, \lambda {\bf I } ) $$

The resulting posterior density function for $\beta$ is *proportional to*:

$$ \exp \left( \frac{1}{2\sigma^2} (Y - X\beta)^T(Y - X\beta) \right) \exp \left( \frac{1}{2\lambda^2} \beta^T\beta \right) $$

and taking the $\log$ of this, and combining some constants, we get:

$$ \ell(\beta) \propto K -  (Y - X\beta)^T(Y - X\beta) - \alpha \beta^T\beta  $$

we arrive at the function we wish to maximize (recall the point that maximizes the posterior distribution is the MAP, or *maximum a posterior*):

$$\hat{ \beta } = \text{argmax}_{\beta} \;\; -(Y - X\beta)^T(Y - X\beta) - \alpha \;\beta^T\beta $$

Equivilantly, we can minimize the negative of the above, and rewriting $\beta^T\beta = ||\beta||_2^2:

$$\hat{ \beta } = \text{argmin}_{\beta} \;\; (Y - X\beta)^T(Y - X\beta) + \alpha \;||\beta||_2^2$$

This above term is exactly Ridge Regression. Thus we can see that ridge regression corresponds to the MAP of a linear model with Normal errors and a Normal prior on $\beta$.

3\. Similarly, if we assume a *Laplace* prior on $\beta$, ie.

$$ f_\beta( \beta) \propto \exp \left(- \lambda ||\beta||_1 \right)$$

and following the same steps as above, we recover:

$$\hat{ \beta } = \text{argmin}_{\beta} \;\; (Y - X\beta)^T(Y - X\beta) + \alpha \;||\beta||_1$$

which is LASSO regression. Some important notes about this equilivance. The sparsity that is a result of using a LASSO regularization is not a result of the prior assigning high probability to sparsity. Quite the opposite actually. It is the combination of the $|| \cdot ||_1$ function and using the MAP that creates sparsity on $\beta$: purely a geometric argument. The prior does contribute to an overall shrinking of the coefficients towards 0 though. An interesing disucssion of this can be found in [2].

##### References

1. http://stats.stackexchange.com/questions/30387/what-is-the-relationship-between-sample-size-and-the-influence-of-prior-on-poste

2. Starck, J.-L., , et al. "Sparsity and the Bayesian Perspective." Astronomy & Astrophysics. (2013): n. page. Print.

3. Kuleshov, Volodymyr, and Doina Precup. "Algorithms for the multi-armed bandit problem." Journal of Machine Learning Research. (2000): 1-49. Print.

```
from IPython.core.display import HTML
def css_styling():
    styles = open("../styles/custom.css", "r").read()
    return HTML(styles)
css_styling()
```

<style>
    @font-face {
        font-family: "Computer Modern";
        src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
    }
    div.cell{
        width:800px;
        margin-left:auto;
        margin-right:auto;
    }
    h1 {
        font-family: Helvetica, serif;
    }
    h4{
        margin-top:12px;
        margin-bottom: 3px;
       }
    div.text_cell_render{
        font-family: Computer Modern, "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
        line-height: 145%;
        font-size: 120%;
        width:800px;
        margin-left:auto;
        margin-right:auto;
    }
    .CodeMirror{
            font-family: "Source Code Pro", source-code-pro,Consolas, monospace;
    }
    .prompt{
        display: None;
    }
    .text_cell_render h5 {
        font-weight: 300;
        font-size: 16pt;
        color: #4057A1;
        font-style: italic;
        margin-bottom: .5em;
        margin-top: 0.5em;
        display: block;
    }

    .warning{
        color: rgb( 240, 20, 20 )
        }

</style>