# Attempt to Replicate the Black Scholes Model Using a Neural Network

```python
import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
import matplotlib.pyplot as plt
import scipy.stats as ss
```

## Define the Function
Here we define our function, the Black Scholes Model (BSM). First, we must initialize the option class, then the Geometric Brownian Motion Class, and finally the BSM class.
Then we test the equation with a test value of 2.

```python
'''=========
option class init
=========='''
class VanillaOption:
    def __init__(
        self,
        otype = 1, # 1: 'call'
                  # -1: 'put'
        strike = 110.,
        maturity = 1.,
        market_price = 10.):
      self.otype = otype               # Put or Call
      self.strike = strike             # Strike K
      self.maturity = maturity         # Maturity T
      self.market_price = market_price #this will be used for calibration

    def payoff(self, s): #s: excercise price
      otype = self.otype
      k = self.strike
      maturity = self.maturity
      return np.max([0, (s - k)*otype])

```

```python
'''============
Gbm class
============='''

class Gbm:
    def __init__(self,
                 init_state = 100.,
                 drift_ratio = .0475,
                 vol_ratio = .2
                ):
        self.init_state = init_state
        self.drift_ratio = drift_ratio
        self.vol_ratio = vol_ratio
```

```python
'''========
Black-Scholes-Merton formula.
=========='''

def bsm_price(self, vanilla_option):
    s0 = self.init_state
    sigma = self.vol_ratio
    r = self.drift_ratio

    otype = vanilla_option.otype
    k = vanilla_option.strike
    maturity = vanilla_option.maturity

    d1 = 1/(sigma*np.sqrt(maturity))*(np.log(s0/k) + (r + np.power(sigma,2)/2)*(maturity))
    d2 = 1/(sigma*np.sqrt(maturity))*(np.log(s0/k) + (r - np.power(sigma,2)/2)*(maturity))
    return (otype * s0 * ss.norm.cdf(otype * d1) #line break needs parenthesis
            - otype * np.exp(-r * maturity) * k * ss.norm.cdf(otype * d2))

Gbm.bsm_price = bsm_price
```

```python
'''=======
Get BSM prices given an option and a Tensor
======='''

def prices_bsm(self, vanilla_option, data):

  # Create the list
  a = []

  # Get tensor size
  sizeT = list(data.size())[0]

  for i in range(sizeT):
    self.init_state = data[i].item()
    callPrice = gbm1.bsm_price(vanilla_option)
    a.append(callPrice)

  # Convert to array then tensor
  arrayOut = np.array(a)
  outputData = torch.from_numpy(arrayOut)
  return outputData

Gbm.prices_bsm = prices_bsm
```

```python
def f(s):
  gbm = Gbm(init_state=s)
  option = VanillaOption(strike=10)
  return gbm.bsm_price(option)

batch_size = 21
x_list = np.linspace(8, 11, batch_size)
y_list = np.array([f(x) for x in x_list])
#plt.plot(x_list, y_list)
```

## Create Model
Next, we create the neural network model. This is done first by setting the inner and outer dimensions with variables. Next we code the model and vary the internal dimensions to attempt to improve the model. At this level, this is essentially a simple linear algebra exercise:

If we have input $x$, internal parameters $a,b$, and solution $f(x)$ then in the one-dimensional case we have:

\begin{equation}
  \left(
    a_{1}x+b_{2}
  \right)
  a_{2} + b_{2}
  = f(x)
\end{equation}

However, we want to get a better estimate for the true equation. So we increase the interior dimension which corresponds to the number of neurons inside the network. For example, we raised the inner dimension to 3. In matrix form we have:

\begin{equation}
\left(
  \begin{bmatrix} x \end{bmatrix}
  \begin{bmatrix} a_{1} & a_{2} & a_{3} \end{bmatrix}
  +
  \begin{bmatrix} b_{1} & b_{2} & b_{3} \end{bmatrix}
\right)
 \begin{bmatrix} a_{4} \\ a_{5} \\ a_{6} \end{bmatrix}
 +
 \begin{bmatrix} b_{4} \\ \end{bmatrix}
 =
 \begin{bmatrix} f(x) \end{bmatrix}
\end{equation}

Graphically, we can render this second neural network as:

**ADD DESCRIPTION**

```python
#model
#nn.Linear
H1 = 50; H2 = 11 #number of hidden layer
model = nn.Sequential(
    nn.Linear(1, H1),
    nn.Sigmoid(),
    nn.Linear(H1, H2),
    nn.Sigmoid(),
    nn.Linear(H2,2),
    nn.Sigmoid(),
    nn.Linear(2,1)
)
```

Here we define the Loss function as the Mean Squared Error(MSE).

Note that by doing so, we are essentially 'cheating' the system. In most applications, we would not know the function $f$ so we would be unable to find the MSE.

```python
#loss function
criterion = nn.MSELoss()
```

Next we choose a learning rate and a method for learning. The learning rate is the percent of the data that is accepted in each iteration. The Methods we tried were SGD and Adam.

```python
#optimizer
learning_rate = 0.1
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.7)
```

```python
batch_size = np.size(x_list)
x_train0 = torch.from_numpy(x_list).reshape(batch_size,1).float()
y_train0 = torch.from_numpy(y_list).reshape(batch_size,1).float()
```

## Train the Model

First we create the training data. This is a batch of random points that we pass through the BSM.

```python
# Create Training Data

#batch_size = 1000

# Create a list
#a = []

# Generate randomized data
#x_train = torch.randn(batch_size, 1)
# Modify the random data to be a stock price
#x_train0 = np.abs(x_train*100)

# Create static training data for conistency
#x_train = torch.tensor([[100], [105], [110], [115], [120]])

# Initialize GBM
#gbm1 = Gbm()
#option = VanillaOption()

# Call the prices_bsm to get y_train
#y_train0 = gbm1.prices_bsm(option, x_train)

#print(x_train[1:10])
#print(y_train[1:10])

```

Once we have the training data, we pass this collection of inputs and solutions into the model. With each iteration we calculate the loss and attempt to optimize the model to further reduce the loss.

```python
# Normalization

def linear_transform(xx, l = 0, u= 1):
  M = torch.max(xx)
  m = torch.min(xx)
  return (u-l)/(M-m)*(xx-m)+l, m, M, l, u
x_train, x_m, x_M, x_l, x_u = linear_transform(x_train0, -1, 1)
y_train, y_m, y_M, y_l, y_u = linear_transform(y_train0, 0, 1)
```

```python
# Train the model

num_epochs = 1000

for epoch in range(num_epochs):

    # Forward pass
    outputs = model(x_train.float())
    loss = criterion(outputs, y_train.float())

    # Backward and optimize
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if epoch == 0 or (epoch+1) % 50 == 0:
        print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1,
                                                    num_epochs, loss.item()))
        print(outputs[1:10])

```

    Epoch [1/1000], Loss: 0.1023
    tensor([[0.3266],
            [0.3266],
            [0.3265],
            [0.3264],
            [0.3264],
            [0.3263],
            [0.3263],
            [0.3262],
            [0.3261]], grad_fn=<SliceBackward>)
    Epoch [50/1000], Loss: 0.0950
    tensor([[0.4094],
            [0.4094],
            [0.4094],
            [0.4094],
            [0.4094],
            [0.4094],
            [0.4095],
            [0.4095],
            [0.4095]], grad_fn=<SliceBackward>)
    Epoch [100/1000], Loss: 0.0946
    tensor([[0.4082],
            [0.4084],
            [0.4085],
            [0.4086],
            [0.4087],
            [0.4088],
            [0.4090],
            [0.4091],
            [0.4092]], grad_fn=<SliceBackward>)
    Epoch [150/1000], Loss: 0.0941
    tensor([[0.4068],
            [0.4071],
            [0.4073],
            [0.4076],
            [0.4078],
            [0.4081],
            [0.4084],
            [0.4086],
            [0.4089]], grad_fn=<SliceBackward>)
    Epoch [200/1000], Loss: 0.0933
    tensor([[0.4047],
            [0.4052],
            [0.4056],
            [0.4061],
            [0.4065],
            [0.4070],
            [0.4075],
            [0.4079],
            [0.4084]], grad_fn=<SliceBackward>)
    Epoch [250/1000], Loss: 0.0920
    tensor([[0.4011],
            [0.4019],
            [0.4027],
            [0.4035],
            [0.4043],
            [0.4051],
            [0.4060],
            [0.4068],
            [0.4077]], grad_fn=<SliceBackward>)
    Epoch [300/1000], Loss: 0.0894
    tensor([[0.3942],
            [0.3956],
            [0.3971],
            [0.3986],
            [0.4002],
            [0.4017],
            [0.4033],
            [0.4049],
            [0.4065]], grad_fn=<SliceBackward>)
    Epoch [350/1000], Loss: 0.0836
    tensor([[0.3789],
            [0.3819],
            [0.3849],
            [0.3880],
            [0.3911],
            [0.3943],
            [0.3975],
            [0.4008],
            [0.4040]], grad_fn=<SliceBackward>)
    Epoch [400/1000], Loss: 0.0697
    tensor([[0.3400],
            [0.3468],
            [0.3538],
            [0.3609],
            [0.3682],
            [0.3757],
            [0.3832],
            [0.3908],
            [0.3985]], grad_fn=<SliceBackward>)
    Epoch [450/1000], Loss: 0.0384
    tensor([[0.2362],
            [0.2529],
            [0.2703],
            [0.2883],
            [0.3069],
            [0.3260],
            [0.3454],
            [0.3652],
            [0.3852]], grad_fn=<SliceBackward>)
    Epoch [500/1000], Loss: 0.0077
    tensor([[0.0706],
            [0.1010],
            [0.1335],
            [0.1679],
            [0.2042],
            [0.2422],
            [0.2816],
            [0.3222],
            [0.3637]], grad_fn=<SliceBackward>)
    Epoch [550/1000], Loss: 0.0020
    tensor([[-0.0010],
            [ 0.0325],
            [ 0.0689],
            [ 0.1083],
            [ 0.1507],
            [ 0.1958],
            [ 0.2436],
            [ 0.2936],
            [ 0.3454]], grad_fn=<SliceBackward>)
    Epoch [600/1000], Loss: 0.0011
    tensor([[-0.0026],
            [ 0.0284],
            [ 0.0625],
            [ 0.0998],
            [ 0.1403],
            [ 0.1841],
            [ 0.2310],
            [ 0.2808],
            [ 0.3332]], grad_fn=<SliceBackward>)
    Epoch [650/1000], Loss: 0.0007
    tensor([[0.0047],
            [0.0333],
            [0.0650],
            [0.0999],
            [0.1382],
            [0.1799],
            [0.2251],
            [0.2735],
            [0.3250]], grad_fn=<SliceBackward>)
    Epoch [700/1000], Loss: 0.0005
    tensor([[0.0103],
            [0.0373],
            [0.0673],
            [0.1005],
            [0.1372],
            [0.1773],
            [0.2211],
            [0.2684],
            [0.3191]], grad_fn=<SliceBackward>)
    Epoch [750/1000], Loss: 0.0003
    tensor([[0.0144],
            [0.0402],
            [0.0690],
            [0.1010],
            [0.1364],
            [0.1755],
            [0.2182],
            [0.2647],
            [0.3148]], grad_fn=<SliceBackward>)
    Epoch [800/1000], Loss: 0.0003
    tensor([[0.0176],
            [0.0425],
            [0.0704],
            [0.1014],
            [0.1359],
            [0.1741],
            [0.2161],
            [0.2619],
            [0.3114]], grad_fn=<SliceBackward>)
    Epoch [850/1000], Loss: 0.0002
    tensor([[0.0201],
            [0.0443],
            [0.0715],
            [0.1018],
            [0.1356],
            [0.1731],
            [0.2144],
            [0.2597],
            [0.3088]], grad_fn=<SliceBackward>)
    Epoch [900/1000], Loss: 0.0002
    tensor([[0.0220],
            [0.0458],
            [0.0724],
            [0.1022],
            [0.1354],
            [0.1724],
            [0.2132],
            [0.2580],
            [0.3068]], grad_fn=<SliceBackward>)
    Epoch [950/1000], Loss: 0.0001
    tensor([[0.0237],
            [0.0470],
            [0.0731],
            [0.1025],
            [0.1353],
            [0.1718],
            [0.2122],
            [0.2567],
            [0.3052]], grad_fn=<SliceBackward>)
    Epoch [1000/1000], Loss: 0.0001
    tensor([[0.0249],
            [0.0479],
            [0.0737],
            [0.1027],
            [0.1352],
            [0.1713],
            [0.2114],
            [0.2556],
            [0.3039]], grad_fn=<SliceBackward>)

## Testing the Model

```python
def learnedfun(x):
  out = (1-(-1))/(x_M-x_m)*(x-x_m)+(-1.)
  out = model(out)
  out = (y_M- y_m)*out+y_m
  return out

y_pred = learnedfun(x_train0)
```

```python
# Test with training data
plt.scatter(x_train0.detach().numpy(), y_train0.detach().numpy(), label='true')
plt.plot(x_train0.detach().numpy(), y_pred.detach().numpy(), label='pred')

plt.legend()
plt.show()
```

```python
# Test the Model with Random data

# Generate random data
x_0 = torch.randn(50,1)

# Get BSM Prices as determined by the formula
y_0 = gbm1.prices_bsm(option, x_)

# Normalize
x_, x_m, x_M, x_l, x_u = linear_transform(x_0)
y_, y_m, y_M, y_l, y_u = linear_transform(x_0)

# Plot x_ versus formula prices
plt.scatter(x_0.detach().numpy(), y_0.detach().numpy(), label='true')

# Get BSM Prices as determined by the model
y_pred = learnedfun(x_0)

# Plot x_ versus the model prices
plt.scatter(x_0.detach().numpy(), y_pred.detach().numpy(), label='pred')

plt.legend()

```

```python

```


# Working with Sampling Distributions
Most statistical analysis involves working with distributions - usually of sample data.

## Sampling and Sampling Distributions
As we discussed earlier, when working with statistics, we usually base our calculations on a sample and not the full population of data. This means we need to allow for some variation between the sample statistics and the true parameters of the full population.

In the previous example, we knew the probability that a security search would be triggered was 25%, so it's pretty easy to calculate that the expected value for a random variable indicating the number of searches per 100 passengers is 25. What if we hadn't known the probability of a search? How could we estimate the expected mean number of searches for a given number of passengers based purely on sample data collected by observing passengers go through security?

### Creating a Proportion Distribution from a Sample
We know that the each passenger will either be searched or not searched, and we can assign the values ***0*** (for not searched) and ***1*** (for searched) to these outcomes. We can conduct a Bernoulli trial in which we sample 16 passengers and calculate the fraction (or *proportion*) of passengers that were searched (which we'll call ***p***), and the remaining proportion of passengers (which are the ones who weren't searched, and can be calculated as ***1-p***).

Let's say we record the following values for our 16-person sample:

    0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0

In this sample, there were 3 searches out of 16 passengers; which as a proportion is 3/16 or 0.1875. This is our proportion (or **p**); but because we know that this is based on a sample, we call it **p̂** (or p-hat). The remaining proportion of passengers is 1-p; in this case 1 - 0.1875, which is 0.8125.

The data itself is *qualitative* (categorical) - we're indicating "no search" or "search"; but because we're using numeric values (0 and 1), we can treat these values as numeric and create a binomial distribution from them - it's the simplest form of a binomial distribution - a Bernoulli distribution with two values.

Because we're treating the results as a numberic distribution, we can also calculate statistics like *mean* and *standard deviation*:

To calculate these, you can use the following formulae:

\begin{equation}\mu_{\hat{p}} = \hat{p}\end{equation}

\begin{equation}\sigma_{\hat{p}} = \sqrt{\hat{p}(1-\hat{p})}\end{equation}

The mean is just the value of **p̂**, so in the case of the passenger search sample it is 0.1875.

The standard deviation is calculated as:

\begin{equation}\sigma_{\hat{p}} = \sqrt{0.1875 \times 0.8125} \approx 0.39\end{equation}

We can use Python to plot the sample distribution and calculate the mean and standard deviation of our sample like this:

```python
%matplotlib inline
from matplotlib import pyplot as plt
import numpy as np

searches = np.array([0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0])

# Set up the graph
plt.xlabel('Search Results')
plt.ylabel('Frequency')
plt.hist(searches)
plt.show()
print('Mean: ' + str(np.mean(searches)))
print('StDev: ' + str(np.std(searches)))
```

When talking about probability, the *mean* is also known as the *expected value*; so based on our single sample of 16 passengers, should we expect the proportion of searched passengers to be 0.1875 (18.75%)?

Well, using a single sample like this can be misleading because the number of searches can vary with each sample. Another person observing 100 passengers may get a (very) different result from you. One way to address this problem is to take multiple samples and combine the resulting means to form a *sampling* distribution. This will help us ensure that the distribution and statistics of our sample data is closer to the true values; even if we can't measure the full population.

### Creating a Sampling Distribution of a Sample Proportion
So, let's collect mulitple 16-passenger samples - here are the resulting sample proportions for 12 samples:

| Sample | Result |
|--------|--------|
| p̂1| 0.1875 |
| p̂2| 0.2500 |
| p̂3| 0.3125 |
| p̂4| 0.1875 |
| p̂5| 0.1250 |
| p̂6| 0.3750 |
| p̂7| 0.2500 |
| p̂8| 0.1875 |
| p̂9| 0.3125 |
| p̂10| 0.2500 |
| p̂11| 0.2500 |
| p̂12| 0.3125 |

We can plot these as a sampling distribution like this:

```python
%matplotlib inline
from matplotlib import pyplot as plt
import numpy as np

searches = np.array([0.1875,0.25,0.3125,0.1875,0.125,0.375,0.25,0.1875,0.3125,0.25,0.25,0.3125])

# Set up the graph
plt.xlabel('Search Results')
plt.ylabel('Frequency')
plt.hist(searches)
plt.show()
```

#### The Central Limit Theorem
You saw previously with the binomial probability distribution, with a large enough sample size (the *n* value indicating the number of binomial experiments), the distribution of values for a random variable started to form an approximately *normal* curve. This is the effect of the *central limit theorem*, and it applies to any distribution of sample data if the size of the sample is large enough. For our airport passenger data, if we collect a large enough number of samples, each based on a large enough number of passenger observations, the sampling distribution will be approximately normal. The larger the sample size, the closer to a perfect *normal* distribution the data will be, and the less variance around the mean there will be.

Run the cell below to see a simulated distribution created by 10,000 random 100-passenger samples:

```python
%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

n, p, s = 100, 0.25, 10000
df = pd.DataFrame(np.random.binomial(n,p,s)/n, columns=['p-hat'])

# Plot the distribution as a histogram
means = df['p-hat']
means.plot.hist(title='Simulated Sampling Distribution')
plt.show()
print ('Mean: ' + str(means.mean()))
print ('Std: ' + str(means.std()))
```

### Mean and Standard Error of a Sampling Distribution of Proportion
The sampling distribution is created from the means of multiple samples, and its mean is therefore the mean of all the sample means. For a distribution of proportion means, this is considered to be the same as **p** (the population mean). In the case of our passenger search samples, this is 0.25.

Because the sampling distribution is based on means, and not totals, its standard deviation is referred to as its *standard error*, and its formula is:

\begin{equation}\sigma_{\hat{p}} = \sqrt{\frac{p(1-p)}{n}}\end{equation}

In this formula, *n* is the size of each sample; and we divide by this to correct for the error introduced by the average values used in the sampling distribution. In this case, our samples were based on observing 16-passengers, so:

\begin{equation}\sigma_{\hat{p}} = \sqrt{\frac{0.25 \times 0.75}{16}} \approx 0.11\end{equation}

In our simulation of 100-passenger samples, the mean remains 0.25. The standard error is:

\begin{equation}\sigma_{\hat{p}} = \sqrt{\frac{0.25 \times 0.75}{100}} \approx 0.043\end{equation}

Note that the effect of the central limit theorem is that as you increase the number and/or size of samples, the mean remains constant but the amount of variance around it is reduced.

Being able to calculate the mean (or *expected value*) and standard error is useful, because we can apply these to what we know about an approximately normal distribution to estimate probabilities for particular values. For example, we know that in a normal distribution, around 95.4% of the values are within two standard deviations of the mean. If we apply that to our sampling distribution of ten thousand 100-passenger samples, we can determine that the proportion of searched passengers in 95.4% of the samples was between 0.164 (16.4%) and 0.336 (36.6%).

How do we know this?

We know that the mean is ***0.25*** and the standard error (which is the same thing as the standard deviation for our sampling distribution) is ***0.043***. We also know that because this is a *normal* distribution, ***95.4%*** of the data lies within two standard deviations (so 2 x 0.043) of the mean, so the value for 95.4% of our samples is 0.25 ± (*plus or minus*) 0.086.

The *plus or minus* value is known as the *margin of error*, and the range of values within it is known as a *confidence interval* - we'll look at these in more detail later. For now, run the following cell to see a visualization of this interval:

```python
%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

n, p, s = 100, 0.25, 10000
df = pd.DataFrame(np.random.binomial(n,p,s)/n, columns=['p-hat'])

# Plot the distribution as a histogram
means = df['p-hat']
m = means.mean()
sd = means.std()
moe1 = m - (sd * 2)
moe2 = m + (sd * 2)

means.plot.hist(title='Simulated Sampling Distribution')

plt.axvline(m, color='red', linestyle='dashed', linewidth=2)
plt.axvline(moe1, color='magenta', linestyle='dashed', linewidth=2)
plt.axvline(moe2, color='magenta', linestyle='dashed', linewidth=2)
plt.show()
```

### Creating a Sampling Distribution of Sample Means
In the previous example, we created a sampling distribution of proportions; which is a suitable way to handle discrete values, like the number of passengers searched or not searched. When you need to work with continuous data, you use slightly different formulae to work with the sampling distribution.

For example, suppose we want to examine the weight of the hand luggage carried by each passenger. It's impractical to weigh every bag that is carried through security, but we could weigh one or more samples, for say, 5 passengers at a time, on twelve occassions. We might end up with some data like this:

| Sample | Weights |
|--------|---------|
| 1      | [4.020992,2.143457,2.260409,2.339641,4.699211] |
| 2      | [3.38532,4.438345,3.170228,3.499913,4.489557] |
| 3      | [3.338228,1.825221,3.53633,3.507952,2.698669] |
| 4      | [2.992756,3.292431,3.38148,3.479455,3.051273] |
| 5      | [2.969977,3.869029,4.149342,2.785682,3.03557] |
| 6      | [3.138055,2.535442,3.530052,3.029846,2.881217] |
| 7      | [1.596558,1.486385,3.122378,3.684084,3.501813] |
| 8      | [2.997384,3.818661,3.118434,3.455269,3.026508] |
| 9      | [4.078268,2.283018,3.606384,4.555053,3.344701] |
| 10     | [2.532509,3.064274,3.32908,2.981303,3.915995] |
| 11     | [4.078268,2.283018,3.606384,4.555053,3.344701] |
| 12     | [2.532509,3.064274,3.32908,2.981303,3.915995] |

Just as we did before, we could take the mean of each of these samples and combine them to form a sampling distribution of the sample means (which we'll call **X**, and which will contain a mean for each sample, which we'll label x̄n):

| Sample | Mean Weight |
|--------|---------|
| x̄1 | 3.092742  |
| x̄2 | 3.7966726 |
| x̄3 | 2.98128   |
| x̄4 | 3.239479  |
| x̄5 | 3.36192   |
| x̄6 | 3.0229224 |
| x̄7 | 2.6782436 |
| x̄8 | 3.2832512 |
| x̄9 | 3.5734848 |
| x̄10 | 3.1646322 |
| x̄11 | 3.5734848 |
| x̄12 | 3.1646322 |

We can plot the distribution for the sampling distribution like this:

```python
%matplotlib inline
from matplotlib import pyplot as plt
import numpy as np

meanweights = np.array([3.092742,
                        3.7966726,
                        2.98128,
                        3.239479,
                        3.36192,
                        3.0229224,
                        2.6782436,
                        3.2832512,
                        3.5734848,
                        3.1646322,
                        3.5734848,
                        3.1646322])

# Set up the graph
plt.xlabel('Mean Weights')
plt.ylabel('Frequency')
plt.hist(meanweights, bins=6)
plt.show()

print('Mean: ' + str(meanweights.mean()))
print('Std: ' + str(meanweights.std()))
```

Just as before, as we increase the sample size, the central limit theorem ensures that our sampling distribution starts to approximate a normal distribution. Our current distribution is based on the means generated from twelve samples, each containing 5 weight observations. Run the following code to see a distribution created from a simulation of 10,000 samples each containing weights for 500 passengers:

>This may take a few minutes to run. The code is not the most efficient way to generate a sample distribution, but it reflects the principle that our sampling distribution is made up of the means from multiple samples. In reality, you could simulate the sampling by just creating a single sample from the ***random.normal*** function with a larger ***n*** value.

```python
%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

mu, sigma, n = 3.2, 1.2, 500
samples = list(range(0, 10000))

# data will hold all of the sample data
data = np.array([])

# sampling will hold the means of the samples
sampling = np.array([])

# Perform 10,000 samples
for s in samples:
    # In each sample, get 500 data points from a normal distribution
    sample = np.random.normal(mu, sigma, n)
    data = np.append(data,sample)
    sampling = np.append(sampling,sample.mean())

# Create a dataframe with the sampling of means
df = pd.DataFrame(sampling, columns=['mean'])

# Plot the distribution as a histogram
means = df['mean']
means.plot.hist(title='Simulated Sampling Distribution', bins=100)
plt.show()

# Print the Mean and StdDev for the full sample and for the sampling distribution
print('Sample Mean: ' + str(data.mean()))
print('Sample StdDev: ' + str(data.std()))
print ('Sampling Mean: ' + str(means.mean()))
print ('Sampling StdErr: ' + str(means.std()))
```

### Mean and Variance of the Sampling Distribution

The following variables are printed beneath the histogram:

- **Sample Mean**: This is the mean for the complete set of sample data - all 10,000 x 500 bag weights.
- **Sample StdDev**: This is the standard deviation for the complete set of sample data - all 10,000 x 500 bag weights.
- **Sampling Mean**: This is the mean for the sampling distribution - the means of the means!
- **Sampling StdErr**: This is the standard deviation (or *standard error*) for the sampling distribution

If we assume that **X** is a random variable representing every possible bag weight, then its mean (indicated as **μx**) is the population mean (**μ**). The mean of the **X** sampling distribution (which is indicated as **μx̄**) is considered to have the same value. Or, as an equation:

\begin{equation}\mu_{x} = \mu_{\bar{x}}\end{equation}

In this case, the full population mean is unknown (unless we weigh every bag in the world!), but we do have the mean of the full set of sample observations we collected (**x̄**), and if we check the values generated by Python for the sample mean and the sampling mean, they're more or less the same: around 3.2.

To find the standard deviation of the sample mean, which is technically the *standard error*, we can use this formula:

\begin{equation}\sigma_{\bar{x}}  = \frac{\sigma}{\sqrt{n}}\end{equation}

In this formula, ***σ*** is the population standard deviation and ***n*** is the size of each sample.

Since our the population standard deviation is unknown, we can use the full sample standard deviation instead:

\begin{equation}SE_{\bar{x}} \approx \frac{s}{\sqrt{n}}\end{equation}

In this case, the standard deviation of our set of sample data is around 1.2, and we have used 500 variables in each sample to calculate our sample means, so:

\begin{equation}SE_{\bar{x}} \approx \frac{1.2}{\sqrt{500}} = \frac{1.2}{22.36} \approx 0.053\end{equation}

## Confidence Intervals
A confidence interval is a range of values around a sample statistic within which we are confident that the true parameter lies. For example, our bag weight sampling distribution is based on samples of the weights of bags carried by passengers through our airport security line. We know that the mean weight (the *expected value* for the weight of a bag) in our sampling distribution is 3.2, and we assume this is also the population mean for all bags; but how confident can we be that the true mean weight of all carry-on bags is close to the value?

Let's start to put some precision onto these terms. We could state the question another way. What's the range of weights within which are confident that the mean weight of a carry-on bag will be 95% of the time? To calculate this, we need to determine the range of values within which the population mean weight is likely to be in 95% of samples. This is known as a *confidence interval*; and it's based on the Z-scores inherent in a normal distribution.

Confidence intervals are expressed as a sample statistic ± (*plus or minus*) a margin of error. To calculate the margin of error, you need to determine the confidence level you want to find (for example, 95%), and determine the Z score that marks the threshold above or below which the values that are *not* within the chosen interval reside. For example, to calculate a 95% confidence interval, you need the critical Z scores that exclude 5% of the values under the curve; with 2.5% of them being lower than the values in the confidence interval range, and 2.5% being higher. In a normal distribution, 95% of the area under the curve is between a Z score of ± 1.96. The following table shows the critical Z values for some other popular confidence interval ranges:

| Confidence  | Z Score |
|-------------|---------|
| 90%         | 1.645   |
| 95%         | 1.96    |
| 99%         | 2.576   |

To calculate a confidence interval around a sample statistic, we simply calculate the *standard error* for that statistic as described previously, and multiply this by the approriate Z score for the confidence interval we want.

To calculate the 95% confidence interval margin of error for our bag weights, we multiply our standard error of 0.053 by the Z score for a 95% confidence level, which is 1.96:

\begin{equation}MoE = 0.053 \times 1.96 = 0.10388 \end{equation}

So we can say that we're confident that the population mean weight is in the range of the sample mean ± 0.10388 with 95% confidence. Thanks to the central limit theorem, if we used an even bigger sample size, the confidence interval would become smaller as the amount of variance in the distribution is reduced. If the number of samples were infinite, the standard error would be 0 and the confidence interval would become a certain value that reflects the true mean weight for all carry-on bags:

\begin{equation}\lim_{n \to \infty} \frac{\sigma}{\sqrt{n}} = 0\end{equation}

In Python, you can use the *scipy.stats.**norm.interval*** function to calculate a confidence interval for a normal distribution. Run the following code to recreate the sampling distribution for bag searches with the same parameters, and display the 95% confidence interval for the mean (again, this may take some time to run):

```python
%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats

mu, sigma, n = 3.2, 1.2, 500
samples = list(range(0, 10000))

# data will hold all of the sample data
data = np.array([])

# sampling will hold the means of the samples
sampling = np.array([])

# Perform 10,000 samples
for s in samples:
    # In each sample, get 500 data points from a normal distribution
    sample = np.random.normal(mu, sigma, n)
    data = np.append(data,sample)
    sampling = np.append(sampling,sample.mean())

# Create a dataframe with the sampling of means
df = pd.DataFrame(sampling, columns=['mean'])

# Get the Mean, StdDev, and 95% CI of the means
means = df['mean']
m = means.mean()
sd = means.std()
ci = stats.norm.interval(0.95, m, sd)

# Plot the distribution, mean, and CI
means.plot.hist(title='Simulated Sampling Distribution', bins=100)
plt.axvline(m, color='red', linestyle='dashed', linewidth=2)
plt.axvline(ci[0], color='magenta', linestyle='dashed', linewidth=2)
plt.axvline(ci[1], color='magenta', linestyle='dashed', linewidth=2)
plt.show()

# Print the Mean, StdDev and 95% CI
print ('Sampling Mean: ' + str(m))
print ('Sampling StdErr: ' + str(sd))
print ('95% Confidence Interval: ' + str(ci))
```

```python

```

Statistics and Probability by Hiren/04-05-Sampling Distributions.ipynb


# Setup

```python
import numpy as num
import scipy.special as scis # scis.gamma(5) --> 24 == 4!
import matplotlib.pyplot as plt
import math # math.exp(), math.sqrt(), math.factorial()
import sympy as sym
import sympy.vector as symvect

sym.init_printing() # Only need to do this once per session.
```

```python
x,y,x1,y1,x2,y2,epsilon = sym.symbols('x y x1 y1 x2 y2 epsilon')
```

C is our coordinate system, with basis vectors $\hat{i}$ and $\hat{j}$ (and $\hat{k}$, but we won't use that).

```python
C = symvect.CoordSys3D('') # As long as we restrict ourselves to vectors involving i & j, we'll have 2D vectors.
        # We give the coordinate system no name (''), so we don't get unnecessary subscript clutter on the
        # basis unit vectors when pretty-printed.
```

```python
v1 = x1*C.i + y1*C.j  # Symbolic computation.
v2 = x2*C.i + y2*C.j
v1.dot(v2)
```

```python
v2-v1
```

# Dot product with side's left normal

Let points $p_1$ and $p_2$ be defined as:
$$
% Each line terminated with "\\"
p_1 = (x_1, y_1) \\
p_2 = (x_2, y_2)
$$

The "left normal" of a vector $\overrightarrow{p_{1}p_{2}}$ from $p_1$ to $p_2$, $(x_{2}, y_{2}) - (x_{1}, y_{1}) = (\Delta x, \Delta y)$, is $(-\Delta y, \Delta x)$.

```python
p1 = x1*C.i + y1*C.j
p2 = x2*C.i + y2*C.j
deltaY = y2-y1
deltaX = x2-x1
leftNormal = -deltaY*C.i + deltaX*C.j
leftNormal
```

Now we create a vector $\overrightarrow{p_{1}p}$ from $p_1$ to a test point $p$ and take its dot product with $\overrightarrow{p_{1}p_{2}}$:

```python
p = x*C.i + y*C.j
p1p = p - p1
p1p
```

```python
leftNormal.dot( p1p)
```

...which is obvious, but I had made the mistake of trying to expand the above expression
by hand and then simplify after cancelling some opposite-sign terms out and factoring.
I think that was doomed to failure.

Anyway, if the above expression is negative, then the test point $p$ is on the opposite side of the line segment $p_{1}p_{2}$ from $p_{1}p_{2}$'s left normal (so... it's on the right).

We see that the totologic blog post has the signs reversed (presumably because
he had the y-axis reversed for typical computer screens).

# Distance from point to side (segment)

We're going to find an expression for $d^2$, the distance $d$ of the test point $p$ from the triangle side $p_{1}p_{2}$, squared.

$\overrightarrow{p_{1}p_{2}} \cdot \overrightarrow{p_{1}p} = |\overrightarrow{p_{1}p_{2}}| \times |\overrightarrow{p_{1}p}| \times \cos{\theta}$, where $\theta$ is the angle between the vectors.

```python
q1 = 1*C.i + 1*C.j
q2 = 3*C.i + 2*C.j
[q1.magnitude(), q2.magnitude()]
```

```python
q1.dot( q2)
```

As you can see from above, multiplying big vectors _can_ result in big numbers.

Using the definitions of $p_1$, $p_2$, and $p$ from above, we define vectors $\overrightarrow{p_{1}p_{2}}$ and $\overrightarrow{p_{1}p}$:

```python
p1p2 = p2 - p1
p1p = p - p1
p1p2.dot( p1p)
```

$|\overrightarrow{p_{1}p}|^2$ is the length of a hypotenuse of a **right triangle**, squared.

```python
hypSqrd = p1p.magnitude()**2
hypSqrd
```

### On unit vectors...

A "unit vector" is a vector in the direction of some other vector, but having length 1.

For example, if we have a vector $(10,0)$ (i.e., extending along the x-axis), the unit vector in that direction is $(1,0)$.

Pretty simple, but suppose we have a vector $(2,3)$.  What is a vector pointing in the same direction but only having length 1?  We need to scale that vector back a bit, but by how much?  This is probably looking pretty obvious by now, but the length of $(2,3)$ is:

```python
b = (2*C.i + 3*C.j)
b.magnitude()
```

and the scaling is dividing by that length:

```python
(b / sym.sqrt(13)).magnitude()
```

So, unit vector $\hat{b}$ is

$$
\hat{b} = \frac{\vec{b}}{|\vec{b}|}
$$

So, the _projection_ of a vector $\vec{a}$ on to another vector $\vec{b}$ is a vector whose _length_ is the length of $\vec{a}$ scaled by the cosine of the angle between $\vec{a}$ and $\vec{b}$, and whose _direction_ is the same as $\hat{b}$.  In other words,

$$
\overrightarrow{a_b} = |\vec{a}| \times \cos \theta \times \hat{b}
$$

(where $\overrightarrow{a_b}$ is the projection of $\vec{a}$ onto $\vec{b}$.)

But that expression is just

$$
\vec{a} \cdot \frac{\vec{b}}{|\vec{b}|}
$$

Finally, the _length_ of the _projection_ is:

```python
(a_x, a_y, b_x, b_y) = sym.symbols('a_x a_y b_x b_y')
a = a_x*C.i + a_y*C.j
b = b_x*C.i + b_y*C.j

proj = a.dot( b / b.magnitude())
proj
```

(where $a_x$ and $a_y$ are the x- and y-components of $a$, and likewise for $b$.)

Or

```python
proj = (a.dot(b)) / b.magnitude()
proj
```

Note that we hate taking square roots unnecessarily when computering, so we're going to find a way to square the above expression to get rid of the radical....

### Having established the basics of projection...

One leg of the right triangle, the projection of $\overrightarrow{p_{1}p}$ onto $\overrightarrow{p_{1}p_{2}}$, has length squared equal to

$$
\frac{(\overrightarrow{p_{1}p_{2}} \cdot \overrightarrow{p_{1}p})^2}{|\overrightarrow{p_{1}p_{2}}|^2}
$$

```python
p1p2Sqrd = p1p2.magnitude()**2
sym.simplify( p1p2Sqrd)
```

First, real quick, let's check our variables and make sure nothing's changed, inadvertently.

```python
dict( p1=p1, p2=p2, p=p, p1p=p1p, p1p2=p1p2, p1p2Sqrd=p1p2Sqrd)
```

    {'p1': x1*.i + y1*.j,
     'p2': x2*.i + y2*.j,
     'p': x*.i + y*.j,
     'p1p': (x - x1)*.i + (y - y1)*.j,
     'p1p2': (-x1 + x2)*.i + (-y1 + y2)*.j,
     'p1p2Sqrd': (-x1 + x2)**2 + (-y1 + y2)**2}

```python
projSqrd = (p1p.dot( p1p2)**2 / p1p2Sqrd)
sym.simplify( projSqrd)
```

Length $d$ of remaining leg of triangle is unknown, *but* Pythagoras tells us

$$
|\overrightarrow{p_{1}p}|^2 - (\overrightarrow{p_{1}p} \cdot \overrightarrow{p_{1}p_{2}})^2 / |\overrightarrow{p_{1}p_{2}}|^2 = d^2
$$

```python
dSqrd = hypSqrd - projSqrd
dSqrd
```

## A little exercise checking correctness so far

If we set up a simple test of a vertical line of length 2 (from (0,-1) to (0,1)), and a test point on the
x-axis ($y=0$) being brought to the y-axis (x -> 0), then the distance of the test point from the line
segment should be exactly $x$. And it is (don't forget we're calculating distance SQUARED):

```python
dSqrd.subs(x1,0).subs(y1,-1).subs(x2,0).subs(y2,1).subs(y,0)
```

If our test point is at $(1,y)$ and we're projecting onto a vertical line on the y-axis, it doesn't matter
what our $x$ value is, because only the $y$ value determines what the projection will be. And if $y$ is
larger than the length of our (vertical, in this example) "triangle side" $p_{1}p_{2}$, then so be it, the projection will also be large.

```python
projSqrd.subs(x1,0).subs(y1,0).subs(x2,0).subs(y2,2).subs(x,1)
```

```python
projSqrd.subs(x1,0).subs(y1,0).subs(x2,0).subs(y2,2).subs(x,1).subs(y,10)
```

So, we have to check if the projection is larger than $|\overrightarrow{p_{1}p_{2}}|$, and, if so, we know that the test vector $\overrightarrow{p_{1}p}$ is actually LONGER than the triangle side, so we truncate the testing against $\epsilon^2$ by simply computing the distance from the test point $p$ to $p_2$, the other ("far") end of the triangle side.

That's not described in this document, but it is in the code.

## Reducing math operations

Ideally, we'd like to keep the number of multiplications and divisions to a minimum, and the number square roots and other transcendental functions to zero, in order to make the code fastfastfast.  (Well, it's gonna be Haskell, but, ya know... the principle of the thing.)

(Note on divisions: any time a division by the difference of two numbers occurs, that's a danger sign.  If the two numbers are almost equal in value, you're dividing by a very small value (yielding a very large value), and if there are errors in the two numbers you're subtracting (say, because they're just digital approximations of real values) you can wind up with very big errors in your result.  Avoid dividing by differences, if you can.  Or maybe find a way to throw a $+1$ in the denominator.)

So, our distance-to-side code looks like this:

   1. If dot product is negative, angle is obtuse, projection will be off side near $p_1$, and we might as well just calculate distance to $p_1$.
   2. If the projection is longer than $p_{1}p_{2}$, the same logic leads us to calculating the distance to $p_2$.  Since calculating the length of $p_{1}p_{2}$ involves dividing by a square root, let's try to avoid that.  We're just comparing two numbers for greater-than, anyway.
   3. Otherwise, yeah, go ahead and run the full algorithm, but _remember_: we just want to know if the distance is within epsilon ($\epsilon$), since we're only running this distance-to-side algorithm if previously-attempted algorithms have _failed_, declaring this point to be _outside_ the triangle.

Here's our code at this moment, before we start verifying/improving it, along with a monstrous chunk of commented-out copied from the totologic blog post:

```haskell
-- | Computes square of distance from given point to given side.
distanceSqrdPointToSide :: Point -> Point -> Point -> Double
distanceSqrdPointToSide
  (Point _ (x,y))               -- ^ The point under test
  (Point _ (x1,y1))             -- ^ Starting point of side
  (Point _ (x2,y2))             -- ^ Ending point of side
  = let p1p2LengthSqrd = (x2-x1)*(x2-x1) + (y2-y1)*(y2-y1)
        dotProduct = ((x-x1)*(x2-x1) + (y-y1)*(y2-y1)) / p1p2LengthSqrd
        pp1LengthSqrd = (x1 - x)*(x1 - x) + (y1 - y)*(y1 - y)
    in
      if dotProduct < 0       then pp1LengthSqrd -- Angle > 90°, closest point of side IS p1
      else if dotProduct <= 1 then pp1LengthSqrd - dotProduct * dotProduct * p1p2LengthSqrd
                              else (x - x2)*(x - x2) + (y - y2)*(y - y2)

      -- TODO: not at ALL sure this is correct.

{-
    function distanceSquarePointToSegment(x1, y1, x2, y2, x, y:Number):Number
    {
     var p1_p2_squareLength:Number = (x2 - x1)*(x2 - x1) + (y2 - y1)*(y2 - y1);
     var dotProduct:Number = ((x - x1)*(x2 - x1) + (y - y1)*(y2 - y1)) / p1_p2_squareLength;
     if ( dotProduct < 0 )
     {
      return (x - x1)*(x - x1) + (y - y1)*(y - y1);
     }
     else if ( dotProduct <= 1 )
     {
      var p_p1_squareLength:Number = (x1 - x)*(x1 - x) + (y1 - y)*(y1 - y);
      return p_p1_squareLength - dotProduct * dotProduct * p1_p2_squareLength;
     }
     else
     {
      return (x - x2)*(x - x2) + (y - y2)*(y - y2);
     }
    }
-}
```

```p1p2LengthSqrd``` looks right.  The signs are reversed, but who cares, because it's going to be squared.

```dotProduct``` needs to change, if possible, because we're dividing (bad) by differences (real bad).  How is it used?

  1. It's compared to 0 and 1.  Easy fix: compare to 0 and the thing it's divided by.
  2. It's multiplied by the thing it's divided by.  Duh, cancel those terms out.

But first, make sure that math is correct.  We're checking to see that the projection is shorter than the side.  ```p1pLengthSqrd``` is the _hypotenuse_ length, and we want to subtract the _length_ of the projection from it, to get $d^2$, the length of the unknown side.

In that calculation, why are we multiplying by ```p1p2LengthSqrd```?

Let's multiply ```dSqrd``` through by ```p1p2Sqrd```.

```python
dSqrd * p1p2Sqrd
```

```python
sym.simplify( dSqrd * p1p2Sqrd)
```

And multiply $\epsilon^2$ by the same figure (```p1p2Sqrd```) and compare those two values.  Means we need to pass $\epsilon$ as an argument to the distance function.  And/or rename it?  ```withinEpsilonOf```, maybe?


# Fast Personalized PageRank Python Implementation

I needed a fast PageRank for [Wikisim](https://github.com/asajadi/wikisim) project. It had to be fast enough to run real time on relatively large graphs. NetworkX was the obvious library to use, however, it needed back and forth translation from my graph representation (which was the pretty standard csr matrix), to its internal graph data structure. These translations were slowing down the process.

I implemented two versions of the algorithm in Python, both inspired by the sparse fast solutions given in [**Cleve Moler**](https://en.wikipedia.org/wiki/Cleve_Moler)'s book, [*Experiments with MATLAB*](https://www.mathworks.com/content/dam/mathworks/mathworks-dot-com/moler/exm/chapters/pagerank.pdf). The power method is much faster with enough precision for our task.

### Personalized PageRank
I modified the algorithm a little bit to be able to calculate **personalized PageRank** as well.

### Comparison with Popular Python Implementations: NetworkX and iGraph
Both implementations (exact solution and *power method*) are much faster than their correspondent methods in NetworkX. The *power method* is also faster than the iGraph native implementation, which is also an eigenvector based solution. Benchmarking is done on a `ml.t3.2xlarge` SageMaker instance.

### What is the major drawback of NetworkX PageRank?
I gave up using NetworkX for one simple reason: I had to calculate PageRank several times, and my internal representation of a graph was a simple sparse matrix. Every time I wanted to calculate PageRank I had to translate it to the graph representation of NetworkX, which was slow. My benchmarking shows that NetworkX  has a pretty fast implementation of PageRank ( `networkx.pagerank_numpy` and  '`networkx.pagerank_scipy`), but translating from its own graph data structure to a csr matrix before doing the actual calculations is exactly what exactly slows down the whole algorithm.

**Note**: I didn't count the time spent on `nx.from_scipy_sparse_matrix` (converting a csr matrix before passing it to NetworkX PageRank) in my benchmarking, But I could! Because that was another bottleneck for me, and for many other cases that one has a `csr` adjacency matrix.

### Python Implementation
The python package is hosted at https://github.com/asajadi/fast-pagerank and you can find the installation guide in the [README.md](https://github.com/asajadi/fast-pagerank#usage) file. You also can find this jupyter notebook in [the notebook directory](https://github.com/asajadi/fast-pagerank/blob/master/notebooks/Fast-PageRank.ipynb).

## Appendix

### What is Google PageRank Algorithm?
PageRank is another link analysis algorithm primarily used to rank search engine results. It is defined as a process in which starting  from a
random node, a random walker moves to a	random neighbour with probability $\alpha$  or jumps to a random vertex with the probability $1-\alpha$ . The PageRank values are the limiting probabilities of finding a walker on each
node. In the original PageRank, the jump can be to any node with a uniform probability, however later in **Personalized PageRank**, this can be any custom probability distribution over the nodes.

### How  Google PageRank is Calculated? [1, 2]

Let $\mathbf{A}$ be the adjacency matrix ($\mathbf{A}_{ij}$ is the weight of the edge from node $i$ to node $j$) and $\vec{s}$ be the *teleporting probability*, that is $\vec{s}_i$ is the probability of jumping to node $i$. Probability of being at node $j$ at time $t+1$  can be determined by two factors:
1. Sum over the out-neighbors $i$ of $j$ of the probability that the walk was at $i$ at time t, times the probability it moved from $i$ to $j$ in time $t+1$.
2. Probability of teleporting from somewhere else in the graph to $j$.

Formally:

\begin{equation}
    \vec{p}_{t+1}(j)=\alpha\sum_{i:(i,j)\in E}\frac{A(i,j)}{d(i)}\vec{p}_t(i)+(1-\alpha)\vec{s}_j,
\end{equation}

where $d(i)$ is the out-degree of node $i$.
To give a matrix form, we define $\mathbf{D}$ to be the diagonal matrix with the out-degree  of each node in $\mathbf{A}$ on
the diagonal. Then the PageRank
vector, initialized with $\vec{s}$, can be obtained from the following recursion:

\begin{equation}
    \vec{pr}_{t+1}=\alpha \mathbf{A}^T \mathbf{D}^{-1}\vec{pr}_{t}+(1-\alpha)\vec{s}.
\end{equation}

There is a serious problem that we need to take care: $\mathbf{D}^{-1}$ is the inverse of $\mathbf{D}$, which for a diagonal matrix it will be simply inverting the elements on the diagonal. This will break if there are nodes with no out neighbors, a.k.a, *dangling nodes*.
What happens when you hit a page with no out link? You only have one option and that is to jump to a random page.

To simulate this behavior we alter $\mathbf{A}$ by adding an edge from every dangling node to every other node $j$ with a weight of $\vec{s}_j$. In other words, we create $\mathbf{\bar{A}}$ by replacing each all zero row by $\vec{s}^T$. Formally, if we define $\vec{r}$ to be the vector of row-wise sum of the elements of $\mathbf{A}$, that is $\vec{r}_i=\sum_{j}A_{ij}$, then:

\begin{align}
\mathbf{\bar{A}}&=\mathbf{A}+\mathbf{B}\\
\mbox{where}\\
\mathbf{B}_{ij} &= \begin{cases}
                        \vec{s}_j & \mbox{if } r_i=0 \\
                        0   & \mbox{else}
                    \end{cases}
\end{align}

We need to re-define $\mathbf{D}$. In our new definition of $\mathbf{D}$, we ignore nodes with no out-neighbors (or in other words, replace $\frac{1}{0}$ by $0$). Similar to $\mathbf{D}$, we define $\mathbf{\bar{D}}$ to be the diagonal matrix of the out-degrees of $\mathbf{\bar{A}}$. So we can rewrite the recursion as:

\begin{equation}
    \vec{pr}_{t+1}=\alpha \mathbf{\bar{A}}^T \mathbf{\bar{D}}^{-1}\vec{pr}_{t}+(1-\alpha)\vec{s}. \tag{I}\label{I}
\end{equation}

Now $\vec{pr}$, the stationary probabilities (i.e, when $\vec{pr}_{t+1}=\vec{pr}_t=\vec{pr}$) can be calculated by either of the following approaches:

**1. Linear System Solving**

We can solve Eq. $\eqref{I}$ and get:

\begin{equation}
    \vec{pr}=(I-\alpha\mathbf{\bar{A}}^T \mathbf{\bar{D}}^{-1})(1-\alpha)\vec{s}.
\end{equation}

And use a linear system solver to calculate $\vec{pr}$.

**2. Power-Method**

Basically, reiterating the Eq. $\eqref{I}$ until it converges.

### How Fast Google PageRank Is Calculated? [3]
To speed up, we need to take advantage of sparse matrix calculations.  The only problem with the current formulation is that $\mathbf{\bar{A}}$ has a lower sparsity than the original $\mathbf{A}$. However, we can move around pieces of the equation a little bit to skip forming this matrix. We know that:

\begin{align}
\mathbf{\bar{A}}^T \mathbf{\bar{D}}
                       &= (\mathbf{A}^T+\mathbf{B}^T)\mathbf{\bar{D}}\\
                       &= \mathbf{A}^T\mathbf{\bar{D}}^{-1}
                       +\mathbf{B}^T\mathbf{\bar{D}}^{-1}
\end{align}

For the first term, multiplying by this diagonal matrix scales each column and $\mathbf{\bar{D}}$ and $\mathbf{D}$ are different only in the elements whose correspondent columns were all zero in $\mathbf{A}^T$, so we can safely replace $\mathbf{\bar{D}}$ with $\mathbf{D}$. Also  $\mathbf{B}^T\mathbf{\bar{D}}^{-1}=\mathbf{B}^T$ because the non zero columns of $\mathbf{B}^T$ are all $\vec{s}$, which add up to $1$, and therefore their correspondent element on $\mathbf{D}$ will be $1$. Therefore,

\begin{align}
\mathbf{\bar{A}}^T \mathbf{\bar{D}}
                       &= \mathbf{A}^T\mathbf{D}^{-1}
                       +\mathbf{B}^T,
\end{align}

and using the above equation we can rewrite Eq. $\eqref{I}$ and get

\begin{align}
    \vec{pr}_{t+1} &= \alpha \mathbf{A}^T\mathbf{D}^{-1}\vec{pr}_{t}
                    +\alpha\mathbf{B}^T\vec{pr}_{t}
                    +(1-\alpha)\vec{s}. \tag{II}\label{II}
\end{align}

This recursion has three multiplications, and the last one is a rather expensive one ($\mathbf{B}$ is a $n\times n$ matrix, therefore the whole multiplication will be $O(n^2)$).

Being a normalized vector, we know that $\vec{1}^T\vec{pr}_t=1$. We can multiply the last term of Eq. $\eqref{II}$ with $\vec{1}^T\vec{pr}_t$ and factor out $\vec{pr}$:

\begin{align}
    \vec{pr} &=  \alpha \mathbf{A}^T\mathbf{D}^{-1}\vec{pr}_t
                  +\alpha\mathbf{B}^T\vec{pr}_t
                  +(1-\alpha)\vec{s}\vec{1}^T\vec{pr}_t \\
                   &= \alpha \mathbf{A}^T\mathbf{D}^{-1}\vec{pr}_t+
                    (\alpha\mathbf{B}^T+
                    (1-\alpha)\vec{s}\vec{1}^T)\vec{pr}_t. \tag{III}\label{III}
\end{align}

Let $\mathbf{C}$ be $\alpha\mathbf{B}^T+(1-\alpha)\vec{s}\vec{1}^T$. Notice that $\vec{s}\vec{1}^T$ is a matrix with $\vec{s}$ as its columns, and substituting the definition of $\mathbf{B}$, the matrix $\mathbf{C}$ will be:

\begin{align}
\mathbf{C}_{ij} &= \begin{cases}
                        \vec{s}_i & \mbox{if } r_j=0 \\
                        (1-\alpha)\vec{s}_i & \mbox{else}
                    \end{cases}
\end{align}

If we let $\vec{z}$ be:

\begin{align}
\vec{z}_i &= \begin{cases}
                1 & \mbox{if } r_i=0 \\
                (1-\alpha) & \mbox{else}
             \end{cases}
\end{align}

then

\begin{equation}
\mathbf{C}=\vec{s}\vec{z}^T
\end{equation}

So by replacing  ($\alpha\mathbf{B}^T+(1-\alpha)\vec{s}\vec{1}^T$) in Eq. $\eqref{III}$ with $\vec{s}\vec{z}^T$, we'll get:

\begin{align}
    \vec{pr}_{t+1} &= \alpha \mathbf{A}^T\mathbf{D}^{-1}\vec{pr}_{t}+(\vec{s}\vec{z}^T)\vec{pr}_{t}. \tag{IV}\label{IV}
\end{align}

How does this help to improve the calculations? We'll see:

**1. Solving a Linear System**

Similar to before, we can solve Eq. $\eqref{IV}$ and get:

\begin{equation}
    \vec{pr}=(I-\alpha \mathbf{A}^T\mathbf{D}^{-1})^{-1}(\vec{s}\vec{z}^T)\vec{pr}.
\end{equation}

Being able to re-parenthesize, $\vec{z}^T\vec{p}$ is just a number, so we can ignore it and renormalize $\vec{pr}$ at the end, and solve:

\begin{equation}
    \vec{pr}=(I-\alpha \mathbf{A}^T\mathbf{D}^{-1})^{-1}\vec{s}.
\end{equation}

We almost have the same linear equation system that we had before, except for one big improvement, we replaced the less-sparse $\mathbf{\bar{A}}$ with $\mathbf{A}$.

**2. Power Method**

We can apply one last smart modification to Eq. $\eqref{IV}$: if we change the parenthesizing of the last multiplication ([remember the famous dynamic programming algorithm](https://en.wikipedia.org/wiki/Matrix_chain_multiplication)?), and also define $\mathbf{W}=\alpha\mathbf{A}^T\mathbf{D}^{-1}$, we will have:

\begin{equation}
\vec{pr}_{t+1} = \mathbf{W}\vec{pr}_{t}+
                \vec{s}(\vec{z}^T\vec{pr}_{t})
\end{equation}

Therefore, the complexity decreased to $O(n)$, and the whole recursion will be $O(n)\times \#iterations$. The rate of convergence is another thing, which we ignore here, and depends on the value of the second eigenvalue ($\lambda_2$) of the modified transition matrix ($\mathbf{T}$), which is defined as:
\begin{equation}
\mathbf{T}=\alpha\mathbf{A}^T\mathbf{D}^{-1}+\vec{s}\vec{z}^T
\end{equation}

## References

[1] [Daniel A. Spielman](https://en.wikipedia.org/wiki/Daniel_Spielman), Graphs and Networks Lecture Notes, [Lecture 11: Cutting Graphs, Personal PageRank and Spilling Paint](http://www.cs.yale.edu/homes/spielman/462/lect11-13.pdf), 2013.

[2] [Daniel A. Spielman](https://en.wikipedia.org/wiki/Daniel_Spielman), Spectral Graph Theory Lecture Notes, [Lecture 10: Random Walks on Graphs](http://www.cs.yale.edu/homes/spielman/561/lect10-18.pdf), 2018

[3] [Cleve Moler](https://en.wikipedia.org/wiki/Cleve_Moler), *Experiments with MATLAB*, [Chapter 7:
Google PageRank](https://www.mathworks.com/content/dam/mathworks/mathworks-dot-com/moler/exm/chapters/pagerank.pdf)

## Implementation

```python
%%writefile ../fast_pagerank/fast_pagerank.py
"""Two fast implementations of PageRank:
    An exact solution using a sparse linear system solver,
    and an a power method approximation.
    Both solutions are taking full advantage of sparse matrix calculations.

    [Reference]:
    Cleve Moler. 2011. Experiments with MATLAB (Electronic ed.).
    MathWorks, Inc.
"""
# uncomment
from __future__ import division

import scipy as sp
import scipy.sparse as sprs
import scipy.spatial
import scipy.sparse.linalg

__author__ = "Armin Sajadi"
__copyright__ = "Copyright 2015, The Wikisim Project"
__email__ = "asajadi@gmail.com"

def pagerank(A, p=0.85,
             personalize=None, reverse=False):
    """ Calculates PageRank given a csr graph

    Inputs:
    -------

    G: a csr graph.
    p: damping factor
    personlize: if not None, should be an array with the size of the nodes
                containing probability distributions.
                It will be normalized automatically
    reverse: If true, returns the reversed-PageRank

    outputs
    -------

    PageRank Scores for the nodes

    """
    # In Moler's algorithm, $A_{ij}$ represents the existences of an edge
    # from node $j$ to $i$, while we have assumed the opposite!
    if reverse:
        A = A.T

    n, _ = A.shape
    r = sp.asarray(A.sum(axis=1)).reshape(-1)

    k = r.nonzero()[0]

    D_1 = sprs.csr_matrix((1 / r[k], (k, k)), shape=(n, n))

    if personalize is None:
        personalize = sp.ones(n)
    personalize = personalize.reshape(n, 1)
    s = (personalize / personalize.sum()) * n

    I = sprs.eye(n)
    x = sprs.linalg.spsolve((I - p * A.T @ D_1), s)

    x = x / x.sum()
    return x

def pagerank_power(A, p=0.85, max_iter=100,
                   tol=1e-06, personalize=None, reverse=False):
    """ Calculates PageRank given a csr graph

    Inputs:
    -------
    A: a csr graph.
    p: damping factor
    max_iter: maximum number of iterations
    personlize: if not None, should be an array with the size of the nodes
                containing probability distributions.
                It will be normalized automatically.
    reverse: If true, returns the reversed-PageRank

    Returns:
    --------
    PageRank Scores for the nodes

    """
    # In Moler's algorithm, $G_{ij}$ represents the existences of an edge
    # from node $j$ to $i$, while we have assumed the opposite!
    if reverse:
        A = A.T

    n, _ = A.shape
    r = sp.asarray(A.sum(axis=1)).reshape(-1)

    k = r.nonzero()[0]

    D_1 = sprs.csr_matrix((1 / r[k], (k, k)), shape=(n, n))

    if personalize is None:
        personalize = sp.ones(n)
    personalize = personalize.reshape(n, 1)
    s = (personalize / personalize.sum()) * n

    z_T = (((1 - p) * (r != 0) + (r == 0)) / n)[sp.newaxis, :]
    W = p * A.T @ D_1

    x = s
    oldx = sp.zeros((n, 1))

    iteration = 0

    while sp.linalg.norm(x - oldx) > tol:
        oldx = x
        x = W @ x + s @ (z_T @ x)
        iteration += 1
        if iteration >= max_iter:
            break
    x = x / sum(x)

    return x.reshape(-1)
```

    Overwriting ../fast_pagerank/fast_pagerank.py

# Testing the algorithm

```python
%%writefile ../test/fast_pagerank_test.py
import os
import sys
import scipy as sp
import scipy.sparse as sparse
from numpy.testing import assert_allclose
import unittest

sys.path.insert(
    0,
    os.path.abspath(
        os.path.join(
            os.path.dirname(__file__),
            '..')))

from fast_pagerank import pagerank
from fast_pagerank import pagerank_power

class TestMolerPageRank(unittest.TestCase):
    def setUp(self):
        # ---G1---
        n1 = 5
        edges1 = sp.array([[0, 1],
                           [1, 2],
                           [2, 1],
                           [2, 3],
                           [2, 4],
                           [3, 0],
                           [3, 2],
                           [4, 0],
                           [4, 2],
                           [4, 3]])
        weights1 = [0.4923,
                    0.0999,
                    0.2132,
                    0.0178,
                    0.5694,
                    0.0406,
                    0.2047,
                    0.8610,
                    0.3849,
                    0.4829]

        self.p1 = 0.83
        self.personalize1 = sp.array([0.6005, 0.1221, 0.2542, 0.4778, 0.4275])
        self.G1 = sparse.csr_matrix(
            (weights1, (edges1[:, 0], edges1[:, 1])), shape=(n1, n1))
        self.pr1 = sp.array([0.1592, 0.2114, 0.3085, 0.1, 0.2208])

        # ---G2---
        n2 = 10
        edges2 = sp.array([[2, 4],
                           [2, 5],
                           [4, 5],
                           [5, 3],
                           [5, 4],
                           [5, 9],
                           [6, 1],
                           [6, 2],
                           [9, 2],
                           [9, 4]])
        weights2 = [0.4565,
                    0.2861,
                    0.5730,
                    0.0025,
                    0.4829,
                    0.3866,
                    0.3041,
                    0.3407,
                    0.2653,
                    0.8079]
        self.G2 = sparse.csr_matrix(
            (weights2, (edges2[:, 0], edges2[:, 1])), shape=(n2, n2))
        self.personalize2 = sp.array([0.8887, 0.6491, 0.7843, 0.7103, 0.7428,
                                      0.6632, 0.7351, 0.3006, 0.8722, 0.1652])
        self.p2 = 0.92
        self.pr2 = sp.array([0.0234, 0.0255, 0.0629, 0.0196, 0.3303,
                             0.3436, 0.0194, 0.0079, 0.023, 0.1445])

        # ---G3---
        n3 = 5
        edges3 = sp.array([[2, 4]])
        weights3 = [0.5441]
        self.G3 = sparse.csr_matrix(
            (weights3, (edges3[:, 0], edges3[:, 1])), shape=(n3, n3))

        self.personalize3 = sp.array([0.0884, 0.2797, 0.3093, 0.5533, 0.985])
        self.p3 = 0.81
        self.pr3 = sp.array([0.0358, 0.1134, 0.1254, 0.2244, 0.501])

        # ---G4---
        n4 = 5
        edges4_rows = []
        edges4_cols = []
        weights4 = []
        self.G4 = sparse.csr_matrix(
            (weights4, (edges4_rows, edges4_cols)), shape=(n4, n4))

        self.personalize4 = sp.array([0.2534, 0.8945, 0.9562, 0.056, 0.9439])
        self.p4 = 0.70
        self.pr4 = sp.array([0.0816, 0.2882, 0.3081, 0.018, 0.3041])

        # ---G5---
        n5 = 0
        edges5_rows = []
        edges5_cols = []
        weights5 = []
        self.G5 = sparse.csr_matrix(
            (weights5, (edges5_rows, edges5_cols)), shape=(n5, n5))

        self.personalize5 = sp.array([])
        self.p5 = 0.70
        self.pr5 = sp.array([])

    def test_pagerank_1(self):
        calculated_pagerank = pagerank(self.G1, p=self.p1,
                                       personalize=self.personalize1)
        assert_allclose(calculated_pagerank, self.pr1, rtol=0, atol=1e-04)

    def test_pagerank_2(self):

        calculated_pagerank = pagerank(self.G2, p=self.p2,
                                       personalize=self.personalize2)
        assert_allclose(calculated_pagerank, self.pr2, rtol=0, atol=1e-04)

    def test_single_edge(self):
        calculated_pagerank = pagerank(self.G3, p=self.p3,
                                       personalize=self.personalize3)
        assert_allclose(calculated_pagerank, self.pr3, rtol=0, atol=1e-04)

    def test_zero_edge(self):
        calculated_pagerank = pagerank(self.G4, p=self.p4,
                                       personalize=self.personalize4)
        assert_allclose(calculated_pagerank, self.pr4, rtol=0, atol=1e-04)

    def test_empty_graph(self):
        calculated_pagerank = pagerank(self.G5, p=self.p5,
                                       personalize=self.personalize5)
        self.assertEqual(calculated_pagerank.size, 0)

    def test_power_pagerank_1(self):
        calculated_pagerank = pagerank_power(self.G1, p=self.p1,
                                             personalize=self.personalize1)
        assert_allclose(calculated_pagerank, self.pr1, rtol=0, atol=1e-04)

    def test_power_pagerank_2(self):

        calculated_pagerank = pagerank_power(self.G2, p=self.p2,
                                             personalize=self.personalize2)
        assert_allclose(calculated_pagerank, self.pr2, rtol=0, atol=1e-04)

    def test_power_single_edge(self):
        calculated_pagerank = pagerank_power(self.G3, p=self.p3,
                                             personalize=self.personalize3)
        assert_allclose(calculated_pagerank, self.pr3, rtol=0, atol=1e-04)

    def test_power_zero_edge(self):
        calculated_pagerank = pagerank_power(self.G4, p=self.p4,
                                             personalize=self.personalize4)
        assert_allclose(calculated_pagerank, self.pr4, rtol=0, atol=1e-04)

    def test_power_empty_graph(self):
        calculated_pagerank = pagerank_power(self.G5, p=self.p5,
                                             personalize=self.personalize5)
        self.assertEqual(calculated_pagerank.size, 0)

#             assert_array_almost_equal(Ynx,  Yml, decimal = 5)
if __name__ == '__main__':
    unittest.main()
```

    Overwriting ../test/fast_pagerank_test.py

```python
!python  ../test/fast_pagerank_test.py
```

    /Users/arminsajadi/anaconda3/lib/python3.7/site-packages/numpy/matrixlib/defmatrix.py:71: PendingDeprecationWarning: the matrix subclass is not the recommended way to represent matrices or deal with linear algebra (see https://docs.scipy.org/doc/numpy/user/numpy-for-matlab-users.html). Please adjust your code to use regular ndarray.
      return matrix(data, dtype=dtype, copy=False)
    ..........
    ----------------------------------------------------------------------
    Ran 10 tests in 0.020s

    OK

# Benchmarking

To avoid the clutter, we only visualize the fastest method from each implementation, that is:

- `networkx.pagerank_scipy`
- Latest implementation of  `iGraph.personalized_pagerank` (PRPACK)
- Our `pagerank_power`

```python
''' Calcualate PageRank on several random graphs.
'''
import scipy as sp
import pandas as pd
import timeit
import os
import sys
import random
import igraph
import networkx as nx
sys.path.insert(0, '..')
from fast_pagerank.pagerank import pagerank
from fast_pagerank.pagerank import pagerank_power

# def print_and_flush(args):

#     sys.stdout.flush()
def get_random_graph(
        min_size=20,
        max_size=2000,
        min_density=0.1,
        max_density=0.5):
    ''' Creates a random graph and a teleport vector and output them
        in different formats for different algorithms

    Inputs
    ------

    min_size and max_size: The size of the graph will be a random number
        in the range of (min_size, max_size)
    min_sparsity and max_sparsity: The sparcity of the graph
        will be a random number in the range of (min_sparsity, max_sparsity)

    Returns
    -------

    nxG: A random Graph for NetworkX
    A: The equivallent csr Adjacency matrix, for our PageRank
    iG: The equivallent iGraph
    personalize_vector: Personalization probabily vector
    personalize_dict: Personalization probabily vector,
                    in the form of a dictionary for NetworkX

    '''
    G_size = random.randint(min_size, max_size)
    p = random.uniform(min_density, max_density)

    A = sp.sparse.random(G_size, G_size, p, format='csr')
    nxG = nx.from_scipy_sparse_matrix(A, create_using=nx.DiGraph())

    iG = igraph.Graph(list(nxG.edges()), directed=True)
    iG.es['weight'] = A.data

    personalize_vector = sp.random.random(G_size)
    personalize_dict = dict(enumerate(personalize_vector.reshape(-1)))
    return A, nxG, iG, personalize_vector, personalize_dict

n = 5
number_of_graphs = 50

node_size_vector = sp.zeros(number_of_graphs)
edge_size_vector = sp.zeros(number_of_graphs)
# netx_pagerank_times = sp.zeros(number_of_graphs)
netx_pagerank_times_numpy = sp.zeros(number_of_graphs)
netx_pagerank_times_scipy = sp.zeros(number_of_graphs)
ig_pagerank_times = sp.zeros(number_of_graphs)
pagerank_times = sp.zeros(number_of_graphs)
pagerank_times_power = sp.zeros(number_of_graphs)

damping_factor = 0.85
tol = 1e-3

for i in range(number_of_graphs):
    A, nxG, iG, personalize_vector, personalize_dict = get_random_graph()
    node_size_vector[i] = A.shape[0]
    edge_size_vector[i] = A.count_nonzero()
    print ("Graph %d: Nodes: %d, Edges: %d ..." %(i, node_size_vector[i], edge_size_vector[i]))
    sys.stdout.flush()

#     networkx.pagerank commented out, because it is too slow

#     netx_pagerank_times[i] = timeit.timeit(
#         lambda: nx.pagerank(nxG, alpha=damping_factor, tol=tol),
#         number=n) / n

    netx_pagerank_times_numpy[i] = timeit.timeit(
        lambda: nx.pagerank_numpy(nxG, alpha=damping_factor),
        number=n) / n

    netx_pagerank_times_scipy[i] = timeit.timeit(
        lambda: nx.pagerank_scipy(nxG, alpha=damping_factor, tol=tol),
        number=n) / n

    #iGraph, only "prpack", which is their latest version.
    ig_pagerank_times[i] = timeit.timeit(
        lambda: iG.personalized_pagerank(directed=True,
                                         damping=damping_factor,
                                         weights=iG.es['weight'],
                                         implementation="prpack"),
        number=n) / n

#     My implementations

    pagerank_times[i] = timeit.timeit(
        lambda: pagerank(A, p=damping_factor),
        number=n) / n
    pagerank_times_power[i] = timeit.timeit(
        lambda: pagerank_power(A, p=damping_factor, tol=tol),
        number=n) / n

argsort = edge_size_vector.argsort()

edge_size_vector_sorted = edge_size_vector[argsort]
node_size_vector_sorted = node_size_vector[argsort]

# netx_pagerank_times_sorted = netx_pagerank_times[argsort]
netx_pagerank_times_numpy_sorted = netx_pagerank_times_numpy[argsort]
netx_pagerank_times_scipy_sorted = netx_pagerank_times_scipy[argsort]

ig_pagerank_times_sorted = ig_pagerank_times[argsort]

pagerank_times_sorted = pagerank_times[argsort]
pagerank_times_power_sorted = pagerank_times_power[argsort]

comparison_table = pd.DataFrame(list(zip(node_size_vector_sorted,
                                         edge_size_vector_sorted,
#                                          netx_pagerank_times_sorted,
                                         netx_pagerank_times_numpy_sorted,
                                         netx_pagerank_times_scipy_sorted,
                                         ig_pagerank_times_sorted,
                                         pagerank_times_sorted,
                                         pagerank_times_power_sorted)),
                                columns=['Nodes', 'Edges',
#                                          'NetX',
                                         'NetX (numpy)',
                                         'NetX (scipy)',
                                         'iGraph',
                                         '(fast) pagerank',
                                         '(fast) pagerank_power']).\
                    astype({'Nodes': 'int32', 'Edges': 'int32'})
comparison_table.to_csv('pagerank_methods_comparison.csv')
print("Done")
```

# Plotting

```python
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

comparison_table = pd.read_csv('pagerank_methods_comparison.csv', index_col=0)
display(comparison_table)

plt.ioff()
fig=plt.figure(num=None, figsize=(10, 10), dpi=80, facecolor='w', edgecolor='k')

# plt.plot(comparison_table['Edges'], comparison_table['NetX'],
#          'o-', ms=8, lw=2, alpha=0.7, color='cyan',
#          label='networkx.PageRank')
plt.plot(comparison_table['Edges'], comparison_table['NetX (numpy)'],
         'v-', ms=8, lw=2, alpha=0.7, color='magenta',
         label='networkx.PageRank_numpy')

plt.plot(comparison_table['Edges'], comparison_table['NetX (scipy)'],
         'P-', ms=8, lw=2, alpha=0.7, color='blue',
         label='networkx.PageRank_scipy')

plt.plot(comparison_table['Edges'], comparison_table['iGraph'],
         'x-', ms=8, lw=2, alpha=0.7, color='black',
         label='iGraph_PageRank_ARPACK')

plt.plot(comparison_table['Edges'], comparison_table['(fast) pagerank'],
         '*-', ms=8, lw=2, alpha=0.7, color='red',
         label='fast_pagerank.pagerank')

plt.plot(comparison_table['Edges'], comparison_table['(fast) pagerank_power'],
         '^-', ms=8, lw=2, alpha=0.7, color='green',
         label='fast_pagerank.pagerank_power')

plt.xlabel('Number of the edges')
plt.ylabel('Time (Seconds)')

plt.tight_layout()
plt.legend(loc=2)
plt.savefig('pagerank_methods_comparison.png')
plt.show()

```


---
## 31. Busqueda de Raices

Eduard Larrañaga (ealarranaga@unal.edu.co)

---

### Resumen

En este cuaderno se describen algunas de las técnicas para encontrar numéricamente las raices de una función.

---

En muchas ocasiones, es necesario encontrar las raices de una función , i.e. aquellos valores de la variable independiente $x_r$ para los cuales $f(x_r) = 0$. En general, $x_r$ puede representar uno o multiples variables y $f$ puede ser una sola función o puede representar un sistema de ecuaciones. También es posible que $f$ dependa explicitamente de $x$ o que $f$ dependa implicitamente de $x$.

---
## Método de Newton-Raphson

Existen multiples métodos para encontrar las raices de una función, pero el más popular de ellos es el método de **Newton-Raphson**. Para describir esta técnica, debemos considerar la expansión en serie de Taylor de la función $f(x)$ alrededor de la raíz $x_r$,

\begin{equation}
f(x_r) = f(x) + (x_r - x) f'(x) +  \mathcal{O}( (x_r - x)^2) \approx 0 .
\end{equation}

El valor $x_r$ debe entenderse como el valor de la raíz en el paso $n$-ésimo en un proceso iterativo. Por lo tanto, en el paso $n+1$ de la iteración, a primer orden, se tendrá

\begin{equation}
f(x_{n+1}) = f(x_n) + (x_{n+1} - x_{n})
f'(x_n) \approx 0 ,
\end{equation}

de donde despejamos

\begin{equation}
x_{n+1} =  x_n - \frac{f(x_n)}{f'(x_n)} .
\end{equation}

La implementación de este procedmiento iterativo se dentendrá cuando el cambio fraccionario entre los resultados de la iteración $n$ y $n+1$ isea más pequeña que la precisión de punto flotante (epsilon de la máquina)

\begin{equation}
\left| \frac{[f(x_{n+1}) - f(x_{n})]}{f({x_n})} \right| < \epsilon .
\end{equation}

La convergencia del método de Newton-Raphson Method es de segundo orden si $f(x)$ se comporta bien y si el valor incial para el procedimiento  $x_0$ es un punto cercano al valor de la raíz.

Nota Importante: Este método necesita conocer la forma analítica de la primera derivada $f'(x)$. Si este no es el caso, se necesitará evaluar numéricamnete esta derivada y la técnica se denominará el **método de la secante**.

---
### Método de la secante

El método de la secante es similar al método de Newton-Raphson, pero la primera derivada $f'(x_n)$ es evaluada numericamente. Usualmente esta se evalua con diferencias hacia atrás, con lo que se obtiene el proceso iterativo

\begin{equation}
x_{n+1} = x_{n} - f(x_n)\frac{x_n - x_{n-1}}{f(x_n) - f(x_{n-1})}\,\,.
\end{equation}

Note que este proceso iterativo necesita dos puntos para iniciar, o al menos un valor inicial de la primera derivada, i.e. $f'(x_0)$.  Debido a la inclusion del procedimiento numérico para estimar la primera derivada, el método de la secante converge mas lentamente que el método de Newton-Raphson.

#### Ejemplo. Raices de una función discreta utilizando el método de Newton-Raphson ( secante )

La información de la función sera leida de un archivo .txt y se encontrará numéricamente la raiz de la función aplicando el método de Newton-Raphson (o secante). Se utilizará una interpolación cuadrática por intervalos para modelar la función.

Antrs de localizar numéricamente la rai´z, se visualizará el comportamiento de la función.

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# Reading the data
data = np.loadtxt( 'data_points.txt', comments='#', delimiter=',')
x_data = data[:,0]
f_data = data[:,1]

plt.figure(figsize=(7,5))
plt.plot(x_data, f_data, color='red')
plt.title('')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.grid()
plt.show()
```

 Del gráfico, se observa qye la raíz debe ubicarse en el intervalo entre $x=0$ y $x=2$. Por ello nos restringiremos a esta región:

```python
plt.figure(figsize=(7,5))
plt.plot(x_data, f_data, color='red')
plt.scatter(x_data, f_data, color='black')
plt.title('')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.xlim(0,2)
plt.ylim(-10,10)
plt.grid()
plt.show()
```

Este zoom muestra el interior del intervalo  $1<x<1.75$. Nos concentraremos en los tres puntos en el medio del intervalo con el fin de interpolar un polinómio cuadrático y aplicar el método de busqueda de raices.

```python
# Find the initial and final points of the restricted interval of
# three points to interpolate.
for i in range(len(x_data)-1):
    if x_data[i]>=1:
        a=i
        break

# The interval will include three points:
# x[a], x[a+1], x[a+2]
# Here we define the arrays that enter as arguments
# for the interpolated polynomial
x_int =[x_data[a], x_data[a+1], x_data[a+2]]
f_int =[f_data[a], f_data[a+1], f_data[a+2]]

print('a = ', a, ': x_a = ', x_data[a], ': f(x_a) = ', f_data[a])
print('a = ', 31, ': x_a = ', x_data[31], ': f(x_a) = ', f_data[31])
```

    a =  30 : x_a =  1.12245 : f(x_a) =  2.44474
    a =  31 : x_a =  1.32653 : f(x_a) =  -0.0279679

Ahora, utilizamos el polinomio cuadrático interpolado y utilizamos el métodos de Newton-Raphson (o secante).

```python
# 2-order Interpolated Polynomial
def p2(x):
    pol = (((x-x_int[1])*(x-x_int[2]))/((x_int[0]-x_int[1])*(x_int[0]-x_int[2])))*f_int[0] +\
          (((x-x_int[0])*(x-x_int[2]))/((x_int[1]-x_int[0])*(x_int[1]-x_int[2])))*f_int[1] +\
          (((x-x_int[0])*(x-x_int[1]))/((x_int[2]-x_int[0])*(x_int[2]-x_int[1])))*f_int[2]
    return pol

# Newton-Raphson Method. We begin with the value at point x[a]
root0 = x_data[a]-1e-5
root = x_data[a]

while np.abs(p2(root))>1e-10:
    new_root = root - p2(root)*(root - root0)/(p2(root) - p2(root0))
    root0, root = root, new_root

print(f'La raiz es x_r = {root:.8f}')
print(f'para la cual la función interpolada da un valor de f(x_r) =  {p2(root):.8e}')

```

    La raiz es x_r = 1.32429525
    para la cual la función interpolada da un valor de f(x_r) =  7.09501902e-16

Podemos visualizar la ubicación de la raíz,

```python
plt.figure(figsize=(7,5))
plt.plot(x_data, f_data, color='red')
plt.scatter(x_data, f_data, color='black')
plt.scatter(root, p2(root), color='blue', marker='*',s=150)
plt.vlines(root, -7, p2(root), linestyle='dashed')
plt.hlines(p2(root),0, root, linestyle='dashed')
plt.title('Root')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.xlim(0.5,2)
plt.ylim(-7,7)
plt.grid()
plt.show()
```

---
## Método de la Bisección

El método de bisección esta basado en el teorema del valor intermedio, que establee que una función continua $f(x)$ debe poseer al menos una raíz en el intervalo $[x_a,x_b]$ si $f(x_a)$ y
$f(x_b)$ tienen signos opuestos.

De esta forma, el método de la bisección se implementa en la siguiente forma,

1. Se eligen dos valores iniciales  $x_a$ y $x_b$ tales que $f(x_a)$ y $f(x_b)$ tnegan signos opuestos.

2. Se calcula el punto medio $x_c = \frac{x_a+x_b}{2}$ y el valor de la función allí, $f(x_c)$.

3. Si $f(x_c) = 0$ o
  $\left| \frac{f(x_c) - f(x_a)}{f(x_a)} \right| < \epsilon$ or $\left| \frac{f(x_c) - f(x_b)}{f(x_b)} \right| <
  \epsilon$, la raíz se ha encontrado.

  Si este no es el caso, entonces:
    * si $f(x_a)$ y $f(x_c)$ tienen signos opuestos, entonces ellos encierran la raíz. Volvemos al punto (2) con $x_a=x_a$ y $x_b=x_c$.

    * Si $f(x_c)$ y $f(x_b)$ tienen signos opuestos, entonces ellos encierran la raíz. Volvemos al punto (2) con $x_a=x_c$ y $x_b=x_b$.

A pesar de que el método de la bisección es muy robusto y eficiente, generalmente no es tán rápido como el método de Newton porque requiere de más iteraciones para encontrar la raíz.

#### Ejemplo. Raices de una función discreta con le método de la bisección.

```python
import numpy as np
import matplotlib.pyplot as plt

# Reading the data
data = np.loadtxt('data_points.txt', comments='#', delimiter=',')
x_data = data[:,0]
f_data = data[:,1]

plt.figure(figsize=(7,5))
plt.plot(x_data, f_data, color='red')
plt.title('')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.grid()
plt.show()
```

```python
plt.figure(figsize=(7,5))
plt.plot(x_data, f_data, color='red')
plt.scatter(x_data, f_data, color='black')
plt.title('')
plt.xlabel(r'$x$')
plt.ylabel(r'$f(x)$')
plt.xlim(0,2)
plt.ylim(-10,10)
plt.grid()
plt.show()
```

```python
# Find the initial and final points of the restricted interval of
# three pints to interpolate.
for i in range(len(x_data)-1):
    if x_data[i]>=1:
        a=i
        break

# The interval will include three points:
# x[a], x[a+1], x[a+2]
# Here we define the arrays that enter as arguments
# for the interpolated polynomial
x_int =[x_data[a], x_data[a+1], x_data[a+2]]
f_int =[f_data[a], f_data[a+1], f_data[a+2]]

print('a = ', a, ': x_a = ', x_data[a], ': f(x_a) = ', f_data[a])

```

    a =  30 : x_a =  1.12245 : f(x_a) =  2.44474

Ahora, interpolamos el polinomio cuadrático y utilizamos el método de la bisección para encontrar la raíz.

```python
# 2-order Interpolated Polynomial
def p2(x):
    pol = (((x-x_int[1])*(x-x_int[2]))/((x_int[0]-x_int[1])*(x_int[0]-x_int[2])))*f_int[0] +\
          (((x-x_int[0])*(x-x_int[2]))/((x_int[1]-x_int[0])*(x_int[1]-x_int[2])))*f_int[1] +\
          (((x-x_int[0])*(x-x_int[1]))/((x_int[2]-x_int[0])*(x_int[2]-x_int[1])))*f_int[2]
    return pol

# Bisection Method.
# We begin with the values at the points x_data[a] and x_data[a+2]
x_a = x_data[a]
x_b = x_data[a+2]
x_c = (x_a + x_b)/2.

while np.abs(p2(x_c)) > 1e-15:
    x_c = (x_a + x_b)/2.
    if p2(x_a)*p2(x_c) < 0:
        x_b = x_c
    else:
        x_a = x_c

root = x_c

print(f'La raíz es x_r = {root:.8f}')
print(f'para la cual la función interpolada da un valor de f(x_r) =  {p2(root):.8e}')

```

    La raíz es x_r = 1.32429525
    para la cual la función interpolada da un valor de f(x_r) =  7.09501902e-16

```python

```

04. Busqueda de Raices/01. Busqueda de Raices.ipynb


# Statistics

## 1.1 Sales for the past week was the following amounts: [3505, 2400, 3027, 2798, 3700, 3250, 2689]. Without using library functions, what is the mean, variance, and standard deviation of of sales from last week? (for extra bonus points, write your own function that can calculate these two values for any sized list)

```
# mean
amounts = [3505, 2400, 3027, 2798, 3700, 3520, 2689]
mean_value = sum(amounts)/len(amounts)
print("mean is: " + str(mean_value))

```

    mean is: 3091.285714285714

```
# standard deviation
def stdev(List):
  avg=sum(List)*1.0/len(List)
  var=list(map(lambda x: (x - avg)**2, List))
  return(sum(var)*1.0/len(var))**0.5
print(stdev(amounts))
```

    455.9679544772901

```
def var(values):
  return mean([(i-mean_value) ** 2 for i in values])
print(var(amounts))
```

    207906.7755102041

```
import numpy as np
import math
import pandas as pd
```

```
# checking from lecture
def subtract_mean(number):
  return number-mean_value

lambda x:x-mean_value
```

    <function __main__.<lambda>>

```
distances = list(map(subtract_mean, amounts))
distances
```

    [413.7142857142858,
     -691.2857142857142,
     -64.28571428571422,
     -293.2857142857142,
     608.7142857142858,
     428.7142857142858,
     -402.2857142857142]

```
squared_distances = list(map(lambda x:x**2, distances))
```

```
list_length = len(squared_distances)-1
list_length
```

    6

```
# variance
variance_amounts = sum(squared_distances)/list_length
variance_amounts
```

    242557.90476190482

```
df['amounts'].mean()
```

    3091.285714285714

```
print(df['amounts'].var(ddof=0))
print(df['amounts'].std(ddof=1))

# used 0 for ddof in var to check on the value i got for the function created above
```

    207906.7755102041
    492.5016799584595

## 1.2 Find the covariance between last week's sales numbers and the number of customers that entered the store last week: [127, 80, 105, 92, 120, 115, 93] (you may use librray functions for calculating the covariance since we didn't specifically talk about its formula)

```
# new list
customers = [127, 80, 105, 92, 120, 115, 93]
```

```
# cov from numpy
np.cov(amounts, customers)
```

    array([[242557.9047619 ,   8073.64285714],
           [  8073.64285714,    290.95238095]])

```
pd.DataFrame(np.cov(amounts, customers))
```

      0
      1

      0
      242557.904762
      8073.642857

      1
      8073.642857
      290.952381

```
df = pd.DataFrame({'amounts': amounts})
df['customers'] = pd.Series(customers)
df
```

      amounts
      customers

      0
      3505
      127

      1
      2400
      80

      2
      3027
      105

      3
      2798
      92

      4
      3700
      120

      5
      3520
      115

      6
      2689
      93

```
# cov from pandas
df.cov()
```

      amounts
      customers

      amounts
      242557.904762
      8073.642857

      customers
      8073.642857
      290.952381

```

```

## 1.3 Find the standard deviation of customers who entered the store last week. Then, use the standard deviations of both sales and customers to standardize the covariance to find the correlation coefficient that summarizes the relationship between sales and customers. (You may use library functions to check your work.)

```
# standard deviation of customers & sales amount
std_c = df['customers'].std(ddof=1)
std_a = df['amounts'].std(ddof=1)
std_c, std_a
```

    (17.057326313123664, 492.5016799584595)

```
# covariance with pandas
cov_ac = df['amounts'].cov(df['customers'])
cov_ac
```

    8073.642857142856

```
denom = std_a*std_c
denom
```

    8400.761864813041

```
# correlation
corr_ac = cov_ac/denom
corr_ac
```

    0.9610607927073451

```
df['amounts'].corr(df['customers'])
```

    0.9610607927073453

```
df.corr()
```

      amounts
      customers

      amounts
      1.000000
      0.961061

      customers
      0.961061
      1.000000

## 1.4 Use pandas to import a cleaned version of the titanic dataset from the following link: [Titanic Dataset](https://raw.githubusercontent.com/Geoyi/Cleaning-Titanic-Data/master/titanic_clean.csv)

## Calculate the variance-covariance matrix and correlation matrix for the titanic dataset's numeric columns. (you can encode some of the categorical variables and include them as a stretch goal if you finish early)

```
url = 'https://raw.githubusercontent.com/Geoyi/Cleaning-Titanic-Data/master/titanic_clean.csv'
titanic = pd.read_csv(url, index_col=0)
titanic.head()
```

      pclass
      survived
      name
      sex
      age
      sibsp
      parch
      ticket
      fare
      cabin
      embarked
      boat
      body
      home.dest
      has_cabin_number

      1
      1.0
      1.0
      Allen, Miss. Elisabeth Walton
      female
      29.0000
      0.0
      0.0
      24160
      211.3375
      B5
      S
      2
      NaN
      St Louis, MO
      1

      2
      1.0
      1.0
      Allison, Master. Hudson Trevor
      male
      0.9167
      1.0
      2.0
      113781
      151.5500
      C22 C26
      S
      11
      NaN
      Montreal, PQ / Chesterville, ON
      1

      3
      1.0
      0.0
      Allison, Miss. Helen Loraine
      female
      2.0000
      1.0
      2.0
      113781
      151.5500
      C22 C26
      S
      NaN
      NaN
      Montreal, PQ / Chesterville, ON
      1

      4
      1.0
      0.0
      Allison, Mr. Hudson Joshua Creighton
      male
      30.0000
      1.0
      2.0
      113781
      151.5500
      C22 C26
      S
      NaN
      135.0
      Montreal, PQ / Chesterville, ON
      1

      5
      1.0
      0.0
      Allison, Mrs. Hudson J C (Bessie Waldo Daniels)
      female
      25.0000
      1.0
      2.0
      113781
      151.5500
      C22 C26
      S
      NaN
      NaN
      Montreal, PQ / Chesterville, ON
      1

```
# covariance for the entire dataset
titanic.cov()
```

      pclass
      survived
      age
      sibsp
      parch
      fare
      body
      has_cabin_number

      pclass
      0.701969
      -0.127248
      -3.954605
      0.053090
      0.013287
      -24.227788
      -2.876653
      -0.249992

      survived
      -0.127248
      0.236250
      -0.314343
      -0.014088
      0.034776
      6.146023
      0.000000
      0.061406

      age
      -3.954605
      -0.314343
      165.850021
      -2.559806
      -1.459378
      114.416613
      81.622922
      1.463138

      sibsp
      0.053090
      -0.014088
      -2.559806
      1.085052
      0.336833
      8.641768
      -8.708471
      -0.003946

      parch
      0.013287
      0.034776
      -1.459378
      0.336833
      0.749195
      9.928031
      4.237190
      0.013316

      fare
      -24.227788
      6.146023
      114.416613
      8.641768
      9.928031
      2678.959738
      -179.164684
      10.976961

      body
      -2.876653
      0.000000
      81.622922
      -8.708471
      4.237190
      -179.164684
      9544.688567
      3.625689

      has_cabin_number
      -0.249992
      0.061406
      1.463138
      -0.003946
      0.013316
      10.976961
      3.625689
      0.174613

```
titanic.corr()
```

      pclass
      survived
      age
      sibsp
      parch
      fare
      body
      has_cabin_number

      pclass
      1.000000
      -0.312469
      -0.366370
      0.060832
      0.018322
      -0.558629
      -0.034642
      -0.713857

      survived
      -0.312469
      1.000000
      -0.050199
      -0.027825
      0.082660
      0.244265
      NaN
      0.302250

      age
      -0.366370
      -0.050199
      1.000000
      -0.190747
      -0.130872
      0.171892
      0.059059
      0.271887

      sibsp
      0.060832
      -0.027825
      -0.190747
      1.000000
      0.373587
      0.160238
      -0.099961
      -0.009064

      parch
      0.018322
      0.082660
      -0.130872
      0.373587
      1.000000
      0.221539
      0.051099
      0.036806

      fare
      -0.558629
      0.244265
      0.171892
      0.160238
      0.221539
      1.000000
      -0.043110
      0.507253

      body
      -0.034642
      NaN
      0.059059
      -0.099961
      0.051099
      -0.043110
      1.000000
      0.083796

      has_cabin_number
      -0.713857
      0.302250
      0.271887
      -0.009064
      0.036806
      0.507253
      0.083796
      1.000000

```
# heatmap
import seaborn as sns
sns.heatmap(titanic.corr());
```

# Orthogonality

## 2.1 Plot two vectors that are orthogonal to each other. What is a synonym for orthogonal?

```
# orthogonal is perpendicular (having a right angle)
import matplotlib.pyplot as plt
vector_1 = [0, 3]
vector_2 = [-4, 0]
```

```
fig, ax = plt.subplots(figsize=(5,5))
ax.grid()
plt.xlim(-5, 5)
plt.ylim(-5, 5)
for vect in [vector_1, vector_2]:
  plt.arrow(0,0, vect[0], vect[1], head_width=.2, head_length=0.2, linewidth=4)
plt.show()
```

```
# check if orthogonal, dot product must be == 0
np.dot(vector_1, vector_2)
```

    0

## 2.2 Are the following vectors orthogonal? Why or why not?

\begin{align}
a = \begin{bmatrix} -5 \\ 3 \\ 7 \end{bmatrix}
\qquad
b = \begin{bmatrix} 6 \\ -8 \\ 2 \end{bmatrix}
\end{align}

```
a = np.array([-5,3,7])
b = np.array([6,-8,2])
np.dot(a,b)
# no because the dot product does not equal to 0
```

    -40

## 2.3 Compute the following values: What do these quantities have in common?

## What is $||c||^2$?

## What is $c \cdot c$?

## What is $c^{T}c$?

\begin{align}
c = \begin{bmatrix} 2 & -15 & 6 & 20 \end{bmatrix}
\end{align}

```
c = np.array([2, -15, 6, 20])
c
```

    array([  2, -15,   6,  20])

```
c_squared = list(map(lambda x: x**2, c))
c_squared
```

    [4, 225, 36, 400]

```
cxc = np.multiply(c,c)
cxc
```

    array([  4, 225,  36, 400])

```
cT = c.T
cTc = np.multiply(cT, c)
print(cT)
cTc
```

    [  2 -15   6  20]

    array([  4, 225,  36, 400])

```
# all quantities yield to same resultts
```

# Unit Vectors

## 3.1 Using Latex, write the following vectors as a linear combination of scalars and unit vectors:

\begin{align}
d = \begin{bmatrix} 7 \\ 12 \end{bmatrix}
\qquad
e = \begin{bmatrix} 2 \\ 11 \\ -8  \end{bmatrix}
\end{align}

\begin{table}[]
\begin{tabular}{|l|l|}
\hline
vector d & vector e \\ \hline
7        & 2        \\ \hline
12       & 11       \\ \hline
         & -8       \\ \hline
\end{tabular}
\end{table}

## 3.2 Turn vector $f$ into a unit vector:

\begin{align}
f = \begin{bmatrix} 4 & 12 & 11 & 9 & 2 \end{bmatrix}
\end{align}

```
# unit vector
f = np.array([4, 12 ,11, 9, 2])
f_hat = f/np.linalg.norm(f)
print(f)
f_hat
```

    [ 4 12 11  9  2]

    array([0.20908335, 0.62725005, 0.57497921, 0.47043754, 0.10454167])

# Linear Independence / Dependence

## 4.1 Plot two vectors that are linearly dependent and two vectors that are linearly independent (bonus points if done in $\mathbb{R}^3$).

```
# linear dependent vectors
vector_01 = [1.5, -.7]
vector_02 = np.multiply(vector_01, 2)
vector_01, vector_02
```

    ([1.5, -0.7], array([ 3. , -1.4]))

```
# plot
fig, ax = plt.subplots(figsize=(4,4))
ax.grid()
plt.xlim(-4, 4)
plt.ylim(-4, 4)
for myvect, mycolor, mywidth in [(vector_01, 'blue', 15), (vector_02, 'red', 5)]:
  plt.arrow(0,0,
            myvect[0],
            myvect[1],
            head_width=.2,
            head_length=.2,
            linewidth=mywidth,
            color=mycolor,
            length_includes_head=True);
```

```
# linearly independent
vector_01 = [1.5, -.7]
vector_03 = [-2, -1.3]
vector_01, vector_03
```

    ([1.5, -0.7], [-2, -1.3])

```
fig, ax = plt.subplots(figsize=(4,4))
ax.grid()
plt.xlim(-4, 4)
plt.ylim(-4, 4)
for myvect, mycolor, mywidth in [(vector_01, 'blue', 15), (vector_03, 'yellow', 5)]:
  plt.arrow(0,0,
            myvect[0],
            myvect[1],
            head_width=.2,
            head_length=.2,
            linewidth=mywidth,
            color=mycolor,
            length_includes_head=True);
```

```

```

# Span

## 5.1 What is the span of the following vectors?

\begin{align}
g = \begin{bmatrix} 1 & 2 \end{bmatrix}
\qquad
h = \begin{bmatrix} 4 & 8 \end{bmatrix}
\end{align}

```
vector_g = np.array([1, 2])
vector_h = np.array([4, 8])
fig, ax = plt.subplots(figsize=(4,4))
ax.grid()
plt.xlim(0,10)
plt.ylim(0,10)

# Display
for myvect in [vector_g, vector_h]:
    plt.arrow(0,0,
              myvect[0],
              myvect[1],
              head_width=.2,
              head_length=0.2,
              linewidth=5,
              color='magenta')
```

```
# possible linear combination
a,b=(4,2)
linear_combo=np.multiply(vector_g, a) + np.multiply(vector_g, b)
print(linear_combo)
```

    [ 6 12]

```
combo_sets=[]
for (a,b) in [(2,3), (4,5), (6,7), (8,9)]:
  linear_combo = np.multiply(vector_1, a) + np.multiply(vector_2, b)
  combo_sets.append(linear_combo)
print(combo_sets)
```

    [array([-12,   6]), array([-20,  12]), array([-28,  18]), array([-36,  24])]

## 5.2 What is the span of $\{l, m, n\}$?

\begin{align}
l = \begin{bmatrix} 1 & 2 & 3 \end{bmatrix}
\qquad
m = \begin{bmatrix} -1 & 0 & 7 \end{bmatrix}
\qquad
n = \begin{bmatrix} 4 & 8  & 2\end{bmatrix}
\end{align}

```
vector_l = np.array([1,2,3])
vector_m = np.array([-1,0,7])
vector_n = np.array([4,8,2])

```

```
a,b,c = (2,3,4)
linear_com = np.multiply(vector_l, a) + np.multiply(vector_m, b) + np.multiply(vector_n, c)
print(linear_com)
```

    [15 36 35]

# Basis

## 6.1 Graph two vectors that form a basis for $\mathbb{R}^2$

```
# find basis vectors in 2-d
vector_1 = [0, 3]
vector_2 = [2, 0]

# Plot the Scaled Vectors
plt.arrow(0,0, vector_1[0], vector_1[1],head_width=.05, head_length=0.05, color ='purple')
plt.arrow(0,0, vector_2[0], vector_2[1],head_width=.05, head_length=0.05, color ='navy')
plt.xlim(-3,3)
plt.ylim(-1,4)
plt.title("Basis for R^2")
plt.show()
```

## 6.2 What does it mean to form a basis?

form a family of linearly independent vectors which spans a given vector space

# Rank

## 7.1 What is the Rank of P?

\begin{align}
P = \begin{bmatrix}
1 & 2 & 3 \\
 -1 & 0 & 7 \\
4 & 8  & 2
\end{bmatrix}
\end{align}

```
# rank 3
P = np.array([[1,2,3],[-1,0,7],[4,8,2]])
np.linalg.matrix_rank(P)
```

    3

## 7.2 What does the rank of a matrix tell us?

Rank means the number of dimensions in the output of a matrix that's used for linear transformation.

# Linear Projections

## 8.1 Line $L$ is formed by all of the vectors that can be created by scaling vector $v$
\begin{align}
v = \begin{bmatrix} 1 & 3 \end{bmatrix}
\end{align}

\begin{align}
w = \begin{bmatrix} -1 & 2 \end{bmatrix}
\end{align}

## find $proj_{L}(w)$

## graph your projected vector to check your work (make sure your axis are square/even)

```
# vectors
v = np.array([1,3])
w = np.array([-1,2])
# dot product
w_dot_v = np.dot(v, w)
v_dot_v = np.dot(v, v)
#projection
fraction = w_dot_v/v_dot_v
projection = np.multiply(fraction,v)
print(projection)
```

    [0.5 1.5]

```
# Plot
plt.xlim(0,4)
plt.ylim(0,4)

axes = plt.gca()
plt.axes().set_aspect('equal')
plt.plot(v, w, '--', color='orange', linewidth=1);
```

# Stretch Goal

## For vectors that begin at the origin, the coordinates of where the vector ends can be interpreted as regular data points. (See 3Blue1Brown videos about Spans, Basis, etc.)

## Write a function that can calculate the linear projection of each point (x,y) (vector) onto the line y=x. run the function and plot the original points in blue and the new projected points on the line y=x in red.

## For extra points plot the orthogonal vectors as a dashed line from the original blue points to the projected red points.

```
import pandas as pd
import matplotlib.pyplot as plt

# Creating a dataframe for you to work with -Feel free to not use the dataframe if you don't want to.
x_values = [1, 4, 7, 3, 9, 4, 5 ]
y_values = [4, 2, 5, 0, 8, 2, 8]

data = {"x": x_values, "y": y_values}

df = pd.DataFrame(data)

df.head()

plt.scatter(df.x, df.y)
plt.show()
```

```

```