# Attempt to Replicate the Black Scholes Model Using a Neural Network

```python
import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
import matplotlib.pyplot as plt
import scipy.stats as ss
import time
```

## Define the Function
Here we define our function, the Black Scholes Model (BSM). First, we must initialize the option class, then the Geometric Brownian Motion Class, and finally the BSM class.
Then we test the equation with a test value of 2.

```python
'''=========
option class init
=========='''
class VanillaOption:
    def __init__(
        self,
        otype = 1, # 1: 'call'
                  # -1: 'put'
        strike = 110.,
        maturity = 1.,
        market_price = 10.):
      self.otype = otype               # Put or Call
      self.strike = strike             # Strike K
      self.maturity = maturity         # Maturity T
      self.market_price = market_price #this will be used for calibration

    def payoff(self, s): #s: excercise price
      otype = self.otype
      k = self.strike
      maturity = self.maturity
      return np.max([0, (s - k)*otype])

```

```python
'''============
Gbm class
============='''

class Gbm:
    def __init__(self,
                 init_state = 100.,
                 drift_ratio = .0475,
                 vol_ratio = .2
                ):
        self.init_state = init_state
        self.drift_ratio = drift_ratio
        self.vol_ratio = vol_ratio
```

```python
'''========
Black-Scholes-Merton formula.
=========='''

def bsm_price(self, vanilla_option):
    s0 = self.init_state
    sigma = self.vol_ratio
    r = self.drift_ratio

    otype = vanilla_option.otype
    k = vanilla_option.strike
    maturity = vanilla_option.maturity

    d1 = 1/(sigma*np.sqrt(maturity))*(np.log(s0/k) + (r + np.power(sigma,2)/2)*(maturity))
    d2 = 1/(sigma*np.sqrt(maturity))*(np.log(s0/k) + (r - np.power(sigma,2)/2)*(maturity))
    return (otype * s0 * ss.norm.cdf(otype * d1) #line break needs parenthesis
            - otype * np.exp(-r * maturity) * k * ss.norm.cdf(otype * d2))

Gbm.bsm_price = bsm_price
```

```python
'''=======
Get BSM prices given an option and a Tensor
======='''

def prices_bsm(self, vanilla_option, data):

  # Create the list
  a = []

  # Get tensor size
  sizeT = list(data.size())[0]

  for i in range(sizeT):
    self.init_state = data[i].item()
    callPrice = gbm1.bsm_price(vanilla_option)
    a.append(callPrice)

  # Convert to array then tensor
  arrayOut = np.array(a)
  outputData = torch.from_numpy(arrayOut)
  return outputData

Gbm.prices_bsm = prices_bsm
```

```python
def f(s, k = 90):
  gbm = Gbm(init_state=s)
  option = VanillaOption(strike=k)
  return gbm.bsm_price(option)

batch_size = 63
x_list = np.linspace(0, 200, batch_size)
y_list = np.array([f(x) for x in x_list])

```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: RuntimeWarning: divide by zero encountered in log

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: RuntimeWarning: divide by zero encountered in log
      from ipykernel import kernelapp as app

## Create Model
Next, we create the neural network model. This is done first by setting the inner and outer dimensions with variables. Next we code the model and vary the internal dimensions to attempt to improve the model. At this level, this is essentially a simple linear algebra exercise:

If we have input $x$, internal parameters $a,b$, and solution $f(x)$ then in the one-dimensional case we have:

\begin{equation}
  \left(
    a_{1}x+b_{2}
  \right)
  a_{2} + b_{2}
  = f(x)
\end{equation}

However, we want to get a better estimate for the true equation. So we increase the interior dimension which corresponds to the number of neurons inside the network. For example, we raised the inner dimension to 3. In matrix form we have:

\begin{equation}
\left(
  \begin{bmatrix} x \end{bmatrix}
  \begin{bmatrix} a_{1} & a_{2} & a_{3} \end{bmatrix}
  +
  \begin{bmatrix} b_{1} & b_{2} & b_{3} \end{bmatrix}
\right)
 \begin{bmatrix} a_{4} \\ a_{5} \\ a_{6} \end{bmatrix}
 +
 \begin{bmatrix} b_{4} \\ \end{bmatrix}
 =
 \begin{bmatrix} f(x) \end{bmatrix}
\end{equation}

Graphically, we can render this second neural network as:

**ADD DESCRIPTION**

```python
#model
#nn.Linear
H1 = 80; H2 = 20 #number of hidden layer
model = nn.Sequential(
    nn.Linear(1, H1),
    nn.Sigmoid(),
    nn.Linear(H1, H2),
    nn.Sigmoid(),
    nn.Linear(H2,2),
    nn.Sigmoid(),
    nn.Linear(2,1)
)
```

Here we define the Loss function as the Mean Squared Error(MSE).

Note that by doing so, we are essentially 'cheating' the system. In most applications, we would not know the function $f$ so we would be unable to find the MSE.

```python
#loss function
criterion = nn.MSELoss()
```

Next we choose a learning rate and a method for learning. The learning rate is the percent of the data that is accepted in each iteration. The Methods we tried were SGD and Adam.

```python
#optimizer
learning_rate = 0.1
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.7)
```

```python
batch_size = np.size(x_list)
x_train0 = torch.from_numpy(x_list).reshape(batch_size,1).float()
y_train0 = torch.from_numpy(y_list).reshape(batch_size,1).float()
```

## Train the Model

First we create the training data. This is a batch of random points that we pass through the BSM.

Once we have the training data, we pass this collection of inputs and solutions into the model. With each iteration we calculate the loss and attempt to optimize the model to further reduce the loss.

```python
# Normalization

def linear_transform(xx, l = 0, u= 1):
  M = torch.max(xx)
  m = torch.min(xx)
  return (u-l)/(M-m)*(xx-m)+l, m, M, l, u
x_train, x_m, x_M, x_l, x_u = linear_transform(x_train0, -1, 1)
y_train, y_m, y_M, y_l, y_u = linear_transform(y_train0, 0, 1)
```

```python
# Train the model

num_epochs = 5000

for epoch in range(num_epochs):

    # Forward pass
    outputs = model(x_train.float())
    loss = criterion(outputs, y_train.float())

    # Backward and optimize
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if epoch == 0 or (epoch+1) % 50 == 0:
        print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1,
                                                    num_epochs, loss.item()))
        #print(outputs[1:10])

```

    Epoch [1/5000], Loss: 0.2129
    Epoch [50/5000], Loss: 0.1069
    Epoch [100/5000], Loss: 0.1040
    Epoch [150/5000], Loss: 0.0978
    Epoch [200/5000], Loss: 0.0823
    Epoch [250/5000], Loss: 0.0469
    Epoch [300/5000], Loss: 0.0165
    Epoch [350/5000], Loss: 0.0090
    Epoch [400/5000], Loss: 0.0059
    Epoch [450/5000], Loss: 0.0041
    Epoch [500/5000], Loss: 0.0030
    Epoch [550/5000], Loss: 0.0023
    Epoch [600/5000], Loss: 0.0018
    Epoch [650/5000], Loss: 0.0015
    Epoch [700/5000], Loss: 0.0013
    Epoch [750/5000], Loss: 0.0012
    Epoch [800/5000], Loss: 0.0011
    Epoch [850/5000], Loss: 0.0010
    Epoch [900/5000], Loss: 0.0009
    Epoch [950/5000], Loss: 0.0009
    Epoch [1000/5000], Loss: 0.0009
    Epoch [1050/5000], Loss: 0.0009
    Epoch [1100/5000], Loss: 0.0008
    Epoch [1150/5000], Loss: 0.0008
    Epoch [1200/5000], Loss: 0.0008
    Epoch [1250/5000], Loss: 0.0008
    Epoch [1300/5000], Loss: 0.0008
    Epoch [1350/5000], Loss: 0.0008
    Epoch [1400/5000], Loss: 0.0007
    Epoch [1450/5000], Loss: 0.0007
    Epoch [1500/5000], Loss: 0.0007
    Epoch [1550/5000], Loss: 0.0007
    Epoch [1600/5000], Loss: 0.0007
    Epoch [1650/5000], Loss: 0.0007
    Epoch [1700/5000], Loss: 0.0007
    Epoch [1750/5000], Loss: 0.0007
    Epoch [1800/5000], Loss: 0.0007
    Epoch [1850/5000], Loss: 0.0006
    Epoch [1900/5000], Loss: 0.0006
    Epoch [1950/5000], Loss: 0.0006
    Epoch [2000/5000], Loss: 0.0006
    Epoch [2050/5000], Loss: 0.0006
    Epoch [2100/5000], Loss: 0.0006
    Epoch [2150/5000], Loss: 0.0006
    Epoch [2200/5000], Loss: 0.0006
    Epoch [2250/5000], Loss: 0.0006
    Epoch [2300/5000], Loss: 0.0006
    Epoch [2350/5000], Loss: 0.0006
    Epoch [2400/5000], Loss: 0.0006
    Epoch [2450/5000], Loss: 0.0006
    Epoch [2500/5000], Loss: 0.0006
    Epoch [2550/5000], Loss: 0.0005
    Epoch [2600/5000], Loss: 0.0005
    Epoch [2650/5000], Loss: 0.0005
    Epoch [2700/5000], Loss: 0.0005
    Epoch [2750/5000], Loss: 0.0005
    Epoch [2800/5000], Loss: 0.0005
    Epoch [2850/5000], Loss: 0.0005
    Epoch [2900/5000], Loss: 0.0005
    Epoch [2950/5000], Loss: 0.0005
    Epoch [3000/5000], Loss: 0.0005
    Epoch [3050/5000], Loss: 0.0005
    Epoch [3100/5000], Loss: 0.0005
    Epoch [3150/5000], Loss: 0.0005
    Epoch [3200/5000], Loss: 0.0005
    Epoch [3250/5000], Loss: 0.0005
    Epoch [3300/5000], Loss: 0.0005
    Epoch [3350/5000], Loss: 0.0005
    Epoch [3400/5000], Loss: 0.0005
    Epoch [3450/5000], Loss: 0.0004
    Epoch [3500/5000], Loss: 0.0004
    Epoch [3550/5000], Loss: 0.0004
    Epoch [3600/5000], Loss: 0.0004
    Epoch [3650/5000], Loss: 0.0004
    Epoch [3700/5000], Loss: 0.0004
    Epoch [3750/5000], Loss: 0.0004
    Epoch [3800/5000], Loss: 0.0004
    Epoch [3850/5000], Loss: 0.0004
    Epoch [3900/5000], Loss: 0.0004
    Epoch [3950/5000], Loss: 0.0004
    Epoch [4000/5000], Loss: 0.0004
    Epoch [4050/5000], Loss: 0.0004
    Epoch [4100/5000], Loss: 0.0004
    Epoch [4150/5000], Loss: 0.0004
    Epoch [4200/5000], Loss: 0.0004
    Epoch [4250/5000], Loss: 0.0004
    Epoch [4300/5000], Loss: 0.0004
    Epoch [4350/5000], Loss: 0.0004
    Epoch [4400/5000], Loss: 0.0004
    Epoch [4450/5000], Loss: 0.0004
    Epoch [4500/5000], Loss: 0.0003
    Epoch [4550/5000], Loss: 0.0003
    Epoch [4600/5000], Loss: 0.0003
    Epoch [4650/5000], Loss: 0.0003
    Epoch [4700/5000], Loss: 0.0003
    Epoch [4750/5000], Loss: 0.0003
    Epoch [4800/5000], Loss: 0.0003
    Epoch [4850/5000], Loss: 0.0003
    Epoch [4900/5000], Loss: 0.0003
    Epoch [4950/5000], Loss: 0.0003
    Epoch [5000/5000], Loss: 0.0003

## Testing the Model

```python
def learnedfun(x):
  out = (1-(-1))/(x_M-x_m)*(x-x_m)+(-1.)
  out = model(out)
  out = (y_M- y_m)*out+y_m
  return out

y_pred = learnedfun(x_train0)
```

```python
# Test with training data
plt.scatter(x_train0.detach().numpy(), y_train0.detach().numpy(), label='true')
plt.scatter(x_train0.detach().numpy(), y_pred.detach().numpy(), label='pred')

plt.legend()
plt.show()
```

```python
# Test the Model with Random data

# Generate random data
x_0 = np.random.randint(0, 200, batch_size)

# Get BSM Prices as determined by the formula
y_0 = np.array([f(x) for x in x_0])

# Transform to tensor
x_0 = torch.from_numpy(x_0).reshape(batch_size,1).float()
y_0 = torch.from_numpy(y_0).reshape(batch_size,1).float()

# Normalize
x_, x_m, x_M, x_l, x_u = linear_transform(x_0)
y_, y_m, y_M, y_l, y_u = linear_transform(y_0)

# Plot x_ versus formula prices
plt.scatter(x_0.detach().numpy(), y_0.detach().numpy(), label='true')

# Get BSM Prices as determined by the model
y_pred = learnedfun(x_0)

# Plot x_ versus the model prices
plt.scatter(x_0.detach().numpy(), y_pred.detach().numpy(), label='pred')

plt.legend()

```

Put the above into a loop and check each number of neurons and layers

```python
# Begin loop through each
# TODO Add time Checking
neuronsList = [[50,12],
               [50,12,10],
               [50,30],
               [50,50],
               [50,50,21],
               [50,100],
               [80,20],
               [100,80],
               [100,100],
               [100,120],
               [100,150],
               [500,100]]
for numNeurons in neuronsList:
  # Define the Model
  if len(numNeurons) == 2:
    H1 = numNeurons[0]; H2 = numNeurons[1] #number of hidden layer neurons
    model = nn.Sequential(
      nn.Linear(1, H1),
      nn.Sigmoid(),
      nn.Linear(H1, H2),
      nn.Sigmoid(),
      nn.Linear(H2,2),
      nn.Sigmoid(),
      nn.Linear(2,1))
  else:
    H1 = numNeurons[0]; H2 = numNeurons[1]; H3 = numNeurons[2] #number of hidden layer neurons
    model = nn.Sequential(
      nn.Linear(1, H1),
      nn.Sigmoid(),
      nn.Linear(H1, H2),
      nn.Sigmoid(),
      nn.Linear(H2,H3),
      nn.Sigmoid(),
      nn.Linear(H3,2),
      nn.Sigmoid(),
      nn.Linear(2,1))

  #loss function
  criterion = nn.MSELoss()

  #optimizer
  learning_rate = 0.1
  optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.7)

  batch_size = np.size(x_list)
  x_train0 = torch.from_numpy(x_list).reshape(batch_size,1).float()
  y_train0 = torch.from_numpy(y_list).reshape(batch_size,1).float()

  # Normalization
  x_train, x_m, x_M, x_l, x_u = linear_transform(x_train0, -1, 1)
  y_train, y_m, y_M, y_l, y_u = linear_transform(y_train0, 0, 1)

  # Train the model

  print("Model: " + str(numNeurons))
  start = time.time()
  num_epochs = 5000

  for epoch in range(num_epochs):

      # Forward pass
      outputs = model(x_train.float())
      loss = criterion(outputs, y_train.float())

      # Backward and optimize
      optimizer.zero_grad()
      loss.backward()
      optimizer.step()

      if epoch == 0 or (epoch+1) % 50 == 0:
          print ('Epoch [{}/{}], Loss: {:.8f}'.format(epoch+1,
                                                      num_epochs, loss.item()))
          #print(outputs[1:10])
  end = time.time()
  timeEl = end-start
  print("Total Time Elapsed (s): " + str(timeEl))
  print("")
```

    Model: [50, 12]
    Epoch [1/5000], Loss: 0.33672205
    Epoch [50/5000], Loss: 0.10862195
    Epoch [100/5000], Loss: 0.10738762
    Epoch [150/5000], Loss: 0.10563789
    Epoch [200/5000], Loss: 0.10259616
    Epoch [250/5000], Loss: 0.09643578
    Epoch [300/5000], Loss: 0.08241504
    Epoch [350/5000], Loss: 0.05174055
    Epoch [400/5000], Loss: 0.01728370
    Epoch [450/5000], Loss: 0.00771000
    Epoch [500/5000], Loss: 0.00525315
    Epoch [550/5000], Loss: 0.00392157
    Epoch [600/5000], Loss: 0.00308993
    Epoch [650/5000], Loss: 0.00253730
    Epoch [700/5000], Loss: 0.00215737
    Epoch [750/5000], Loss: 0.00189060
    Epoch [800/5000], Loss: 0.00170050
    Epoch [850/5000], Loss: 0.00156336
    Epoch [900/5000], Loss: 0.00146328
    Epoch [950/5000], Loss: 0.00138933
    Epoch [1000/5000], Loss: 0.00133389
    Epoch [1050/5000], Loss: 0.00129160
    Epoch [1100/5000], Loss: 0.00125865
    Epoch [1150/5000], Loss: 0.00123235
    Epoch [1200/5000], Loss: 0.00121077
    Epoch [1250/5000], Loss: 0.00119254
    Epoch [1300/5000], Loss: 0.00117669
    Epoch [1350/5000], Loss: 0.00116252
    Epoch [1400/5000], Loss: 0.00114955
    Epoch [1450/5000], Loss: 0.00113742
    Epoch [1500/5000], Loss: 0.00112590
    Epoch [1550/5000], Loss: 0.00111483
    Epoch [1600/5000], Loss: 0.00110409
    Epoch [1650/5000], Loss: 0.00109360
    Epoch [1700/5000], Loss: 0.00108331
    Epoch [1750/5000], Loss: 0.00107319
    Epoch [1800/5000], Loss: 0.00106321
    Epoch [1850/5000], Loss: 0.00105336
    Epoch [1900/5000], Loss: 0.00104363
    Epoch [1950/5000], Loss: 0.00103402
    Epoch [2000/5000], Loss: 0.00102451
    Epoch [2050/5000], Loss: 0.00101510
    Epoch [2100/5000], Loss: 0.00100579
    Epoch [2150/5000], Loss: 0.00099657
    Epoch [2200/5000], Loss: 0.00098744
    Epoch [2250/5000], Loss: 0.00097840
    Epoch [2300/5000], Loss: 0.00096944
    Epoch [2350/5000], Loss: 0.00096056
    Epoch [2400/5000], Loss: 0.00095175
    Epoch [2450/5000], Loss: 0.00094302
    Epoch [2500/5000], Loss: 0.00093436
    Epoch [2550/5000], Loss: 0.00092577
    Epoch [2600/5000], Loss: 0.00091724
    Epoch [2650/5000], Loss: 0.00090878
    Epoch [2700/5000], Loss: 0.00090038
    Epoch [2750/5000], Loss: 0.00089204
    Epoch [2800/5000], Loss: 0.00088376
    Epoch [2850/5000], Loss: 0.00087553
    Epoch [2900/5000], Loss: 0.00086737
    Epoch [2950/5000], Loss: 0.00085926
    Epoch [3000/5000], Loss: 0.00085120
    Epoch [3050/5000], Loss: 0.00084320
    Epoch [3100/5000], Loss: 0.00083526
    Epoch [3150/5000], Loss: 0.00082737
    Epoch [3200/5000], Loss: 0.00081953
    Epoch [3250/5000], Loss: 0.00081174
    Epoch [3300/5000], Loss: 0.00080401
    Epoch [3350/5000], Loss: 0.00079633
    Epoch [3400/5000], Loss: 0.00078871
    Epoch [3450/5000], Loss: 0.00078114
    Epoch [3500/5000], Loss: 0.00077362
    Epoch [3550/5000], Loss: 0.00076616
    Epoch [3600/5000], Loss: 0.00075875
    Epoch [3650/5000], Loss: 0.00075139
    Epoch [3700/5000], Loss: 0.00074409
    Epoch [3750/5000], Loss: 0.00073685
    Epoch [3800/5000], Loss: 0.00072966
    Epoch [3850/5000], Loss: 0.00072252
    Epoch [3900/5000], Loss: 0.00071545
    Epoch [3950/5000], Loss: 0.00070842
    Epoch [4000/5000], Loss: 0.00070146
    Epoch [4050/5000], Loss: 0.00069455
    Epoch [4100/5000], Loss: 0.00068770
    Epoch [4150/5000], Loss: 0.00068091
    Epoch [4200/5000], Loss: 0.00067417
    Epoch [4250/5000], Loss: 0.00066749
    Epoch [4300/5000], Loss: 0.00066088
    Epoch [4350/5000], Loss: 0.00065432
    Epoch [4400/5000], Loss: 0.00064782
    Epoch [4450/5000], Loss: 0.00064137
    Epoch [4500/5000], Loss: 0.00063499
    Epoch [4550/5000], Loss: 0.00062867
    Epoch [4600/5000], Loss: 0.00062241
    Epoch [4650/5000], Loss: 0.00061621
    Epoch [4700/5000], Loss: 0.00061006
    Epoch [4750/5000], Loss: 0.00060398
    Epoch [4800/5000], Loss: 0.00059796
    Epoch [4850/5000], Loss: 0.00059200
    Epoch [4900/5000], Loss: 0.00058610
    Epoch [4950/5000], Loss: 0.00058026
    Epoch [5000/5000], Loss: 0.00057448
    Total Time Elapsed (s): 4.441021919250488

    Model: [50, 12, 10]
    Epoch [1/5000], Loss: 0.16300529
    Epoch [50/5000], Loss: 0.10887753
    Epoch [100/5000], Loss: 0.10883829
    Epoch [150/5000], Loss: 0.10879560
    Epoch [200/5000], Loss: 0.10874850
    Epoch [250/5000], Loss: 0.10869586
    Epoch [300/5000], Loss: 0.10863618
    Epoch [350/5000], Loss: 0.10856763
    Epoch [400/5000], Loss: 0.10848791
    Epoch [450/5000], Loss: 0.10839393
    Epoch [500/5000], Loss: 0.10828161
    Epoch [550/5000], Loss: 0.10814541
    Epoch [600/5000], Loss: 0.10797762
    Epoch [650/5000], Loss: 0.10776732
    Epoch [700/5000], Loss: 0.10749846
    Epoch [750/5000], Loss: 0.10714704
    Epoch [800/5000], Loss: 0.10667550
    Epoch [850/5000], Loss: 0.10602285
    Epoch [900/5000], Loss: 0.10508500
    Epoch [950/5000], Loss: 0.10367378
    Epoch [1000/5000], Loss: 0.10142446
    Epoch [1050/5000], Loss: 0.09757282
    Epoch [1100/5000], Loss: 0.09039161
    Epoch [1150/5000], Loss: 0.07595895
    Epoch [1200/5000], Loss: 0.04876474
    Epoch [1250/5000], Loss: 0.02047332
    Epoch [1300/5000], Loss: 0.01075574
    Epoch [1350/5000], Loss: 0.00730362
    Epoch [1400/5000], Loss: 0.00536273
    Epoch [1450/5000], Loss: 0.00412358
    Epoch [1500/5000], Loss: 0.00328189
    Epoch [1550/5000], Loss: 0.00268961
    Epoch [1600/5000], Loss: 0.00226343
    Epoch [1650/5000], Loss: 0.00195198
    Epoch [1700/5000], Loss: 0.00172164
    Epoch [1750/5000], Loss: 0.00154956
    Epoch [1800/5000], Loss: 0.00141974
    Epoch [1850/5000], Loss: 0.00132080
    Epoch [1900/5000], Loss: 0.00124453
    Epoch [1950/5000], Loss: 0.00118494
    Epoch [2000/5000], Loss: 0.00113766
    Epoch [2050/5000], Loss: 0.00109948
    Epoch [2100/5000], Loss: 0.00106802
    Epoch [2150/5000], Loss: 0.00104154
    Epoch [2200/5000], Loss: 0.00101875
    Epoch [2250/5000], Loss: 0.00099873
    Epoch [2300/5000], Loss: 0.00098078
    Epoch [2350/5000], Loss: 0.00096440
    Epoch [2400/5000], Loss: 0.00094923
    Epoch [2450/5000], Loss: 0.00093502
    Epoch [2500/5000], Loss: 0.00092158
    Epoch [2550/5000], Loss: 0.00090879
    Epoch [2600/5000], Loss: 0.00089654
    Epoch [2650/5000], Loss: 0.00088477
    Epoch [2700/5000], Loss: 0.00087343
    Epoch [2750/5000], Loss: 0.00086247
    Epoch [2800/5000], Loss: 0.00085188
    Epoch [2850/5000], Loss: 0.00084161
    Epoch [2900/5000], Loss: 0.00083165
    Epoch [2950/5000], Loss: 0.00082198
    Epoch [3000/5000], Loss: 0.00081257
    Epoch [3050/5000], Loss: 0.00080342
    Epoch [3100/5000], Loss: 0.00079450
    Epoch [3150/5000], Loss: 0.00078580
    Epoch [3200/5000], Loss: 0.00077732
    Epoch [3250/5000], Loss: 0.00076904
    Epoch [3300/5000], Loss: 0.00076093
    Epoch [3350/5000], Loss: 0.00075301
    Epoch [3400/5000], Loss: 0.00074526
    Epoch [3450/5000], Loss: 0.00073767
    Epoch [3500/5000], Loss: 0.00073023
    Epoch [3550/5000], Loss: 0.00072293
    Epoch [3600/5000], Loss: 0.00071578
    Epoch [3650/5000], Loss: 0.00070875
    Epoch [3700/5000], Loss: 0.00070186
    Epoch [3750/5000], Loss: 0.00069509
    Epoch [3800/5000], Loss: 0.00068844
    Epoch [3850/5000], Loss: 0.00068191
    Epoch [3900/5000], Loss: 0.00067548
    Epoch [3950/5000], Loss: 0.00066917
    Epoch [4000/5000], Loss: 0.00066295
    Epoch [4050/5000], Loss: 0.00065684
    Epoch [4100/5000], Loss: 0.00065083
    Epoch [4150/5000], Loss: 0.00064491
    Epoch [4200/5000], Loss: 0.00063909
    Epoch [4250/5000], Loss: 0.00063336
    Epoch [4300/5000], Loss: 0.00062771
    Epoch [4350/5000], Loss: 0.00062216
    Epoch [4400/5000], Loss: 0.00061668
    Epoch [4450/5000], Loss: 0.00061129
    Epoch [4500/5000], Loss: 0.00060598
    Epoch [4550/5000], Loss: 0.00060075
    Epoch [4600/5000], Loss: 0.00059560
    Epoch [4650/5000], Loss: 0.00059053
    Epoch [4700/5000], Loss: 0.00058553
    Epoch [4750/5000], Loss: 0.00058061
    Epoch [4800/5000], Loss: 0.00057575
    Epoch [4850/5000], Loss: 0.00057097
    Epoch [4900/5000], Loss: 0.00056626
    Epoch [4950/5000], Loss: 0.00056162
    Epoch [5000/5000], Loss: 0.00055705
    Total Time Elapsed (s): 5.121997117996216

    Model: [50, 30]
    Epoch [1/5000], Loss: 0.62863886
    Epoch [50/5000], Loss: 0.10792189
    Epoch [100/5000], Loss: 0.10681345
    Epoch [150/5000], Loss: 0.10490339
    Epoch [200/5000], Loss: 0.10100080
    Epoch [250/5000], Loss: 0.09161557
    Epoch [300/5000], Loss: 0.06696136
    Epoch [350/5000], Loss: 0.02526072
    Epoch [400/5000], Loss: 0.01159334
    Epoch [450/5000], Loss: 0.00950693
    Epoch [500/5000], Loss: 0.00805856
    Epoch [550/5000], Loss: 0.00690484
    Epoch [600/5000], Loss: 0.00595400
    Epoch [650/5000], Loss: 0.00515692
    Epoch [700/5000], Loss: 0.00448502
    Epoch [750/5000], Loss: 0.00391912
    Epoch [800/5000], Loss: 0.00344442
    Epoch [850/5000], Loss: 0.00304840
    Epoch [900/5000], Loss: 0.00271996
    Epoch [950/5000], Loss: 0.00244908
    Epoch [1000/5000], Loss: 0.00222680
    Epoch [1050/5000], Loss: 0.00204513
    Epoch [1100/5000], Loss: 0.00189711
    Epoch [1150/5000], Loss: 0.00177674
    Epoch [1200/5000], Loss: 0.00167889
    Epoch [1250/5000], Loss: 0.00159928
    Epoch [1300/5000], Loss: 0.00153432
    Epoch [1350/5000], Loss: 0.00148107
    Epoch [1400/5000], Loss: 0.00143713
    Epoch [1450/5000], Loss: 0.00140053
    Epoch [1500/5000], Loss: 0.00136973
    Epoch [1550/5000], Loss: 0.00134346
    Epoch [1600/5000], Loss: 0.00132074
    Epoch [1650/5000], Loss: 0.00130077
    Epoch [1700/5000], Loss: 0.00128295
    Epoch [1750/5000], Loss: 0.00126680
    Epoch [1800/5000], Loss: 0.00125194
    Epoch [1850/5000], Loss: 0.00123810
    Epoch [1900/5000], Loss: 0.00122507
    Epoch [1950/5000], Loss: 0.00121268
    Epoch [2000/5000], Loss: 0.00120081
    Epoch [2050/5000], Loss: 0.00118938
    Epoch [2100/5000], Loss: 0.00117831
    Epoch [2150/5000], Loss: 0.00116757
    Epoch [2200/5000], Loss: 0.00115711
    Epoch [2250/5000], Loss: 0.00114691
    Epoch [2300/5000], Loss: 0.00113694
    Epoch [2350/5000], Loss: 0.00112719
    Epoch [2400/5000], Loss: 0.00111765
    Epoch [2450/5000], Loss: 0.00110829
    Epoch [2500/5000], Loss: 0.00109911
    Epoch [2550/5000], Loss: 0.00109010
    Epoch [2600/5000], Loss: 0.00108124
    Epoch [2650/5000], Loss: 0.00107253
    Epoch [2700/5000], Loss: 0.00106395
    Epoch [2750/5000], Loss: 0.00105550
    Epoch [2800/5000], Loss: 0.00104716
    Epoch [2850/5000], Loss: 0.00103893
    Epoch [2900/5000], Loss: 0.00103080
    Epoch [2950/5000], Loss: 0.00102276
    Epoch [3000/5000], Loss: 0.00101481
    Epoch [3050/5000], Loss: 0.00100693
    Epoch [3100/5000], Loss: 0.00099913
    Epoch [3150/5000], Loss: 0.00099139
    Epoch [3200/5000], Loss: 0.00098372
    Epoch [3250/5000], Loss: 0.00097610
    Epoch [3300/5000], Loss: 0.00096852
    Epoch [3350/5000], Loss: 0.00096100
    Epoch [3400/5000], Loss: 0.00095352
    Epoch [3450/5000], Loss: 0.00094608
    Epoch [3500/5000], Loss: 0.00093868
    Epoch [3550/5000], Loss: 0.00093131
    Epoch [3600/5000], Loss: 0.00092397
    Epoch [3650/5000], Loss: 0.00091665
    Epoch [3700/5000], Loss: 0.00090936
    Epoch [3750/5000], Loss: 0.00090210
    Epoch [3800/5000], Loss: 0.00089485
    Epoch [3850/5000], Loss: 0.00088763
    Epoch [3900/5000], Loss: 0.00088043
    Epoch [3950/5000], Loss: 0.00087324
    Epoch [4000/5000], Loss: 0.00086607
    Epoch [4050/5000], Loss: 0.00085891
    Epoch [4100/5000], Loss: 0.00085177
    Epoch [4150/5000], Loss: 0.00084465
    Epoch [4200/5000], Loss: 0.00083754
    Epoch [4250/5000], Loss: 0.00083044
    Epoch [4300/5000], Loss: 0.00082335
    Epoch [4350/5000], Loss: 0.00081628
    Epoch [4400/5000], Loss: 0.00080922
    Epoch [4450/5000], Loss: 0.00080217
    Epoch [4500/5000], Loss: 0.00079514
    Epoch [4550/5000], Loss: 0.00078812
    Epoch [4600/5000], Loss: 0.00078111
    Epoch [4650/5000], Loss: 0.00077412
    Epoch [4700/5000], Loss: 0.00076715
    Epoch [4750/5000], Loss: 0.00076019
    Epoch [4800/5000], Loss: 0.00075324
    Epoch [4850/5000], Loss: 0.00074632
    Epoch [4900/5000], Loss: 0.00073941
    Epoch [4950/5000], Loss: 0.00073252
    Epoch [5000/5000], Loss: 0.00072565
    Total Time Elapsed (s): 4.549135208129883

    Model: [50, 50]
    Epoch [1/5000], Loss: 0.11309291
    Epoch [50/5000], Loss: 0.10730787
    Epoch [100/5000], Loss: 0.10465167
    Epoch [150/5000], Loss: 0.09938221
    Epoch [200/5000], Loss: 0.08650237
    Epoch [250/5000], Loss: 0.05471025
    Epoch [300/5000], Loss: 0.01761661
    Epoch [350/5000], Loss: 0.00896102
    Epoch [400/5000], Loss: 0.00628320
    Epoch [450/5000], Loss: 0.00464596
    Epoch [500/5000], Loss: 0.00354725
    Epoch [550/5000], Loss: 0.00278774
    Epoch [600/5000], Loss: 0.00225800
    Epoch [650/5000], Loss: 0.00188790
    Epoch [700/5000], Loss: 0.00162941
    Epoch [750/5000], Loss: 0.00144886
    Epoch [800/5000], Loss: 0.00132254
    Epoch [850/5000], Loss: 0.00123375
    Epoch [900/5000], Loss: 0.00117085
    Epoch [950/5000], Loss: 0.00112572
    Epoch [1000/5000], Loss: 0.00109274
    Epoch [1050/5000], Loss: 0.00106804
    Epoch [1100/5000], Loss: 0.00104898
    Epoch [1150/5000], Loss: 0.00103372
    Epoch [1200/5000], Loss: 0.00102105
    Epoch [1250/5000], Loss: 0.00101012
    Epoch [1300/5000], Loss: 0.00100037
    Epoch [1350/5000], Loss: 0.00099141
    Epoch [1400/5000], Loss: 0.00098299
    Epoch [1450/5000], Loss: 0.00097493
    Epoch [1500/5000], Loss: 0.00096713
    Epoch [1550/5000], Loss: 0.00095951
    Epoch [1600/5000], Loss: 0.00095202
    Epoch [1650/5000], Loss: 0.00094463
    Epoch [1700/5000], Loss: 0.00093732
    Epoch [1750/5000], Loss: 0.00093008
    Epoch [1800/5000], Loss: 0.00092290
    Epoch [1850/5000], Loss: 0.00091577
    Epoch [1900/5000], Loss: 0.00090870
    Epoch [1950/5000], Loss: 0.00090168
    Epoch [2000/5000], Loss: 0.00089472
    Epoch [2050/5000], Loss: 0.00088780
    Epoch [2100/5000], Loss: 0.00088094
    Epoch [2150/5000], Loss: 0.00087413
    Epoch [2200/5000], Loss: 0.00086737
    Epoch [2250/5000], Loss: 0.00086066
    Epoch [2300/5000], Loss: 0.00085401
    Epoch [2350/5000], Loss: 0.00084740
    Epoch [2400/5000], Loss: 0.00084085
    Epoch [2450/5000], Loss: 0.00083434
    Epoch [2500/5000], Loss: 0.00082789
    Epoch [2550/5000], Loss: 0.00082149
    Epoch [2600/5000], Loss: 0.00081514
    Epoch [2650/5000], Loss: 0.00080883
    Epoch [2700/5000], Loss: 0.00080257
    Epoch [2750/5000], Loss: 0.00079637
    Epoch [2800/5000], Loss: 0.00079020
    Epoch [2850/5000], Loss: 0.00078408
    Epoch [2900/5000], Loss: 0.00077801
    Epoch [2950/5000], Loss: 0.00077199
    Epoch [3000/5000], Loss: 0.00076600
    Epoch [3050/5000], Loss: 0.00076007
    Epoch [3100/5000], Loss: 0.00075417
    Epoch [3150/5000], Loss: 0.00074832
    Epoch [3200/5000], Loss: 0.00074250
    Epoch [3250/5000], Loss: 0.00073673
    Epoch [3300/5000], Loss: 0.00073100
    Epoch [3350/5000], Loss: 0.00072530
    Epoch [3400/5000], Loss: 0.00071965
    Epoch [3450/5000], Loss: 0.00071403
    Epoch [3500/5000], Loss: 0.00070845
    Epoch [3550/5000], Loss: 0.00070291
    Epoch [3600/5000], Loss: 0.00069741
    Epoch [3650/5000], Loss: 0.00069194
    Epoch [3700/5000], Loss: 0.00068650
    Epoch [3750/5000], Loss: 0.00068110
    Epoch [3800/5000], Loss: 0.00067574
    Epoch [3850/5000], Loss: 0.00067041
    Epoch [3900/5000], Loss: 0.00066511
    Epoch [3950/5000], Loss: 0.00065984
    Epoch [4000/5000], Loss: 0.00065461
    Epoch [4050/5000], Loss: 0.00064941
    Epoch [4100/5000], Loss: 0.00064424
    Epoch [4150/5000], Loss: 0.00063911
    Epoch [4200/5000], Loss: 0.00063400
    Epoch [4250/5000], Loss: 0.00062893
    Epoch [4300/5000], Loss: 0.00062388
    Epoch [4350/5000], Loss: 0.00061887
    Epoch [4400/5000], Loss: 0.00061388
    Epoch [4450/5000], Loss: 0.00060893
    Epoch [4500/5000], Loss: 0.00060401
    Epoch [4550/5000], Loss: 0.00059911
    Epoch [4600/5000], Loss: 0.00059424
    Epoch [4650/5000], Loss: 0.00058941
    Epoch [4700/5000], Loss: 0.00058460
    Epoch [4750/5000], Loss: 0.00057981
    Epoch [4800/5000], Loss: 0.00057506
    Epoch [4850/5000], Loss: 0.00057034
    Epoch [4900/5000], Loss: 0.00056564
    Epoch [4950/5000], Loss: 0.00056097
    Epoch [5000/5000], Loss: 0.00055633
    Total Time Elapsed (s): 4.464671850204468

    Model: [50, 50, 21]
    Epoch [1/5000], Loss: 0.44244829
    Epoch [50/5000], Loss: 0.10902856
    Epoch [100/5000], Loss: 0.10901625
    Epoch [150/5000], Loss: 0.10900392
    Epoch [200/5000], Loss: 0.10899136
    Epoch [250/5000], Loss: 0.10897847
    Epoch [300/5000], Loss: 0.10896508
    Epoch [350/5000], Loss: 0.10895099
    Epoch [400/5000], Loss: 0.10893600
    Epoch [450/5000], Loss: 0.10891990
    Epoch [500/5000], Loss: 0.10890245
    Epoch [550/5000], Loss: 0.10888338
    Epoch [600/5000], Loss: 0.10886230
    Epoch [650/5000], Loss: 0.10883886
    Epoch [700/5000], Loss: 0.10881259
    Epoch [750/5000], Loss: 0.10878289
    Epoch [800/5000], Loss: 0.10874906
    Epoch [850/5000], Loss: 0.10871022
    Epoch [900/5000], Loss: 0.10866527
    Epoch [950/5000], Loss: 0.10861281
    Epoch [1000/5000], Loss: 0.10855100
    Epoch [1050/5000], Loss: 0.10847746
    Epoch [1100/5000], Loss: 0.10838903
    Epoch [1150/5000], Loss: 0.10828140
    Epoch [1200/5000], Loss: 0.10814855
    Epoch [1250/5000], Loss: 0.10798204
    Epoch [1300/5000], Loss: 0.10776953
    Epoch [1350/5000], Loss: 0.10749252
    Epoch [1400/5000], Loss: 0.10712235
    Epoch [1450/5000], Loss: 0.10661265
    Epoch [1500/5000], Loss: 0.10588472
    Epoch [1550/5000], Loss: 0.10479687
    Epoch [1600/5000], Loss: 0.10307554
    Epoch [1650/5000], Loss: 0.10014495
    Epoch [1700/5000], Loss: 0.09466598
    Epoch [1750/5000], Loss: 0.08323690
    Epoch [1800/5000], Loss: 0.05812399
    Epoch [1850/5000], Loss: 0.02197139
    Epoch [1900/5000], Loss: 0.01048546
    Epoch [1950/5000], Loss: 0.00847082
    Epoch [2000/5000], Loss: 0.00710294
    Epoch [2050/5000], Loss: 0.00606098
    Epoch [2100/5000], Loss: 0.00523985
    Epoch [2150/5000], Loss: 0.00457759
    Epoch [2200/5000], Loss: 0.00403556
    Epoch [2250/5000], Loss: 0.00358793
    Epoch [2300/5000], Loss: 0.00321632
    Epoch [2350/5000], Loss: 0.00290701
    Epoch [2400/5000], Loss: 0.00264931
    Epoch [2450/5000], Loss: 0.00243465
    Epoch [2500/5000], Loss: 0.00225598
    Epoch [2550/5000], Loss: 0.00210742
    Epoch [2600/5000], Loss: 0.00198403
    Epoch [2650/5000], Loss: 0.00188165
    Epoch [2700/5000], Loss: 0.00179673
    Epoch [2750/5000], Loss: 0.00172630
    Epoch [2800/5000], Loss: 0.00166782
    Epoch [2850/5000], Loss: 0.00161919
    Epoch [2900/5000], Loss: 0.00157861
    Epoch [2950/5000], Loss: 0.00154462
    Epoch [3000/5000], Loss: 0.00151597
    Epoch [3050/5000], Loss: 0.00149165
    Epoch [3100/5000], Loss: 0.00147082
    Epoch [3150/5000], Loss: 0.00145280
    Epoch [3200/5000], Loss: 0.00143700
    Epoch [3250/5000], Loss: 0.00142299
    Epoch [3300/5000], Loss: 0.00141038
    Epoch [3350/5000], Loss: 0.00139887
    Epoch [3400/5000], Loss: 0.00138822
    Epoch [3450/5000], Loss: 0.00137824
    Epoch [3500/5000], Loss: 0.00136876
    Epoch [3550/5000], Loss: 0.00135967
    Epoch [3600/5000], Loss: 0.00135086
    Epoch [3650/5000], Loss: 0.00134226
    Epoch [3700/5000], Loss: 0.00133381
    Epoch [3750/5000], Loss: 0.00132545
    Epoch [3800/5000], Loss: 0.00131717
    Epoch [3850/5000], Loss: 0.00130891
    Epoch [3900/5000], Loss: 0.00130068
    Epoch [3950/5000], Loss: 0.00129245
    Epoch [4000/5000], Loss: 0.00128421
    Epoch [4050/5000], Loss: 0.00127595
    Epoch [4100/5000], Loss: 0.00126768
    Epoch [4150/5000], Loss: 0.00125938
    Epoch [4200/5000], Loss: 0.00125107
    Epoch [4250/5000], Loss: 0.00124273
    Epoch [4300/5000], Loss: 0.00123438
    Epoch [4350/5000], Loss: 0.00122600
    Epoch [4400/5000], Loss: 0.00121761
    Epoch [4450/5000], Loss: 0.00120921
    Epoch [4500/5000], Loss: 0.00120079
    Epoch [4550/5000], Loss: 0.00119236
    Epoch [4600/5000], Loss: 0.00118393
    Epoch [4650/5000], Loss: 0.00117549
    Epoch [4700/5000], Loss: 0.00116705
    Epoch [4750/5000], Loss: 0.00115860
    Epoch [4800/5000], Loss: 0.00115015
    Epoch [4850/5000], Loss: 0.00114170
    Epoch [4900/5000], Loss: 0.00113325
    Epoch [4950/5000], Loss: 0.00112480
    Epoch [5000/5000], Loss: 0.00111636
    Total Time Elapsed (s): 5.457895278930664

    Model: [50, 100]
    Epoch [1/5000], Loss: 0.80929923
    Epoch [50/5000], Loss: 0.10855693
    Epoch [100/5000], Loss: 0.10765429
    Epoch [150/5000], Loss: 0.10623858
    Epoch [200/5000], Loss: 0.10367380
    Epoch [250/5000], Loss: 0.09845596
    Epoch [300/5000], Loss: 0.08662076
    Epoch [350/5000], Loss: 0.05960444
    Epoch [400/5000], Loss: 0.02272379
    Epoch [450/5000], Loss: 0.00965402
    Epoch [500/5000], Loss: 0.00641815
    Epoch [550/5000], Loss: 0.00457655
    Epoch [600/5000], Loss: 0.00338604
    Epoch [650/5000], Loss: 0.00259253
    Epoch [700/5000], Loss: 0.00205945
    Epoch [750/5000], Loss: 0.00170108
    Epoch [800/5000], Loss: 0.00146040
    Epoch [850/5000], Loss: 0.00129885
    Epoch [900/5000], Loss: 0.00119032
    Epoch [950/5000], Loss: 0.00111718
    Epoch [1000/5000], Loss: 0.00106760
    Epoch [1050/5000], Loss: 0.00103364
    Epoch [1100/5000], Loss: 0.00101003
    Epoch [1150/5000], Loss: 0.00099327
    Epoch [1200/5000], Loss: 0.00098101
    Epoch [1250/5000], Loss: 0.00097174
    Epoch [1300/5000], Loss: 0.00096441
    Epoch [1350/5000], Loss: 0.00095837
    Epoch [1400/5000], Loss: 0.00095317
    Epoch [1450/5000], Loss: 0.00094852
    Epoch [1500/5000], Loss: 0.00094422
    Epoch [1550/5000], Loss: 0.00094015
    Epoch [1600/5000], Loss: 0.00093623
    Epoch [1650/5000], Loss: 0.00093240
    Epoch [1700/5000], Loss: 0.00092863
    Epoch [1750/5000], Loss: 0.00092490
    Epoch [1800/5000], Loss: 0.00092118
    Epoch [1850/5000], Loss: 0.00091747
    Epoch [1900/5000], Loss: 0.00091376
    Epoch [1950/5000], Loss: 0.00091006
    Epoch [2000/5000], Loss: 0.00090635
    Epoch [2050/5000], Loss: 0.00090263
    Epoch [2100/5000], Loss: 0.00089890
    Epoch [2150/5000], Loss: 0.00089517
    Epoch [2200/5000], Loss: 0.00089143
    Epoch [2250/5000], Loss: 0.00088769
    Epoch [2300/5000], Loss: 0.00088393
    Epoch [2350/5000], Loss: 0.00088017
    Epoch [2400/5000], Loss: 0.00087640
    Epoch [2450/5000], Loss: 0.00087262
    Epoch [2500/5000], Loss: 0.00086883
    Epoch [2550/5000], Loss: 0.00086503
    Epoch [2600/5000], Loss: 0.00086123
    Epoch [2650/5000], Loss: 0.00085741
    Epoch [2700/5000], Loss: 0.00085359
    Epoch [2750/5000], Loss: 0.00084977
    Epoch [2800/5000], Loss: 0.00084593
    Epoch [2850/5000], Loss: 0.00084209
    Epoch [2900/5000], Loss: 0.00083823
    Epoch [2950/5000], Loss: 0.00083437
    Epoch [3000/5000], Loss: 0.00083051
    Epoch [3050/5000], Loss: 0.00082663
    Epoch [3100/5000], Loss: 0.00082275
    Epoch [3150/5000], Loss: 0.00081886
    Epoch [3200/5000], Loss: 0.00081496
    Epoch [3250/5000], Loss: 0.00081106
    Epoch [3300/5000], Loss: 0.00080715
    Epoch [3350/5000], Loss: 0.00080323
    Epoch [3400/5000], Loss: 0.00079930
    Epoch [3450/5000], Loss: 0.00079537
    Epoch [3500/5000], Loss: 0.00079143
    Epoch [3550/5000], Loss: 0.00078749
    Epoch [3600/5000], Loss: 0.00078353
    Epoch [3650/5000], Loss: 0.00077957
    Epoch [3700/5000], Loss: 0.00077561
    Epoch [3750/5000], Loss: 0.00077164
    Epoch [3800/5000], Loss: 0.00076766
    Epoch [3850/5000], Loss: 0.00076367
    Epoch [3900/5000], Loss: 0.00075968
    Epoch [3950/5000], Loss: 0.00075569
    Epoch [4000/5000], Loss: 0.00075168
    Epoch [4050/5000], Loss: 0.00074768
    Epoch [4100/5000], Loss: 0.00074367
    Epoch [4150/5000], Loss: 0.00073965
    Epoch [4200/5000], Loss: 0.00073562
    Epoch [4250/5000], Loss: 0.00073160
    Epoch [4300/5000], Loss: 0.00072756
    Epoch [4350/5000], Loss: 0.00072352
    Epoch [4400/5000], Loss: 0.00071948
    Epoch [4450/5000], Loss: 0.00071544
    Epoch [4500/5000], Loss: 0.00071139
    Epoch [4550/5000], Loss: 0.00070733
    Epoch [4600/5000], Loss: 0.00070328
    Epoch [4650/5000], Loss: 0.00069922
    Epoch [4700/5000], Loss: 0.00069515
    Epoch [4750/5000], Loss: 0.00069108
    Epoch [4800/5000], Loss: 0.00068701
    Epoch [4850/5000], Loss: 0.00068294
    Epoch [4900/5000], Loss: 0.00067887
    Epoch [4950/5000], Loss: 0.00067479
    Epoch [5000/5000], Loss: 0.00067071
    Total Time Elapsed (s): 4.867865562438965

    Model: [80, 20]
    Epoch [1/5000], Loss: 0.37225315
    Epoch [50/5000], Loss: 0.10600577
    Epoch [100/5000], Loss: 0.09975398
    Epoch [150/5000], Loss: 0.08354797
    Epoch [200/5000], Loss: 0.04151233
    Epoch [250/5000], Loss: 0.01087520
    Epoch [300/5000], Loss: 0.00692577
    Epoch [350/5000], Loss: 0.00522529
    Epoch [400/5000], Loss: 0.00412688
    Epoch [450/5000], Loss: 0.00336540
    Epoch [500/5000], Loss: 0.00281939
    Epoch [550/5000], Loss: 0.00242110
    Epoch [600/5000], Loss: 0.00212771
    Epoch [650/5000], Loss: 0.00191015
    Epoch [700/5000], Loss: 0.00174786
    Epoch [750/5000], Loss: 0.00162602
    Epoch [800/5000], Loss: 0.00153381
    Epoch [850/5000], Loss: 0.00146327
    Epoch [900/5000], Loss: 0.00140856
    Epoch [950/5000], Loss: 0.00136537
    Epoch [1000/5000], Loss: 0.00133056
    Epoch [1050/5000], Loss: 0.00130180
    Epoch [1100/5000], Loss: 0.00127742
    Epoch [1150/5000], Loss: 0.00125618
    Epoch [1200/5000], Loss: 0.00123721
    Epoch [1250/5000], Loss: 0.00121988
    Epoch [1300/5000], Loss: 0.00120376
    Epoch [1350/5000], Loss: 0.00118854
    Epoch [1400/5000], Loss: 0.00117401
    Epoch [1450/5000], Loss: 0.00116004
    Epoch [1500/5000], Loss: 0.00114652
    Epoch [1550/5000], Loss: 0.00113340
    Epoch [1600/5000], Loss: 0.00112062
    Epoch [1650/5000], Loss: 0.00110817
    Epoch [1700/5000], Loss: 0.00109600
    Epoch [1750/5000], Loss: 0.00108409
    Epoch [1800/5000], Loss: 0.00107243
    Epoch [1850/5000], Loss: 0.00106101
    Epoch [1900/5000], Loss: 0.00104979
    Epoch [1950/5000], Loss: 0.00103877
    Epoch [2000/5000], Loss: 0.00102793
    Epoch [2050/5000], Loss: 0.00101726
    Epoch [2100/5000], Loss: 0.00100675
    Epoch [2150/5000], Loss: 0.00099639
    Epoch [2200/5000], Loss: 0.00098617
    Epoch [2250/5000], Loss: 0.00097608
    Epoch [2300/5000], Loss: 0.00096611
    Epoch [2350/5000], Loss: 0.00095626
    Epoch [2400/5000], Loss: 0.00094653
    Epoch [2450/5000], Loss: 0.00093690
    Epoch [2500/5000], Loss: 0.00092738
    Epoch [2550/5000], Loss: 0.00091796
    Epoch [2600/5000], Loss: 0.00090864
    Epoch [2650/5000], Loss: 0.00089941
    Epoch [2700/5000], Loss: 0.00089027
    Epoch [2750/5000], Loss: 0.00088122
    Epoch [2800/5000], Loss: 0.00087226
    Epoch [2850/5000], Loss: 0.00086338
    Epoch [2900/5000], Loss: 0.00085459
    Epoch [2950/5000], Loss: 0.00084588
    Epoch [3000/5000], Loss: 0.00083725
    Epoch [3050/5000], Loss: 0.00082870
    Epoch [3100/5000], Loss: 0.00082023
    Epoch [3150/5000], Loss: 0.00081183
    Epoch [3200/5000], Loss: 0.00080351
    Epoch [3250/5000], Loss: 0.00079527
    Epoch [3300/5000], Loss: 0.00078711
    Epoch [3350/5000], Loss: 0.00077901
    Epoch [3400/5000], Loss: 0.00077100
    Epoch [3450/5000], Loss: 0.00076305
    Epoch [3500/5000], Loss: 0.00075518
    Epoch [3550/5000], Loss: 0.00074738
    Epoch [3600/5000], Loss: 0.00073965
    Epoch [3650/5000], Loss: 0.00073200
    Epoch [3700/5000], Loss: 0.00072442
    Epoch [3750/5000], Loss: 0.00071690
    Epoch [3800/5000], Loss: 0.00070946
    Epoch [3850/5000], Loss: 0.00070209
    Epoch [3900/5000], Loss: 0.00069479
    Epoch [3950/5000], Loss: 0.00068756
    Epoch [4000/5000], Loss: 0.00068041
    Epoch [4050/5000], Loss: 0.00067332
    Epoch [4100/5000], Loss: 0.00066630
    Epoch [4150/5000], Loss: 0.00065936
    Epoch [4200/5000], Loss: 0.00065248
    Epoch [4250/5000], Loss: 0.00064568
    Epoch [4300/5000], Loss: 0.00063894
    Epoch [4350/5000], Loss: 0.00063228
    Epoch [4400/5000], Loss: 0.00062568
    Epoch [4450/5000], Loss: 0.00061916
    Epoch [4500/5000], Loss: 0.00061270
    Epoch [4550/5000], Loss: 0.00060632
    Epoch [4600/5000], Loss: 0.00060000
    Epoch [4650/5000], Loss: 0.00059376
    Epoch [4700/5000], Loss: 0.00058758
    Epoch [4750/5000], Loss: 0.00058148
    Epoch [4800/5000], Loss: 0.00057545
    Epoch [4850/5000], Loss: 0.00056948
    Epoch [4900/5000], Loss: 0.00056359
    Epoch [4950/5000], Loss: 0.00055776
    Epoch [5000/5000], Loss: 0.00055201
    Total Time Elapsed (s): 4.29575514793396

    Model: [100, 80]
    Epoch [1/5000], Loss: 0.21111298
    Epoch [50/5000], Loss: 0.10763869
    Epoch [100/5000], Loss: 0.10485002
    Epoch [150/5000], Loss: 0.09834027
    Epoch [200/5000], Loss: 0.07932000
    Epoch [250/5000], Loss: 0.03391503
    Epoch [300/5000], Loss: 0.00940233
    Epoch [350/5000], Loss: 0.00507464
    Epoch [400/5000], Loss: 0.00316898
    Epoch [450/5000], Loss: 0.00216550
    Epoch [500/5000], Loss: 0.00161255
    Epoch [550/5000], Loss: 0.00130186
    Epoch [600/5000], Loss: 0.00112499
    Epoch [650/5000], Loss: 0.00102290
    Epoch [700/5000], Loss: 0.00096288
    Epoch [750/5000], Loss: 0.00092659
    Epoch [800/5000], Loss: 0.00090373
    Epoch [850/5000], Loss: 0.00088847
    Epoch [900/5000], Loss: 0.00087751
    Epoch [950/5000], Loss: 0.00086897
    Epoch [1000/5000], Loss: 0.00086181
    Epoch [1050/5000], Loss: 0.00085542
    Epoch [1100/5000], Loss: 0.00084946
    Epoch [1150/5000], Loss: 0.00084374
    Epoch [1200/5000], Loss: 0.00083816
    Epoch [1250/5000], Loss: 0.00083265
    Epoch [1300/5000], Loss: 0.00082718
    Epoch [1350/5000], Loss: 0.00082173
    Epoch [1400/5000], Loss: 0.00081630
    Epoch [1450/5000], Loss: 0.00081087
    Epoch [1500/5000], Loss: 0.00080545
    Epoch [1550/5000], Loss: 0.00080003
    Epoch [1600/5000], Loss: 0.00079461
    Epoch [1650/5000], Loss: 0.00078920
    Epoch [1700/5000], Loss: 0.00078380
    Epoch [1750/5000], Loss: 0.00077839
    Epoch [1800/5000], Loss: 0.00077299
    Epoch [1850/5000], Loss: 0.00076759
    Epoch [1900/5000], Loss: 0.00076220
    Epoch [1950/5000], Loss: 0.00075681
    Epoch [2000/5000], Loss: 0.00075142
    Epoch [2050/5000], Loss: 0.00074604
    Epoch [2100/5000], Loss: 0.00074067
    Epoch [2150/5000], Loss: 0.00073530
    Epoch [2200/5000], Loss: 0.00072994
    Epoch [2250/5000], Loss: 0.00072458
    Epoch [2300/5000], Loss: 0.00071922
    Epoch [2350/5000], Loss: 0.00071387
    Epoch [2400/5000], Loss: 0.00070853
    Epoch [2450/5000], Loss: 0.00070319
    Epoch [2500/5000], Loss: 0.00069786
    Epoch [2550/5000], Loss: 0.00069253
    Epoch [2600/5000], Loss: 0.00068721
    Epoch [2650/5000], Loss: 0.00068190
    Epoch [2700/5000], Loss: 0.00067659
    Epoch [2750/5000], Loss: 0.00067129
    Epoch [2800/5000], Loss: 0.00066600
    Epoch [2850/5000], Loss: 0.00066072
    Epoch [2900/5000], Loss: 0.00065544
    Epoch [2950/5000], Loss: 0.00065017
    Epoch [3000/5000], Loss: 0.00064491
    Epoch [3050/5000], Loss: 0.00063966
    Epoch [3100/5000], Loss: 0.00063442
    Epoch [3150/5000], Loss: 0.00062918
    Epoch [3200/5000], Loss: 0.00062396
    Epoch [3250/5000], Loss: 0.00061874
    Epoch [3300/5000], Loss: 0.00061354
    Epoch [3350/5000], Loss: 0.00060835
    Epoch [3400/5000], Loss: 0.00060317
    Epoch [3450/5000], Loss: 0.00059800
    Epoch [3500/5000], Loss: 0.00059284
    Epoch [3550/5000], Loss: 0.00058770
    Epoch [3600/5000], Loss: 0.00058257
    Epoch [3650/5000], Loss: 0.00057746
    Epoch [3700/5000], Loss: 0.00057236
    Epoch [3750/5000], Loss: 0.00056727
    Epoch [3800/5000], Loss: 0.00056221
    Epoch [3850/5000], Loss: 0.00055716
    Epoch [3900/5000], Loss: 0.00055213
    Epoch [3950/5000], Loss: 0.00054711
    Epoch [4000/5000], Loss: 0.00054212
    Epoch [4050/5000], Loss: 0.00053714
    Epoch [4100/5000], Loss: 0.00053219
    Epoch [4150/5000], Loss: 0.00052726
    Epoch [4200/5000], Loss: 0.00052235
    Epoch [4250/5000], Loss: 0.00051746
    Epoch [4300/5000], Loss: 0.00051260
    Epoch [4350/5000], Loss: 0.00050776
    Epoch [4400/5000], Loss: 0.00050295
    Epoch [4450/5000], Loss: 0.00049817
    Epoch [4500/5000], Loss: 0.00049341
    Epoch [4550/5000], Loss: 0.00048868
    Epoch [4600/5000], Loss: 0.00048398
    Epoch [4650/5000], Loss: 0.00047932
    Epoch [4700/5000], Loss: 0.00047468
    Epoch [4750/5000], Loss: 0.00047008
    Epoch [4800/5000], Loss: 0.00046551
    Epoch [4850/5000], Loss: 0.00046097
    Epoch [4900/5000], Loss: 0.00045647
    Epoch [4950/5000], Loss: 0.00045200
    Epoch [5000/5000], Loss: 0.00044758
    Total Time Elapsed (s): 4.736923694610596

    Model: [100, 100]
    Epoch [1/5000], Loss: 0.23017994
    Epoch [50/5000], Loss: 0.10714433
    Epoch [100/5000], Loss: 0.10411204
    Epoch [150/5000], Loss: 0.09681196
    Epoch [200/5000], Loss: 0.07549313
    Epoch [250/5000], Loss: 0.02876798
    Epoch [300/5000], Loss: 0.00772562
    Epoch [350/5000], Loss: 0.00429361
    Epoch [400/5000], Loss: 0.00279419
    Epoch [450/5000], Loss: 0.00201149
    Epoch [500/5000], Loss: 0.00158244
    Epoch [550/5000], Loss: 0.00134205
    Epoch [600/5000], Loss: 0.00120531
    Epoch [650/5000], Loss: 0.00112626
    Epoch [700/5000], Loss: 0.00107953
    Epoch [750/5000], Loss: 0.00105096
    Epoch [800/5000], Loss: 0.00103260
    Epoch [850/5000], Loss: 0.00101998
    Epoch [900/5000], Loss: 0.00101058
    Epoch [950/5000], Loss: 0.00100298
    Epoch [1000/5000], Loss: 0.00099638
    Epoch [1050/5000], Loss: 0.00099032
    Epoch [1100/5000], Loss: 0.00098456
    Epoch [1150/5000], Loss: 0.00097896
    Epoch [1200/5000], Loss: 0.00097342
    Epoch [1250/5000], Loss: 0.00096791
    Epoch [1300/5000], Loss: 0.00096241
    Epoch [1350/5000], Loss: 0.00095690
    Epoch [1400/5000], Loss: 0.00095137
    Epoch [1450/5000], Loss: 0.00094583
    Epoch [1500/5000], Loss: 0.00094026
    Epoch [1550/5000], Loss: 0.00093467
    Epoch [1600/5000], Loss: 0.00092905
    Epoch [1650/5000], Loss: 0.00092342
    Epoch [1700/5000], Loss: 0.00091776
    Epoch [1750/5000], Loss: 0.00091207
    Epoch [1800/5000], Loss: 0.00090637
    Epoch [1850/5000], Loss: 0.00090065
    Epoch [1900/5000], Loss: 0.00089490
    Epoch [1950/5000], Loss: 0.00088913
    Epoch [2000/5000], Loss: 0.00088334
    Epoch [2050/5000], Loss: 0.00087753
    Epoch [2100/5000], Loss: 0.00087170
    Epoch [2150/5000], Loss: 0.00086585
    Epoch [2200/5000], Loss: 0.00085997
    Epoch [2250/5000], Loss: 0.00085408
    Epoch [2300/5000], Loss: 0.00084817
    Epoch [2350/5000], Loss: 0.00084225
    Epoch [2400/5000], Loss: 0.00083630
    Epoch [2450/5000], Loss: 0.00083034
    Epoch [2500/5000], Loss: 0.00082435
    Epoch [2550/5000], Loss: 0.00081836
    Epoch [2600/5000], Loss: 0.00081234
    Epoch [2650/5000], Loss: 0.00080631
    Epoch [2700/5000], Loss: 0.00080027
    Epoch [2750/5000], Loss: 0.00079421
    Epoch [2800/5000], Loss: 0.00078813
    Epoch [2850/5000], Loss: 0.00078205
    Epoch [2900/5000], Loss: 0.00077595
    Epoch [2950/5000], Loss: 0.00076984
    Epoch [3000/5000], Loss: 0.00076372
    Epoch [3050/5000], Loss: 0.00075759
    Epoch [3100/5000], Loss: 0.00075145
    Epoch [3150/5000], Loss: 0.00074531
    Epoch [3200/5000], Loss: 0.00073915
    Epoch [3250/5000], Loss: 0.00073299
    Epoch [3300/5000], Loss: 0.00072683
    Epoch [3350/5000], Loss: 0.00072066
    Epoch [3400/5000], Loss: 0.00071449
    Epoch [3450/5000], Loss: 0.00070832
    Epoch [3500/5000], Loss: 0.00070214
    Epoch [3550/5000], Loss: 0.00069597
    Epoch [3600/5000], Loss: 0.00068981
    Epoch [3650/5000], Loss: 0.00068364
    Epoch [3700/5000], Loss: 0.00067749
    Epoch [3750/5000], Loss: 0.00067133
    Epoch [3800/5000], Loss: 0.00066519
    Epoch [3850/5000], Loss: 0.00065906
    Epoch [3900/5000], Loss: 0.00065294
    Epoch [3950/5000], Loss: 0.00064684
    Epoch [4000/5000], Loss: 0.00064075
    Epoch [4050/5000], Loss: 0.00063467
    Epoch [4100/5000], Loss: 0.00062862
    Epoch [4150/5000], Loss: 0.00062259
    Epoch [4200/5000], Loss: 0.00061658
    Epoch [4250/5000], Loss: 0.00061059
    Epoch [4300/5000], Loss: 0.00060463
    Epoch [4350/5000], Loss: 0.00059870
    Epoch [4400/5000], Loss: 0.00059280
    Epoch [4450/5000], Loss: 0.00058694
    Epoch [4500/5000], Loss: 0.00058111
    Epoch [4550/5000], Loss: 0.00057531
    Epoch [4600/5000], Loss: 0.00056955
    Epoch [4650/5000], Loss: 0.00056384
    Epoch [4700/5000], Loss: 0.00055816
    Epoch [4750/5000], Loss: 0.00055254
    Epoch [4800/5000], Loss: 0.00054695
    Epoch [4850/5000], Loss: 0.00054142
    Epoch [4900/5000], Loss: 0.00053594
    Epoch [4950/5000], Loss: 0.00053051
    Epoch [5000/5000], Loss: 0.00052513
    Total Time Elapsed (s): 4.812922477722168

    Model: [100, 120]
    Epoch [1/5000], Loss: 0.22831614
    Epoch [50/5000], Loss: 0.10717131
    Epoch [100/5000], Loss: 0.10354848
    Epoch [150/5000], Loss: 0.09402908
    Epoch [200/5000], Loss: 0.06507313
    Epoch [250/5000], Loss: 0.01813616
    Epoch [300/5000], Loss: 0.00643272
    Epoch [350/5000], Loss: 0.00376724
    Epoch [400/5000], Loss: 0.00252877
    Epoch [450/5000], Loss: 0.00187473
    Epoch [500/5000], Loss: 0.00151146
    Epoch [550/5000], Loss: 0.00130380
    Epoch [600/5000], Loss: 0.00118227
    Epoch [650/5000], Loss: 0.00110928
    Epoch [700/5000], Loss: 0.00106390
    Epoch [750/5000], Loss: 0.00103433
    Epoch [800/5000], Loss: 0.00101385
    Epoch [850/5000], Loss: 0.00099859
    Epoch [900/5000], Loss: 0.00098634
    Epoch [950/5000], Loss: 0.00097584
    Epoch [1000/5000], Loss: 0.00096634
    Epoch [1050/5000], Loss: 0.00095745
    Epoch [1100/5000], Loss: 0.00094890
    Epoch [1150/5000], Loss: 0.00094058
    Epoch [1200/5000], Loss: 0.00093239
    Epoch [1250/5000], Loss: 0.00092431
    Epoch [1300/5000], Loss: 0.00091630
    Epoch [1350/5000], Loss: 0.00090834
    Epoch [1400/5000], Loss: 0.00090043
    Epoch [1450/5000], Loss: 0.00089257
    Epoch [1500/5000], Loss: 0.00088476
    Epoch [1550/5000], Loss: 0.00087698
    Epoch [1600/5000], Loss: 0.00086924
    Epoch [1650/5000], Loss: 0.00086154
    Epoch [1700/5000], Loss: 0.00085387
    Epoch [1750/5000], Loss: 0.00084625
    Epoch [1800/5000], Loss: 0.00083866
    Epoch [1850/5000], Loss: 0.00083110
    Epoch [1900/5000], Loss: 0.00082358
    Epoch [1950/5000], Loss: 0.00081609
    Epoch [2000/5000], Loss: 0.00080863
    Epoch [2050/5000], Loss: 0.00080121
    Epoch [2100/5000], Loss: 0.00079382
    Epoch [2150/5000], Loss: 0.00078646
    Epoch [2200/5000], Loss: 0.00077913
    Epoch [2250/5000], Loss: 0.00077183
    Epoch [2300/5000], Loss: 0.00076457
    Epoch [2350/5000], Loss: 0.00075733
    Epoch [2400/5000], Loss: 0.00075013
    Epoch [2450/5000], Loss: 0.00074295
    Epoch [2500/5000], Loss: 0.00073581
    Epoch [2550/5000], Loss: 0.00072869
    Epoch [2600/5000], Loss: 0.00072161
    Epoch [2650/5000], Loss: 0.00071455
    Epoch [2700/5000], Loss: 0.00070753
    Epoch [2750/5000], Loss: 0.00070053
    Epoch [2800/5000], Loss: 0.00069356
    Epoch [2850/5000], Loss: 0.00068663
    Epoch [2900/5000], Loss: 0.00067972
    Epoch [2950/5000], Loss: 0.00067285
    Epoch [3000/5000], Loss: 0.00066601
    Epoch [3050/5000], Loss: 0.00065920
    Epoch [3100/5000], Loss: 0.00065242
    Epoch [3150/5000], Loss: 0.00064568
    Epoch [3200/5000], Loss: 0.00063897
    Epoch [3250/5000], Loss: 0.00063229
    Epoch [3300/5000], Loss: 0.00062565
    Epoch [3350/5000], Loss: 0.00061905
    Epoch [3400/5000], Loss: 0.00061248
    Epoch [3450/5000], Loss: 0.00060595
    Epoch [3500/5000], Loss: 0.00059946
    Epoch [3550/5000], Loss: 0.00059300
    Epoch [3600/5000], Loss: 0.00058659
    Epoch [3650/5000], Loss: 0.00058023
    Epoch [3700/5000], Loss: 0.00057390
    Epoch [3750/5000], Loss: 0.00056762
    Epoch [3800/5000], Loss: 0.00056138
    Epoch [3850/5000], Loss: 0.00055519
    Epoch [3900/5000], Loss: 0.00054905
    Epoch [3950/5000], Loss: 0.00054295
    Epoch [4000/5000], Loss: 0.00053691
    Epoch [4050/5000], Loss: 0.00053092
    Epoch [4100/5000], Loss: 0.00052498
    Epoch [4150/5000], Loss: 0.00051909
    Epoch [4200/5000], Loss: 0.00051326
    Epoch [4250/5000], Loss: 0.00050749
    Epoch [4300/5000], Loss: 0.00050178
    Epoch [4350/5000], Loss: 0.00049612
    Epoch [4400/5000], Loss: 0.00049053
    Epoch [4450/5000], Loss: 0.00048500
    Epoch [4500/5000], Loss: 0.00047953
    Epoch [4550/5000], Loss: 0.00047413
    Epoch [4600/5000], Loss: 0.00046879
    Epoch [4650/5000], Loss: 0.00046352
    Epoch [4700/5000], Loss: 0.00045832
    Epoch [4750/5000], Loss: 0.00045318
    Epoch [4800/5000], Loss: 0.00044812
    Epoch [4850/5000], Loss: 0.00044313
    Epoch [4900/5000], Loss: 0.00043821
    Epoch [4950/5000], Loss: 0.00043336
    Epoch [5000/5000], Loss: 0.00042859
    Total Time Elapsed (s): 5.106760263442993

    Model: [100, 150]
    Epoch [1/5000], Loss: 0.52620155
    Epoch [50/5000], Loss: 0.10907277
    Epoch [100/5000], Loss: 0.10901745
    Epoch [150/5000], Loss: 0.10897497
    Epoch [200/5000], Loss: 0.10893525
    Epoch [250/5000], Loss: 0.10889068
    Epoch [300/5000], Loss: 0.10883275
    Epoch [350/5000], Loss: 0.10874801
    Epoch [400/5000], Loss: 0.10861004
    Epoch [450/5000], Loss: 0.10835803
    Epoch [500/5000], Loss: 0.10782155
    Epoch [550/5000], Loss: 0.10636045
    Epoch [600/5000], Loss: 0.10056719
    Epoch [650/5000], Loss: 0.07990784
    Epoch [700/5000], Loss: 0.02977942
    Epoch [750/5000], Loss: 0.01028732
    Epoch [800/5000], Loss: 0.00778946
    Epoch [850/5000], Loss: 0.00607155
    Epoch [900/5000], Loss: 0.00473792
    Epoch [950/5000], Loss: 0.00372188
    Epoch [1000/5000], Loss: 0.00296751
    Epoch [1050/5000], Loss: 0.00241901
    Epoch [1100/5000], Loss: 0.00202692
    Epoch [1150/5000], Loss: 0.00175015
    Epoch [1200/5000], Loss: 0.00155619
    Epoch [1250/5000], Loss: 0.00142038
    Epoch [1300/5000], Loss: 0.00132463
    Epoch [1350/5000], Loss: 0.00125606
    Epoch [1400/5000], Loss: 0.00120567
    Epoch [1450/5000], Loss: 0.00116732
    Epoch [1500/5000], Loss: 0.00113689
    Epoch [1550/5000], Loss: 0.00111167
    Epoch [1600/5000], Loss: 0.00108991
    Epoch [1650/5000], Loss: 0.00107049
    Epoch [1700/5000], Loss: 0.00105273
    Epoch [1750/5000], Loss: 0.00103620
    Epoch [1800/5000], Loss: 0.00102062
    Epoch [1850/5000], Loss: 0.00100582
    Epoch [1900/5000], Loss: 0.00099169
    Epoch [1950/5000], Loss: 0.00097815
    Epoch [2000/5000], Loss: 0.00096514
    Epoch [2050/5000], Loss: 0.00095261
    Epoch [2100/5000], Loss: 0.00094052
    Epoch [2150/5000], Loss: 0.00092884
    Epoch [2200/5000], Loss: 0.00091752
    Epoch [2250/5000], Loss: 0.00090655
    Epoch [2300/5000], Loss: 0.00089588
    Epoch [2350/5000], Loss: 0.00088551
    Epoch [2400/5000], Loss: 0.00087540
    Epoch [2450/5000], Loss: 0.00086553
    Epoch [2500/5000], Loss: 0.00085589
    Epoch [2550/5000], Loss: 0.00084645
    Epoch [2600/5000], Loss: 0.00083719
    Epoch [2650/5000], Loss: 0.00082811
    Epoch [2700/5000], Loss: 0.00081919
    Epoch [2750/5000], Loss: 0.00081040
    Epoch [2800/5000], Loss: 0.00080175
    Epoch [2850/5000], Loss: 0.00079323
    Epoch [2900/5000], Loss: 0.00078481
    Epoch [2950/5000], Loss: 0.00077650
    Epoch [3000/5000], Loss: 0.00076829
    Epoch [3050/5000], Loss: 0.00076017
    Epoch [3100/5000], Loss: 0.00075212
    Epoch [3150/5000], Loss: 0.00074416
    Epoch [3200/5000], Loss: 0.00073627
    Epoch [3250/5000], Loss: 0.00072845
    Epoch [3300/5000], Loss: 0.00072070
    Epoch [3350/5000], Loss: 0.00071301
    Epoch [3400/5000], Loss: 0.00070538
    Epoch [3450/5000], Loss: 0.00069780
    Epoch [3500/5000], Loss: 0.00069029
    Epoch [3550/5000], Loss: 0.00068283
    Epoch [3600/5000], Loss: 0.00067543
    Epoch [3650/5000], Loss: 0.00066808
    Epoch [3700/5000], Loss: 0.00066078
    Epoch [3750/5000], Loss: 0.00065354
    Epoch [3800/5000], Loss: 0.00064636
    Epoch [3850/5000], Loss: 0.00063923
    Epoch [3900/5000], Loss: 0.00063215
    Epoch [3950/5000], Loss: 0.00062514
    Epoch [4000/5000], Loss: 0.00061818
    Epoch [4050/5000], Loss: 0.00061128
    Epoch [4100/5000], Loss: 0.00060444
    Epoch [4150/5000], Loss: 0.00059766
    Epoch [4200/5000], Loss: 0.00059094
    Epoch [4250/5000], Loss: 0.00058429
    Epoch [4300/5000], Loss: 0.00057771
    Epoch [4350/5000], Loss: 0.00057120
    Epoch [4400/5000], Loss: 0.00056476
    Epoch [4450/5000], Loss: 0.00055839
    Epoch [4500/5000], Loss: 0.00055209
    Epoch [4550/5000], Loss: 0.00054587
    Epoch [4600/5000], Loss: 0.00053973
    Epoch [4650/5000], Loss: 0.00053367
    Epoch [4700/5000], Loss: 0.00052769
    Epoch [4750/5000], Loss: 0.00052179
    Epoch [4800/5000], Loss: 0.00051598
    Epoch [4850/5000], Loss: 0.00051025
    Epoch [4900/5000], Loss: 0.00050461
    Epoch [4950/5000], Loss: 0.00049906
    Epoch [5000/5000], Loss: 0.00049361
    Total Time Elapsed (s): 5.278343200683594

    Model: [500, 100]
    Epoch [1/5000], Loss: 0.33187458
    Epoch [50/5000], Loss: 0.09644475
    Epoch [100/5000], Loss: 0.03475765
    Epoch [150/5000], Loss: 0.00591425
    Epoch [200/5000], Loss: 0.00324807
    Epoch [250/5000], Loss: 0.00220118
    Epoch [300/5000], Loss: 0.00169791
    Epoch [350/5000], Loss: 0.00143689
    Epoch [400/5000], Loss: 0.00129257
    Epoch [450/5000], Loss: 0.00120545
    Epoch [500/5000], Loss: 0.00114630
    Epoch [550/5000], Loss: 0.00110077
    Epoch [600/5000], Loss: 0.00106201
    Epoch [650/5000], Loss: 0.00102684
    Epoch [700/5000], Loss: 0.00099391
    Epoch [750/5000], Loss: 0.00096267
    Epoch [800/5000], Loss: 0.00093292
    Epoch [850/5000], Loss: 0.00090455
    Epoch [900/5000], Loss: 0.00087751
    Epoch [950/5000], Loss: 0.00085173
    Epoch [1000/5000], Loss: 0.00082716
    Epoch [1050/5000], Loss: 0.00080371
    Epoch [1100/5000], Loss: 0.00078134
    Epoch [1150/5000], Loss: 0.00075998
    Epoch [1200/5000], Loss: 0.00073958
    Epoch [1250/5000], Loss: 0.00072008
    Epoch [1300/5000], Loss: 0.00070143
    Epoch [1350/5000], Loss: 0.00068359
    Epoch [1400/5000], Loss: 0.00066652
    Epoch [1450/5000], Loss: 0.00065017
    Epoch [1500/5000], Loss: 0.00063452
    Epoch [1550/5000], Loss: 0.00061951
    Epoch [1600/5000], Loss: 0.00060513
    Epoch [1650/5000], Loss: 0.00059134
    Epoch [1700/5000], Loss: 0.00057812
    Epoch [1750/5000], Loss: 0.00056544
    Epoch [1800/5000], Loss: 0.00055328
    Epoch [1850/5000], Loss: 0.00054161
    Epoch [1900/5000], Loss: 0.00053042
    Epoch [1950/5000], Loss: 0.00051968
    Epoch [2000/5000], Loss: 0.00050938
    Epoch [2050/5000], Loss: 0.00049951
    Epoch [2100/5000], Loss: 0.00049004
    Epoch [2150/5000], Loss: 0.00048097
    Epoch [2200/5000], Loss: 0.00047227
    Epoch [2250/5000], Loss: 0.00046394
    Epoch [2300/5000], Loss: 0.00045597
    Epoch [2350/5000], Loss: 0.00044833
    Epoch [2400/5000], Loss: 0.00044102
    Epoch [2450/5000], Loss: 0.00043402
    Epoch [2500/5000], Loss: 0.00042733
    Epoch [2550/5000], Loss: 0.00042094
    Epoch [2600/5000], Loss: 0.00041483
    Epoch [2650/5000], Loss: 0.00040899
    Epoch [2700/5000], Loss: 0.00040342
    Epoch [2750/5000], Loss: 0.00039810
    Epoch [2800/5000], Loss: 0.00039303
    Epoch [2850/5000], Loss: 0.00038818
    Epoch [2900/5000], Loss: 0.00038357
    Epoch [2950/5000], Loss: 0.00037916
    Epoch [3000/5000], Loss: 0.00037497
    Epoch [3050/5000], Loss: 0.00037097
    Epoch [3100/5000], Loss: 0.00036716
    Epoch [3150/5000], Loss: 0.00036353
    Epoch [3200/5000], Loss: 0.00036007
    Epoch [3250/5000], Loss: 0.00035678
    Epoch [3300/5000], Loss: 0.00035365
    Epoch [3350/5000], Loss: 0.00035066
    Epoch [3400/5000], Loss: 0.00034782
    Epoch [3450/5000], Loss: 0.00034511
    Epoch [3500/5000], Loss: 0.00034253
    Epoch [3550/5000], Loss: 0.00034007
    Epoch [3600/5000], Loss: 0.00033773
    Epoch [3650/5000], Loss: 0.00033549
    Epoch [3700/5000], Loss: 0.00033336
    Epoch [3750/5000], Loss: 0.00033133
    Epoch [3800/5000], Loss: 0.00032939
    Epoch [3850/5000], Loss: 0.00032754
    Epoch [3900/5000], Loss: 0.00032577
    Epoch [3950/5000], Loss: 0.00032408
    Epoch [4000/5000], Loss: 0.00032247
    Epoch [4050/5000], Loss: 0.00032092
    Epoch [4100/5000], Loss: 0.00031944
    Epoch [4150/5000], Loss: 0.00031802
    Epoch [4200/5000], Loss: 0.00031666
    Epoch [4250/5000], Loss: 0.00031536
    Epoch [4300/5000], Loss: 0.00031410
    Epoch [4350/5000], Loss: 0.00031290
    Epoch [4400/5000], Loss: 0.00031174
    Epoch [4450/5000], Loss: 0.00031062
    Epoch [4500/5000], Loss: 0.00030955
    Epoch [4550/5000], Loss: 0.00030851
    Epoch [4600/5000], Loss: 0.00030751
    Epoch [4650/5000], Loss: 0.00030654
    Epoch [4700/5000], Loss: 0.00030561
    Epoch [4750/5000], Loss: 0.00030470
    Epoch [4800/5000], Loss: 0.00030383
    Epoch [4850/5000], Loss: 0.00030298
    Epoch [4900/5000], Loss: 0.00030215
    Epoch [4950/5000], Loss: 0.00030135
    Epoch [5000/5000], Loss: 0.00030057
    Total Time Elapsed (s): 7.834435224533081

```python

```


# Estimadores de máxima verosimilitud: Repaso

> La estimación por máxima verosimilitud (maximum likelihood estimation - MLE) es uno de los métodos más utilizados para estimar los parámetros de un modelo.

> La idea básica, es elegir los parámetros que maximizan la función de verosimilitud. Intuitivamente, esto corresponde a elegir los parámetros que maximizan la probabilidad de los datos observados.

Referencia:

- https://www.univ-orleans.fr/deg/masters/ESA/CH/Chapter2_MLE.pdf

## 1. Principios básicos

**Retomamos el ejemplo de la moneda**.

Definimos la variable aleatoria (discreta) binomial:

$$
X = \left\{ \begin{array}{ccc} 1 & \text{si} & \text{la moneda cae cara} \\
                               0 & \text{si} & \text{la moneda cae sello} \end{array} \right.
$$

con función de masa de probabilidad dada por:

- $P(X=1) = P(X=1 | \theta) = \theta$;
- $P(X=0) = P(X=0 | \theta) = 1 - P(X=1) = 1 - \theta$.

Ahora, suponemos que $X_1, X_2, \dots, X_N$ representan las variables aleatorias correspondientes a $N$ tiros de la moneda. Estas variables aleatorias son **independientes**, e **idénticamente distribuidas** (i.i.d.).

> En muchos contextos (incluidas las aplicaciones de aprendizaje de máquinas) se supone que las diferentes observaciones de los datos son i.i.d.

Ahora, ¿cuál es la probabilidad de observar determinada sucesión de resultados de tiros de la moneda suponiendo que esta sigue la distribución de probabilidad dada arriba?

Supongamos que observamos $\{x_1, x_2, \dots, x_N\}$, donde $x_i = 0, 1$ para todo $i=1, \dots, N$. Queremos entonces averiguar la probabilidad:

$$
P(X_1=x_1, X_2=x_2, \dots, X_N=x_N | \theta)
$$

> Esta probabilidad es una función del parámetro $\theta$ (desconocido) y corresponde a la **verosimilitud de la muestra** $\{x_1, x_2, \dots, x_N\}$, la cual denotamos por:

  $$
  \mathcal{L}(\theta; x_1, x_2, \dots, x_N) = P(X_1=x_1, X_2=x_2, \dots, X_N=x_N | \theta).
  $$

Como suponemos que las variables son **independientes**, tenemos que:

\begin{align}
\mathcal{L}(\theta; x_1, x_2, \dots, x_N) & = P(X_1=x_1, X_2=x_2, \dots, X_N=x_N | \theta) \\
                                          & = P(X_1=x_1 | \theta) P(X_2=x_2| \theta) \dots P(X_N=x_N | \theta) \\
                                          & = \prod_{j=1}^{N} P(X_j = x_j | \theta).
\end{align}

Ahora, analicemos el término $P(X_j = x_j | \theta)$. ¿Qué posibles valores puede tomar?

$$
P(X_j = x_j | \theta) = \left\{ \begin{array}{ccc} \theta   & \text{si} & x_j=1 \\
                                                   1-\theta & \text{si} & x_j=0 \end{array} \right.
$$

De modo que la función de verosimilitud solo depende de cuantas veces de los $N$ tiros cayó cara, y cuantas veces de los $N$ tiros cayó sello.

Denotemos por:

- $N_0$: Número de veces que cayó sello.

- $N_1$: Número de veces que cayó cara.

Se debe satisfacer, claramente, que $N_0 + N_1 = N$.

De esta manera:

$$
\mathcal{L}(\theta; x_1, x_2, \dots, x_N) = \theta^{N_1} (1 - \theta)^{N_0}.
$$

**Ejemplo:**

Supongamos que tiramos la moneda 10 veces, y obtenemos:

$$
\{0, 0, 1, 0, 1, 1, 1, 1, 0, 1\}
$$

En este caso:
- $N_0 = 4$
- $N_1 = 6$

Una idea razonable es encontrar $\theta$ de manera que **la función verosimilitud sea máxima**, o equivalentemente, tal que **se maximice la probabilidad de la muestra observada**.

```python
# Importamos numpy
import numpy as np
# Importamos matplotlib.pyplot
from matplotlib import pyplot as plt
%matplotlib inline
```

```python
# Calculamos la función de verosimilitud con los datos del ejemplo
N0 = 4
N1 = 6
theta = np.linspace(0, 1, 101).astype('float64')
L = theta**N1 * (1 - theta)**N0
l = np.log(L)
```

    /home/esteban/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: RuntimeWarning: divide by zero encountered in log

```python
# Dibujamos la función de verosimilitud
plt.figure(figsize=(6, 4))
plt.plot(theta, L, lw=3, label=r'$\mathcal{L}(\theta: \mathcal{D})=\theta^{N_1}(1-\theta)^{N_0}$')
plt.plot(theta, l, lw=3, label=r'$\mathcal{l}(\theta: \mathcal{D})=\theta^{N_1}(1-\theta)^{N_0}$')
plt.grid()
# plt.axvline(x=theta[L.argmax()], c='k', ls='--')
plt.xlabel(r'$\theta$')
plt.ylabel(r'$\mathcal{L}$')
plt.legend(loc='upper left', bbox_to_anchor=(1.05, 1))
```

```python
# Aproximamos el valor máximo de la verosimilitud
theta[L.argmax()]
```

    0.6

¿Cuál es el valor óptimo para $\theta$?

- Como dicta la intuición es ... 6 / 10 = 0.6
- Como dicta la intuición es ... 60 / 100 = 0.6

Entonces, lo que nos dice el principio de máxima verosimilitud es: elegir $\theta$ tal que

$$
\hat{\theta}_{MLE} = \arg \max_{\theta} \mathcal{L}(\theta; x_1, x_2, \dots, x_N) = \theta^{N_1} (1 - \theta)^{N_0}.
$$

### Truco común:

- El logaritmo (natural) es una función creciente y cóncava.
  - Por tanto,

    $$
    \arg \max_{\theta} \mathcal{L}(\theta; x_1, x_2, \dots, x_N) = \arg \max_{\theta} \underbrace{\log \mathcal{L}(\theta; x_1, x_2, \dots, x_N)}_{l(\theta; x_1, x_2, \dots, x_N): \text{ log-verosimilitud}}
    $$

- La **log-verosimilitud** ofrece la ventaja de convertir las multiplicaciones en sumas:

  \begin{align}
  l(\theta; x_1, x_2, \dots, x_N) & = \log \mathcal{L}(\theta; x_1, x_2, \dots, x_N) \\
                                  & = \log \left(\theta^{N_1} (1 - \theta)^{N_0} \right) \\
                                  & = N_1 \log \theta + N_0 \log(1 - \theta).
  \end{align}

  y en general, las sumas son más fáciles de derivar (y maximizar) que los productos.

> En general, la función de verosimilitud para muestras i.i.d. es:
>
> $$
  \mathcal{L}(\theta; x_1, x_2, \dots, x_N) = \prod_{j=1}^{N} P(X_j = x_j | \theta)
  $$
>
> de manera que la log-verosimilitud será:
>
> $$
  \mathcal{l}(\theta; x_1, x_2, \dots, x_N) = \sum_{j=1}^{N} \log P(X_j = x_j | \theta)
  $$

<font color=green> Maximizar la log-verosimilitud para el ejemplo de la moneda en el pizarrón. </font>

## 2. Estimación de máxima verosimilitud en una distribución normal

En este caso tenemos:

- Variable aleatoria $X \sim \mathcal{N}(\mu, \sigma^2)$.

- $p(x;\mu,\sigma^2)=\frac{1}{\sqrt{2 \pi \sigma^2}} \exp\left(-\frac{(x-\mu)^2}{2\sigma^2}\right)$.

Suponemos que las observaciones

$$\mathcal{D} = \{x_1, \dots, x_N\}$$

son independientes e idénticamente distribuidas.

Mostrar que los estimadores de máxima verosimilitud de $\mu$ y $\sigma$ son:

$$\hat{\mu} = \frac{1}{N} \sum_{j=1}^{N}x_j \qquad \text{y} \qquad \hat{\sigma}=\sqrt{\frac{1}{N}\sum_{j=1}^{N}(x_j-\hat{\mu})^2}.$$

<footer id="attribution" style="float:right; color:#808080; background:#fff;">
Created with Jupyter by Esteban Jiménez Rodríguez.
</footer>


# Day 17: No such thing as too much

```python
containers = [50, 44, 11, 49, 42, 46, 18, 32, 26, 40, 21, 7, 18, 43, 10, 47, 36, 24, 22, 40]
```

## Day 17.1

```python
from sympy.abc import x
from sympy import Poly
```

```python
def combine(total, containers):
    poly = 1
    for i in containers:
        poly *= (1 + x**i)
    poly = Poly(poly)
    return poly.all_coeffs()[total]
```

```python
combine(150, containers)
```

    654

## Day 17.2

```python
from sympy.abc import x, y
```

```python
def min_combine(total, containers):
    poly = 1
    for i in containers:
        poly *= (y + x**i)
    poly = Poly(poly)
    coeff_dict = dict(zip(poly.monoms(), poly.coeffs()))
    master_monom = sorted([m for m in poly.monoms() if m[0] == total], key=lambda a: a[1], reverse=True)[0]
    return coeff_dict[master_monom]
```

```python
min_combine(150, containers)
```

    57


```python
# https://colab.research.google.com/github/kassbohm/wb-snippets/blob/master/ipynb/HTM_03/Selbst/4.3.ipynb

from sympy import *

# Shortcuts:
Ca, Sa, Ct, St = var("cₐ, sₐ, cₜ, sₜ")

#
l, m = var("l, m", positive = True)
a, t = var("α, θ")

ca, sa = cos(a), sin(a)
ct, st = cos(t), sin(t)

sub_list = [
    (ca, Ca),
    (sa, Sa),
    (ct, Ct),
    (st, St),
    ]

R2 = Matrix([
    [ 1, 0,  0  ],
    [ 0, ct, st ],
    [ 0,-st, ct ]
    ])
R1 = Matrix([
    [ ca, 0, -sa ],
    [  0, 1,   0 ],
    [ sa, 0, -ca ]
    ])

R = R2*R1
Rt = R.transpose()

JOt = Matrix([
    [  1, 0, 0 ],
    [  0, 0, 0 ],
    [  0, 0, 1 ]
    ])

JO = Rt*JOt*R

pprint("\nR:")
tmp = R.subs(sub_list)
pprint(tmp)

pprint("\nJO / (ml²/3):")
tmp = JO.subs(sub_list)
pprint(tmp)

# R:
# ⎡ cₐ     0    -sₐ  ⎤
# ⎢                  ⎥
# ⎢sₐ⋅sₜ  cₜ   -cₐ⋅sₜ⎥
# ⎢                  ⎥
# ⎣cₜ⋅sₐ  -sₜ  -cₐ⋅cₜ⎦
#
# JO / (ml²/3):
# ⎡     2     2   2                       2           ⎤
# ⎢   cₐ  + cₜ ⋅sₐ      -cₜ⋅sₐ⋅sₜ  - cₐ⋅cₜ ⋅sₐ - cₐ⋅sₐ⎥
# ⎢                                                   ⎥
# ⎢                          2                        ⎥
# ⎢     -cₜ⋅sₐ⋅sₜ          sₜ           cₐ⋅cₜ⋅sₜ      ⎥
# ⎢                                                   ⎥
# ⎢       2                             2   2     2   ⎥
# ⎣- cₐ⋅cₜ ⋅sₐ - cₐ⋅sₐ  cₐ⋅cₜ⋅sₜ      cₐ ⋅cₜ  + sₐ    ⎦

```


# 17 PDEs: waves in 2D

## 2D wave equation

The wave equation in 2D can be derived in the same fashion as for the 1D case and reads

$$
\left(\frac{\partial^2}{\partial x^2} + \frac{\partial^2}{\partial y^2}\right)u(x, y, t) - \frac{1}{c^2} \frac{\partial^2 u}{\partial t^2} = 0
$$

or in general

$$
\boldsymbol{\nabla}^2 u(\mathbf{x}, t) - \frac{1}{c^2} \frac{\partial^2}{\partial t^2}u(\mathbf{x}, t) = 0.
$$

For the case of a string or a membrane with areal density $\rho$ and constant tension $T$, the wave velocity is

$$
c = \sqrt{\frac{T}{\rho}}.
$$

## Numerical solution: Finite difference discretization

As before, we replace the derivatives with their finite difference approximations, hence the name *finite difference method* (FDM) for this approach.

\begin{align}
\left(\frac{\partial^2}{\partial x^2} + \frac{\partial^2}{\partial y^2}\right)u(x, y, t) &\approx
 \frac{u(x+\Delta x, y, t) - 2u(x, y, t) + u(x-\Delta x, y, t)}{\Delta x^2} \notag\\
 &+ \frac{u(x, y + \Delta y, t) - 2u(x, y, t) + u(x, y-\Delta y, t)}{\Delta y^2}
\end{align}

Setting $\Delta x = \Delta y = \delta$ (same spacing in $x$ and $y$) and $u_{ijn} := u(i\delta, j\delta, n\Delta t)$ we obtain the discretized 2D wave equation

$$
u_{i,j,n+1} = 2(1 - 2\beta^2)u_{i,j,n} - u_{i,j,n-1} + \beta^2(u_{i+1,j,n} + u_{i-1,j,n} + u_{i,j+1,n} + u_{i,j-1,n})\\
\beta := \frac{c}{\delta / \Delta t}
$$

## Implementation
Packaged as a function.

```python
import matplotlib.pyplot as plt
import numpy as np

plt.style.use('ggplot')
```

```python
def wave2D(L=0.5, N=50, Dt=1e-4, Nt=100, step=1,
           rho=1.5e-3, tension=5., aspect=1):

    Lx = L
    Ly = aspect*L
    Nx = N
    Ny = int(aspect*N)
    delta = L/N

    c = np.sqrt(tension/rho)

    beta = c*Dt/delta
    beta2 = beta**2

    print("c = {0} m/s".format(c))
    print("delta = {0} m,  Dt = {1} s, delta/Dt = {2} m/s".format(delta, Dt, delta/Dt))
    print("beta = {}".format(beta))
    print("beta * sqrt(2) = {}".format(beta * np.sqrt(2)))

    x = np.linspace(0, Lx, Nx+1)  # need N+1!
    y = np.linspace(0, Ly, Ny+1)
    # note: we use u[i*delta, j*delta] = u(x, y)
    #       (which is the transpose of the numpy row-column layout so we transpose X and Y)
    X, Y = np.meshgrid(x, y)
    X, Y = X.T, Y.T

    def gaussian(x, u0=0.05, x0=None, sigma=0.1*L):
        x0 = np.mean(x) if x0 is None else x0
        g = u0/np.sqrt(2*np.pi*sigma**2) * np.exp(-(x-x0)**2 / (2*sigma**2))
        return g

    # u[i*delta, j*delta] = u(x, y)
    # displacements at j-1, j, j+1
    u0 = np.zeros((Nx+1, Ny+1))
    u1 = np.zeros_like(u0)
    u2 = np.zeros_like(u0)

    # save array
    u_t = np.zeros((int(np.ceil(Nt/step)), Nx+1, Ny+1))

    # use a boolean mask to define regions
    inside = np.zeros_like(u0, dtype=np.bool)
    inside[1:-1, 1:-1] = True
    boundary = np.logical_not(inside)

    # boundary conditions
    def set_boundary_conditions(u):
        u[boundary] = 0

    set_boundary_conditions(u0)
    set_boundary_conditions(u1)
    set_boundary_conditions(u2)

    # initial conditions: velocity 0, i.e. no difference between y0 and y1
    u0[inside] = u1[inside] = (gaussian(X)*gaussian(Y))[inside]

    # save initial
    t_index = 0
    u_t[t_index, :, :] = u0
    if step == 1:
        t_index += 1
        u_t[t_index, :, :] = u1

    for nt in range(2, Nt):
        u2[1:-1, 1:-1] = 2*(1-2*beta2)*u1[1:-1, 1:-1] - u0[1:-1, 1:-1] + beta2*(u1[2:, 1:-1]
                                            + u1[:-2, 1:-1] + u1[1:-1, 2:] + u1[1:-1, :-2])
        u0[:, :], u1[:, :] = u1, u2

        if nt % step == 0 or nt == Nt-1:
            t_index += 1
            u_t[t_index, :, :] = u2
            print("Iteration {0:5d}".format(nt), end="\r")
    else:
        print("Completed {0:5d} iterations: t={1} s".format(nt, nt*Dt))

    return u_t, X, Y, delta, Dt, step

```

In 2D, the Courant criterion for stability is

$$
\beta = \frac{c}{\delta/\Delta t} ≤ \frac{1}{\sqrt{2}}
$$

or

$$
\sqrt{2} \beta ≤ 1.
$$

If the solution becomes unstable, check $\sqrt{2}\beta$!

```python
u_t, X, Y, delta, Dt, step = wave2D(Nt=1000)
```

    c = 57.735026918962575 m/s
    delta = 0.01 m,  Dt = 0.0001 s, delta/Dt = 100.0 m/s
    beta = 0.5773502691896257
    beta * sqrt(2) = 0.816496580927726
    Completed   999 iterations: t=0.0999 s

```python
times = Dt * np.arange(0, len(u_t))
```

Look at the minimum and maximum values for each frame:

```python
fig = plt.figure(figsize=(6,3))
ax = fig.add_subplot(111)
ax.plot(times, u_t.min(axis=(1, 2)),
         times, u_t.max(axis=(1, 2)))
ax.set_xlabel("time (s)")
fig.tight_layout()
plt.show()
```

The min/max values do not appear to grow, indicating stable integration.

#### 2D Animation
For 2D animation to work in a Jupyter notebook, use

```python
%matplotlib widget
```

If no animations are visible, restart kernel and execute the `%matplotlib widget` cell as the very first one in the notebook.

We use `matplotlib.animation` to look at movies of our solution:

```python
import matplotlib.animation as animation
```

The `update_wave()` function simply re-draws our image for every `frame`.

```python
u_limits = 0.5*u_t.min(), 0.5*u_t.max()

fig1 = plt.figure(figsize=(5,5))
ax = fig1.add_subplot(111)
ax.set_aspect(1)

def update_wave(frame, data):
    global ax, Dt, u_limits, X, Y
    ax.clear()
    ax.set_xlabel("x")
    ax.set_ylabel("y")
    ax.contour(X, Y, data[frame], 30, cmap=plt.cm.viridis, norm=plt.Normalize(*u_limits))
    ax.contourf(X, Y, data[frame], 30, cmap=plt.cm.viridis, norm=plt.Normalize(*u_limits), alpha=0.8)
    ax.text(0.1, 0.9, "t = {0:g} s".format(frame*Dt), transform=ax.transAxes)

wave_anim = animation.FuncAnimation(fig1, update_wave, len(u_t), fargs=(u_t, ),
                                    interval=5, blit=False, repeat_delay=1000)

```

If you have `ffmpeg` installed then you can export to a MP4 movie file:

```python
wave_anim.save("square_drum_contours.mp4", fps=30, dpi=300)
```

#### 3D animation

```python
from mpl_toolkits.mplot3d import Axes3D
```

```python
def plot_wireframe(X, Y, u, limits=None, ax=None):
    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111, projection="3d")
    ax.plot_wireframe(X, Y, u)
    ax.set_xlabel(r"position $x$ (m)")
    ax.set_ylabel(r"position $y$ (m)")
    ax.set_zlabel(r"displacement $u$ (m)")
    ax.set_zlim(limits)
    ax.figure.tight_layout()
    return ax
```

```python
plot_wireframe(X, Y, u_t[0])
```

```python
u_limits = 0.5*u_t.min(), 0.5*u_t.max()

fig2 = plt.figure(figsize=(7,7))
ax = fig2.add_subplot(111, projection="3d")
#ax.set_aspect(1)

def update_wire(frame, data):
    global ax, u_limits, X, Y
    ax.clear()
    plot_wireframe(X, Y, data[frame], limits=u_limits, ax=ax)
    ax.text(0.1, 0.9, 0.9, "t = {0:3.1f} ms".format(frame*Dt*1000), transform=ax.transAxes)

wire_anim = animation.FuncAnimation(fig2, update_wire, len(u_t), fargs=(u_t,),
                                    interval=30, blit=False, repeat_delay=1000)

```

```python
wire_anim.save("square_drum_wire.mp4", fps=30, dpi=300)
```

#### Making pretty graphics

We want to us a surface representation where the color indicates the displacement. We also want to show our animation on a [webpage](http://asu-compmethodsphysics-phy494.github.io/ASU-PHY494/2017/04/20/17_PDEs_waves/#leap-frog-algorithm-for-solving-the-wave-equation).

```python
import matplotlib.colors
```

```python
def plot_surface(X, Y, u, limits=None, ax=None, **kwargs):
    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111, projection="3d")
    ax.plot_surface(X, Y, u, rstride=1, cstride=1, shade=True,
                    cmap=plt.cm.coolwarm, norm=norm, **kwargs)
    ax.set_xlabel(r"$x$ (m)")
    ax.set_ylabel(r"$y$ (m)")
    ax.set_zlabel(r"$u$ (m)")

    # fix scaling see https://stackoverflow.com/a/19933125
    #scaling = np.array([getattr(ax, 'get_{}lim'.format(dim))() for dim in 'xyz'])
    scaling = np.array([X.min(), X.max(), Y.min(), Y.max(), min(limits), max(limits)])
    ax.auto_scale_xyz(*[[np.min(scaling), np.max(scaling)]]*3)

    ax.set_zlim(limits)

    ax.figure.tight_layout()
    return ax

u_limits = 0.5*u_t.min(), 0.5*u_t.max()
umax = 0.4*np.max(np.abs(u_limits))
norm = matplotlib.colors.Normalize(vmin=-umax, vmax=umax)

fig3 = plt.figure(figsize=(3.5, 3.5))
ax = fig3.add_subplot(111, projection="3d")
# ax.set_aspect(1)

def update_surface(frame, data):
    global ax, u_limits, X, Y
    ax.clear()
    plot_surface(X, Y, data[frame], limits=u_limits, ax=ax, alpha=0.9)
    ax.text(0.2, 0.9, 0.9, "$t = {0:3.1f}$ ms".format(frame*Dt*1000), transform=ax.transAxes)
    ax.figure.tight_layout()

wire_anim = animation.FuncAnimation(fig3, update_surface, len(u_t), fargs=(u_t,),
                                    interval=30, blit=False, repeat_delay=1000)

```

The whole video can be incorporated into an html page (uses the HTML5 `<video>` tag).

```python
tag = wire_anim.to_html5_video()
with open("square_drum_redblue_2d.html", "w") as html:
    html.write(tag)
```

or exported as a MPEG4 movie (uses `ffmpeg`):

```python
wire_anim.save("square_drum_surface.mp4", fps=30, dpi=300)
```

```python

```


```python
# Load modules
from __future__ import print_function
import os
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

```

```python
# Change working Directory
os.chdir('../Data Files')

#read the data from into a pandas.DataFrame
wisc_emp = pd.read_csv('wisconsin-employment-time-series.csv')

#Let's find out the shape of the DataFrame
print('Shape of the DataFrame:', wisc_emp.shape)

#Let's see first 10 rows of it
wisc_emp.head()

# plot the wisconsin employment dataset
wisc_emp.plot(figsize=(16,8))
```

```python
# Capture seasonality component
def initialize_T(x, seasonLength):
    total=0.0
    for i in range(seasonLength):
        total+=float(x[i+seasonLength]-x[i])/seasonLength
    return total

initialize_T(wisc_emp['Employment'], 12)

# Initialize seasonal trend
def initialize_seasonalilty(x, seasonLength):
    seasons={}
    seasonsMean=[]
    num_season=int(len(x)/seasonLength)
    # Compute season average
    for i in range(num_season):
        seasonsMean.append(sum(x[seasonLength*i:seasonLength*i+seasonLength])/float(seasonLength))

    # compute season intial values
    for i in range(seasonLength):
        tot=0.0
        for j in range(num_season):
            tot+=x[seasonLength*j+i]-seasonsMean[j]
        seasons[i]=tot/num_season
    return seasons

initialize_seasonalilty(wisc_emp['Employment'], 12)

# Triple Exponential Smoothing Forecast
def triple_exp_smoothing(x, seasonLength, alpha, beta, gamma, h):
    yhat=[]
    S = initialize_seasonalilty(x, seasonLength)
    for i in range(len(x)+h):
        if i == 0:
            F = x[0]
            T = initialize_T(x, seasonLength)
            yhat.append(x[0])
            continue
        if i >= len(x):
            m = i - len(x) + 1
            yhat.append((F + m*T) + S[i%seasonLength])
        else:
            obsval = x[i]
            F_last, F= F, alpha*(obsval-S[i%seasonLength]) + (1-alpha)*(F+T)
            T = beta * (F-F_last) + (1-beta)*T
            S[i%seasonLength] = gamma*(obsval-F) + (1-gamma)*S[i%seasonLength]
            yhat.append(F+T+S[i%seasonLength])
    return yhat

```

```python
# Triple exponential smoothing
wisc_emp['TES']=triple_exp_smoothing(wisc_emp['Employment'], 12, 0.4, 0.6, 0.2, 0)

### Plot Single Exponential Smoothing forecasted value
fig = plt.figure(figsize=(16, 8))
ax = fig.add_subplot(2,1,1)
wisc_emp['Employment'].plot(ax=ax)
ax.set_title('Beer Production')
ax = fig.add_subplot(2,1,2)
wisc_emp['TES'].plot(ax=ax, color='r')
ax.set_title('Triple Smoothing Forecast')
plt.savefig('../plots/ch3/B07887_03_16.png', format='png', dpi=300)
```

\begin{equation}
F_t = \alpha(x_t + S_{t-L})+(1-\alpha)(F_{t-1}+T_{t-1})
\end{equation}

\begin{equation}
T_t = \beta(F_t - F_{t-1}) + (1-\beta)(T_{t-1})
\end{equation}

\begin{equation}
S_t = \gamma(x_t - F_t) + (1-\gamma)S_{t-C}
\end{equation}

```python
# Effect of alpha and beta
wisc_emp['TEF001'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 0, 0, 1, 0)
wisc_emp['TEF010'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 0, 1, 0, 0)
wisc_emp['TEF100'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 1, 0, 0, 0)

```

```python
# Plot alpha=0, beta=0, gamma=1
fig = plt.figure(figsize=(16, 8))
ax = fig.add_subplot(3,1,1)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF001'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=0, beta=0, gamma=1')
ax = fig.add_subplot(3,1,2)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF010'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=0, beta=1, gamma=0')
ax = fig.add_subplot(3,1,3)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF100'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=1, beta=0, gamma=0')
fig.subplots_adjust(hspace=.5)
plt.savefig('../plots/ch3/B07887_02_16.png', format='png', dpi=300)

```

```python
# Effect of alpha and beta
wisc_emp['TEF110'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 1, 1, 0, 0)
wisc_emp['TEF101'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 1, 0, 1, 0)
wisc_emp['TEF111'] = triple_exp_smoothing(wisc_emp['Employment'], 12, 1, 1, 1, 0)
```

```python
fig = plt.figure(figsize=(16, 8))
ax = fig.add_subplot(3,1,1)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF110'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=1, beta=1, gamma=0')
ax = fig.add_subplot(3,1,2)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF101'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=1, beta=0, gamma=1')
ax = fig.add_subplot(3,1,3)
wisc_emp['Employment'].plot(color='b', linestyle = '-', ax=ax)
wisc_emp['TEF111'].plot(color='r', linestyle = '--', ax=ax)
ax.set_title('TES: alpha=1, beta=1, gamma=1')
fig.subplots_adjust(hspace=.5)
plt.savefig('../plots/ch3/B07887_01_16.png', format='png', dpi=300)
```

```python

```


The theory behind the QR algortihm and its different variations implemented in Python

This algorithm is solely based on the QR factorisation of matrices in $\mathbb{R}^{mxn}$ and it is one of the most modern methods for approximating the eigenvalues of a matrix. Throughout this method, although it is not necessary, we assume that the matrix that we are trying to approximate its eigenvalues is invertible so its QR decomposition is unique, if we also require the diagonal elements of the upper triangular matrix $R$ such as $A=QR$ where $Q$  is unitary (orthogonal if the matrix that we start with is real), that is $Q^*=Q^{-1}$, **to be positive**. Below we see the one practical way that we saw in our notes on how to find the QR factorisation of our starting matrix (the Gram-Schmidt way is not studied here due to its numerical instability - see Linear Algebra II notes on how that method works and the QR Decomposition using Givens Rotations is impractical and outshined by the Householder method discussed below), and then we see the 3 variations of the QR algorithm that we studies in our lecture notes.

But like we said, its important to note that the requirement that A is invertible is NOT essential - any matrix (even non square and even singular) can assume a QR decomposition - its just not going to be unique, up to the matrix $R$!

___

QR decomposition using Householder reflections

A Householder reflection is a matrix $P \in \mathbb{C}^{nxn}$ where: $P=I_n-2\frac{\vec{v}\vec{v}^*}{\vec{v}^*\vec{v}}$ and $\vec{v} \in \mathbb{C}^n$.

It is known that P is hermitian ($P^*=P$) and unitary ($P^*=P^{-1}$) so its an **involution** - $P^2=I_n$.

Now it is possible for every $\vec{x} \in \mathbb{C}^n$ to find a Householder transformation $P=P(\vec{x})$ such as, if $\vec{x}=(x_1,\dots,x_n)^T$ then:

\begin{align}
    P\vec{x} &= P\begin{pmatrix}
                  x_{1} \\
                  x_{2} \\
                  \vdots \\
                  x_k \\
                  x_{k+1} \\
                  \vdots \\
                  x_{j-1} \\
                  x_j \\
                  \vdots \\
                  x_{n}
                  \end{pmatrix} \\
             &= \begin{pmatrix}
                  x_{1} \\
                  x_{2} \\
                  \vdots \\
                  x_{k-1} \\
                  -sgn(x_k)\alpha \\
                  0 \\
                  \vdots \\
                  0 \\
                  x_{j+1} \\
                  \vdots \\
                  x_{n}
                  \end{pmatrix}
\end{align}

where $\alpha=||\vec{y}||_2$ (2-norm in $\mathbb{C}^n$ (or $\mathbb{R}^n$)) where $\vec{y}=\begin{pmatrix}
                  x_{k} \\
                  x_{k+1} \\
                  \vdots \\
                  x_{j}
                  \end{pmatrix} \in \mathbb{C}^{j-k+1}$

Now we proved in our notes that this matrix is none other than:

\[ P=I_n-2\frac{\vec{v}\vec{v}^*}{\vec{v}^*\vec{v}}  \]

for $\vec{v}= \begin{pmatrix}
                  0 \\
                  \vdots \\
                  0 \\
                  x_{k} + sgn(x_k)\alpha \\
                  x_{k+1} \\
                  \vdots \\
                  x_{j} \\
                  0 \\
                  \vdots \\
                  0
                  \end{pmatrix} \in \mathbb{C}^n $

Now this is essential in our methodology for the QR decomposition because we can use these Householder transformations in the following way:

Essentially $H_i$ is a Householder matrix, that transforms the $i$-th column in the manner discussed above, of the product $H_{i-1}A, \ \forall i=1,\dots,n$ - this works due to the definition of matrix multiplication where if $C$ and $D$ matrices (where for simplicity both are $nxn$ square matrices), then if $\vec{d}_1,\dots,\vec{d}_ n$ the $n$ columns of matrix $D$:

\[ CD = \begin{pmatrix}
        \uparrow  \ \  \dots \ \ \uparrow \\
        C\vec{d}_1 \ \ \dots \ \ C\vec{d}_n \\
        \downarrow  \ \  \dots \ \ \downarrow \\
        \end{pmatrix} \]

Although there is already a QR decomposition function in Python under the numpy module - `numpy.linalg.qr`, we create a function here for educational purposes, using the above methodology:

```python
import numpy as np
import sympy
from functools import reduce
sympy.init_printing(use_latex="mathjax")

def qr_householder(A):
    """A function that calculates the QR decomposition of the input matrix A
    and outputs the resulting matrices using sympy and also returns them as np.arrays"""
    try:
        if np.linalg.det(A) == 0:
            print("For our implementation A has to be an invertible matrix, please try again.")
            return None
    except numpy.linalg.LinAlgError:
        print("For our implementation A has to be a square matrix, please try again.")
        return None
    n = A.shape[0]
    R = A; H_list = []
    for i in range(n-1):
        norm_col = np.linalg.norm(R[i:,i])
        first_el = np.array([R[i,i]+norm_col]) # or np.array([R[i,i]-norm_col]) -
                                               # doesn't make a difference
        if i==0:
            v = np.concatenate((first_el,R[i+1:,i]))
        else:
            v = np.concatenate((np.zeros((i)),first_el,R[i+1:,i]))
        v.shape = (n,1)
        H = np.eye(n)-2*(v @ np.transpose(v))/np.linalg.norm(v)**2
        H_list.append(H)
        R = H @ R
    Q = reduce(lambda X,Y: X @ Y, H_list)
    return [Q, R]
```

```python
# Testing to see if everything is okay
A = np.array([[12, -51, 4], [6, 167, -68], [-4, 24, -41]])
Q, R = qr_householder(A)
```

```python
sympy.Matrix(Q)
```

$\displaystyle \left[\begin{matrix}-0.857142857142857 & 0.394285714285714 & 0.331428571428571\\-0.428571428571429 & -0.902857142857143 & -0.0342857142857143\\0.285714285714286 & -0.171428571428571 & 0.942857142857143\end{matrix}\right]$

```python
sympy.Matrix(R) # note that it is indeed, to the accuracy of the
                # epsilon of the machine, upper triangular
```

$\displaystyle \left[\begin{matrix}-14.0 & -21.0 & 14.0\\-7.7835174203336 \cdot 10^{-16} & -175.0 & 70.0\\-5.33590265989075 \cdot 10^{-16} & 0.0 & -35.0\end{matrix}\right]$

```python
sympy.Matrix(Q)*sympy.Matrix(R) # see that we got back A
```

$\displaystyle \left[\begin{matrix}12.0 & -51.0 & 4.0\\6.0 & 167.0 & -68.0\\-4.0 & 24.0 & -41.0\end{matrix}\right]$

With this out of the way, its time to see the 3 variations of the QR algorithm that we saw in the class - for more theory see corresponding lectures in "Numerical Linear Algebra" from Trefethen and Bau. We will not get into the details of each variation of the algorithm - for why these methods work intuitively (or proven rigorously), see the lecture notes and the handwritten complementary theory and observations accompanying this chapter for this information.

### NOTE:
### In all the variations below, we can accelerate their order of convergece (the ratio of convergence stays the same - the ratio of 2 consecutive eigenvalues) by first "*relaxing*" A with *Householder matrices* and bringing it in its upper triangular Hessenberg form (this is because the QR algorithm *PRESERVES* upper triangular Hesseberg matrices) and then using consecutive *deflations* of the resulting matrices to decrease the number of eigenvalues we need to find, each time we "pinpoint" an eigenvalue, thus decreasing the dimensions of the problem each time by 1.

___

### Simple QR algorithm (withour shifts)

```python
def simple_qr(A, tol, maxiter):
    """Uses the simple no-shift QR algorithm to approximate
    the eigenvalues of the input matrix"""
    A_new=A
    n = 0
    while np.linalg.norm(
        np.tril(A_new)-np.diag(np.diag(A_new)), ord=2) >= tol and n<maxiter:
        # we can use whatever norm we want due to the equivelance of operator
        # norms in vector spaces of finite dimensions
        Q, R = qr_householder(A_new)
        A_new = R @ Q
        n += 1
    return [np.diag(A_new), n]
```

```python
# Testing to see if everything is okay
A = np.array([[8, 7, 7], [5, 8, 4], [2, 0, 8]])
eigvals, iterations = simple_qr(A, 1E-16, 100)
```

```python
print(f"""Number of iterations it took to converge to the
          specified tolerace: {iterations}""")
```

    Number of iterations it took to converge to the
              specified tolerace: 45

```python
# Approximation using the simple QR algorithm
sympy.Matrix(eigvals)
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\6.82394694026641\\1.66251756658343\end{matrix}\right]$

```python
# "Exact" eigvalues of matrix A, using the np.linalg.eig() function
sympy.Matrix(np.linalg.eig(A)[0])
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\1.66251756658343\\6.82394694026639\end{matrix}\right]$

### QR algorithm with simple shifts

```python
def simple_shift_qr(A, tol, maxiter):
    """Uses the simple shift QR algorithm to approximate the
    eigenvalues of the input matrix"""
    n = A.shape[0]
    sigma=A[n-1,n-1]
    niter = 0
    while np.linalg.norm(
        np.tril(A-sigma*np.eye(n,n))-np.diag(np.diag(A-sigma*np.eye(n,n))), ord=2) \
        >= tol and niter<maxiter:
        # we can use whatever norm we want due to the equivelance of operator norms in
        # vector spaces of finite dimensions (frobenius norm, 2/1/inf-norm)
        Q, R = qr_householder(A-sigma*np.eye(n,n))
        A = R @ Q + sigma*np.eye(n,n)
        sigma=A[n-1,n-1]
        niter += 1
    return [np.diag(A), n]
```

```python
# Testing to see if everything is okay
A = np.array([[8, 7, 7], [5, 8, 4], [2, 0, 8]])
eigvals, iterations = simple_shift_qr(A, 1E-20, 100)
```

```python
print(f"""Number of iterations it took to converge to the
          specified tolerace: {iterations}""")
```

    Number of iterations it took to converge to the
              specified tolerace: 3

```python
# Approximation using the simple-shift QR algorithm
sympy.Matrix(eigvals)
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\1.66251756657081\\6.823946940279\end{matrix}\right]$

```python
# "Exact" eigvalues of matrix A, using the np.linalg.eig() function
sympy.Matrix(np.linalg.eig(A)[0])
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\1.66251756658343\\6.82394694026639\end{matrix}\right]$

### QR algorithm with Wilkinson shifts

```python
def wilkinson_shift_qr(A, tol, maxiter):
    """Uses the Wilkinson shift QR algorithm to approximate the
    eigenvalues of the input matrix"""
    n = A.shape[0]
    sub_matrix = A[n-2:n,n-2:n]
    char_pol = [1, -np.trace(sub_matrix), np.linalg.det(sub_matrix)]
    sigma=max(map(lambda x: abs(x), numpy.roots(char_pol)))
    niter = 0
    while np.linalg.norm(
        np.tril(A-sigma*np.eye(n,n))-np.diag(np.diag(A-sigma*np.eye(n,n))), ord=2) \
        >= tol and niter<maxiter:
        # we can use whatever norm we want due to the equivelance of operator norms in
        # vector spaces of finite dimensions (frobenius norm, 2/1/inf-norm)
        Q, R = qr_householder(A-sigma*np.eye(n,n))
        A = R @ Q + sigma*np.eye(n,n)
        sub_matrix = A[n-2:n,n-2:n]
        char_pol = [1, -np.trace(sub_matrix), np.linalg.det(sub_matrix)]
        sigma=max(map(lambda x: abs(x), numpy.roots(char_pol)))
        niter += 1
    return [np.diag(A), n]
```

```python
# Testing to see if everything is okay
A = np.array([[8, 7, 7], [5, 8, 4], [2, 0, 8]])
eigvals, iterations = simple_shift_qr(A, 1E-20, 100)
```

```python
print(f"""Number of iterations it took to converge to the
          specified tolerace: {iterations}""")
```

    Number of iterations it took to converge to the
              specified tolerace: 3

```python
# Approximation using the simple-shift QR algorithm
sympy.Matrix(eigvals)
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\1.66251756657081\\6.823946940279\end{matrix}\right]$

```python
# "Exact" eigvalues of matrix A, using the np.linalg.eig() function
sympy.Matrix(np.linalg.eig(A)[0])
```

$\displaystyle \left[\begin{matrix}15.5135354931502\\1.66251756658343\\6.82394694026639\end{matrix}\right]$

Variations on the QR algorithm for approximating eigenvalues of matrices.ipynb