```python
!pip install nangs
```

    Requirement already satisfied: nangs in /usr/local/lib/python3.7/dist-packages (0.1.2)

```python
# autoreload nangs
%reload_ext autoreload
%autoreload 2
%matplotlib inline
```

```python

```

```python
#imports

import numpy as np
import matplotlib.pyplot as plt
import torch
import pandas as pd
import nangs
from nangs import *

device = "cuda" if torch.cuda.is_available() else "cpu"

nangs.__version__, torch.__version__
```

    ('0.1.2', '1.9.0+cu102')

```python
class Sine(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.sin(x)

def block(i, o):
    fc = torch.nn.Linear(i, o)
    return torch.nn.Sequential(
        #Sine(),
        torch.nn.GELU(),
        torch.nn.Linear(i, o)
    )

class MLP(torch.nn.Module):
    def __init__(self, inputs, outputs, layers, neurons):
        super().__init__()
        fc_in = torch.nn.Linear(inputs, neurons)
        fc_hidden = [
            block(neurons, neurons)
            for layer in range(layers-1)
        ]
        fc_out = block(neurons, outputs)

        self.mlp = torch.nn.Sequential(
            fc_in,
            *fc_hidden,
            fc_out
        )

    def forward(self, x):
        return self.mlp(x)
```

```python
device

```

    'cuda'

```python
alpha=10
class Difussion2d(PDE):
    def computePDELoss(self, inputs, outputs):
        p = outputs[:, 0]
        grads = self.computeGrads(p, inputs)
        dpdx, dpdt = grads[:, 0], grads[:, 1]
        # second order derivatives
        dp2dx2 = self.computeGrads(dpdx, inputs)[:, 0]
        #dp2dy2 = self.computeGrads(dpdy, inputs)[:, 1]
        return {'pde':dpdt - dp2dx2}
```

```python
class NeumannX(Neumann):
    def computeBocoLoss(self, inputs, outputs):
        p = outputs[:, 0]
        grads = self.computeGrads(p, inputs)
        dpdx = grads[:, 0]
        return {'gradX': dpdx}
```

```python
class PBC0(Neumann):
    def computeBocoLoss(self, inputs, outputs):
        pbc = 0
        p = outputs[:, 0]
        return {'PWF': 100*(p-pbc)}
```

```python
pde = Difussion2d(inputs=('x', 't'), outputs='p')
```

```python
def append(arr, values, axis=None):
    arr = np.asanyarray(arr)
    if axis is None:
        if arr.ndim != 1:
            arr = arr.ravel()
        values = np.ravel(values)
        axis = arr.ndim-1
    return np.concatenate((arr, values), axis=axis)
```

```python
# mesh

x = np.linspace(0,1,100)

t = np.linspace(0,1,100)
#t = append(t,np.linspace(0.2,1,10))
#t=np.array([0])
#tt=np.logspace(-2,0,20)
#t = append(t,tt)
mesh = Mesh({'x': x, 't':t}, device=device)
pde.set_mesh(mesh)
```

```python
t
```

    array([0.        , 0.01010101, 0.02020202, 0.03030303, 0.04040404,
           0.05050505, 0.06060606, 0.07070707, 0.08080808, 0.09090909,
           0.1010101 , 0.11111111, 0.12121212, 0.13131313, 0.14141414,
           0.15151515, 0.16161616, 0.17171717, 0.18181818, 0.19191919,
           0.2020202 , 0.21212121, 0.22222222, 0.23232323, 0.24242424,
           0.25252525, 0.26262626, 0.27272727, 0.28282828, 0.29292929,
           0.3030303 , 0.31313131, 0.32323232, 0.33333333, 0.34343434,
           0.35353535, 0.36363636, 0.37373737, 0.38383838, 0.39393939,
           0.4040404 , 0.41414141, 0.42424242, 0.43434343, 0.44444444,
           0.45454545, 0.46464646, 0.47474747, 0.48484848, 0.49494949,
           0.50505051, 0.51515152, 0.52525253, 0.53535354, 0.54545455,
           0.55555556, 0.56565657, 0.57575758, 0.58585859, 0.5959596 ,
           0.60606061, 0.61616162, 0.62626263, 0.63636364, 0.64646465,
           0.65656566, 0.66666667, 0.67676768, 0.68686869, 0.6969697 ,
           0.70707071, 0.71717172, 0.72727273, 0.73737374, 0.74747475,
           0.75757576, 0.76767677, 0.77777778, 0.78787879, 0.7979798 ,
           0.80808081, 0.81818182, 0.82828283, 0.83838384, 0.84848485,
           0.85858586, 0.86868687, 0.87878788, 0.88888889, 0.8989899 ,
           0.90909091, 0.91919192, 0.92929293, 0.93939394, 0.94949495,
           0.95959596, 0.96969697, 0.97979798, 0.98989899, 1.        ])

```python
from nangs import Dirichlet
```

```python
t0 = np.array([0])
```

```python
_x = np.meshgrid(x)
#p0 = np.sin(2*np.pi*_x)*np.sin(2*np.pi*_y)
```

```python
_x[0].shape
```

    (100,)

```python
p0=np.zeros((100))
p0[:]=1
p0[0]=0
#p0[47:53,47:53]=1
```

```python
p0.shape
```

    (100,)

```python
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
fig = plt.figure()
#ax = fig.gca(projection='3d')
plt.plot(_x[0], p0)
#fig.colorbar(surf, shrink=0.5, aspect=5)
#plt.show()
```

```python
initial_condition = Dirichlet({'x': x,'t':t0}, {'p': p0.reshape(-1)}, device=device, name="initial")
```

```python
p0.shape
```

    (100,)

```python
pde.add_boco(initial_condition)
#p0=np.ones(len(y))
#p1=np.zeros(len(y))
```

```python
pwf = PBC0({'x': np.array([0]),'t':t}, name='pwf', device=device)
#left = Dirichlet({'x': x, 'y': y}, {'p': p0.reshape(-1)}, device=device, name="left")
```

```python
pde.add_boco(pwf)
```

```python
#right = PBC0({'x': np.array([1]), 'y': y, 't':t}, name='right', device=device)
#right = Dirichlet({'x': x[-1], 'y': y}, {'p': p1.reshape(-1)}, device=device, name="right")
right =  NeumannX({'x': x[-1],'t':t}, name='right', device=device)
```

```python
pde.add_boco(right)
```

```python
#from nangs import MLP

BATCH_SIZE = 128
LR = 1e-2
EPOCHS = 250
NUM_LAYERS = 4
NUM_HIDDEN = 50

mlp = MLP(len(pde.inputs), len(pde.outputs), NUM_LAYERS, NUM_HIDDEN).to(device)
optimizer = torch.optim.Adam(mlp.parameters())
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR, pct_start=0.1, total_steps=EPOCHS)
```

```python
mlp
```

    MLP(
      (mlp): Sequential(
        (0): Linear(in_features=2, out_features=50, bias=True)
        (1): Sequential(
          (0): GELU()
          (1): Linear(in_features=50, out_features=50, bias=True)
        )
        (2): Sequential(
          (0): GELU()
          (1): Linear(in_features=50, out_features=50, bias=True)
        )
        (3): Sequential(
          (0): GELU()
          (1): Linear(in_features=50, out_features=50, bias=True)
        )
        (4): Sequential(
          (0): GELU()
          (1): Linear(in_features=50, out_features=1, bias=True)
        )
      )
    )

```python

scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR, pct_start=0.1, total_steps=EPOCHS)
```

```python
pde.compile(mlp, optimizer, scheduler)
```

```python

%time hist = pde.solve(EPOCHS, BATCH_SIZE)
```

Epoch 1/250  | lr 0.0004 | initial 0.95832 | pwf_PWF 15.29493 | right_gradX 0.0 | pde 4e-05
Epoch 2/250  | lr 0.00044 | initial 0.98294 | pwf_PWF 0.15229 | right_gradX 1e-05 | pde 0.0
Epoch 3/250  | lr 0.00056 | initial 0.98339 | pwf_PWF 0.003 | right_gradX 3e-05 | pde 0.0
Epoch 4/250  | lr 0.00077 | initial 0.98035 | pwf_PWF 0.00048 | right_gradX 7e-05 | pde 0.0
Epoch 5/250  | lr 0.00104 | initial 0.97519 | pwf_PWF 0.00028 | right_gradX 0.0002 | pde 1e-05
Epoch 6/250  | lr 0.00139 | initial 0.96547 | pwf_PWF 0.00024 | right_gradX 0.00063 | pde 5e-05
Epoch 7/250  | lr 0.00181 | initial 0.94369 | pwf_PWF 0.00019 | right_gradX 0.00274 | pde 0.00037
Epoch 8/250  | lr 0.00228 | initial 0.88718 | pwf_PWF 0.00017 | right_gradX 0.01245 | pde 0.00223
Epoch 9/250  | lr 0.0028 | initial 0.72948 | pwf_PWF 0.00022 | right_gradX 0.03554 | pde 0.01616
Epoch 10/250  | lr 0.00336 | initial 0.38798 | pwf_PWF 0.00136 | right_gradX 0.06169 | pde 0.04815
Epoch 11/250  | lr 0.00396 | initial 0.19855 | pwf_PWF 0.00665 | right_gradX 0.00658 | pde 0.04493
Epoch 12/250  | lr 0.00457 | initial 0.21199 | pwf_PWF 0.12296 | right_gradX 0.0014 | pde 0.02391
Epoch 13/250  | lr 0.0052 | initial 0.17552 | pwf_PWF 0.00044 | right_gradX 0.00109 | pde 0.01123
Epoch 14/250  | lr 0.00583 | initial 0.17307 | pwf_PWF 0.00052 | right_gradX 0.00149 | pde 0.0075
Epoch 15/250  | lr 0.00644 | initial 0.17098 | pwf_PWF 0.0007 | right_gradX 0.00154 | pde 0.00717
Epoch 16/250  | lr 0.00704 | initial 0.197 | pwf_PWF 0.1088 | right_gradX 0.00157 | pde 0.01369
Epoch 17/250  | lr 0.0076 | initial 0.16858 | pwf_PWF 0.00088 | right_gradX 0.00168 | pde 0.00999
Epoch 18/250  | lr 0.00812 | initial 0.16678 | pwf_PWF 0.00098 | right_gradX 0.00172 | pde 0.00838
Epoch 19/250  | lr 0.00859 | initial 0.21402 | pwf_PWF 0.17851 | right_gradX 0.00219 | pde 0.02287
Epoch 20/250  | lr 0.00901 | initial 0.1659 | pwf_PWF 0.00131 | right_gradX 0.00165 | pde 0.01052
Epoch 21/250  | lr 0.00936 | initial 0.16492 | pwf_PWF 0.01162 | right_gradX 0.00189 | pde 0.00955
Epoch 22/250  | lr 0.00963 | initial 0.164 | pwf_PWF 0.02431 | right_gradX 0.00239 | pde 0.00971
Epoch 23/250  | lr 0.00984 | initial 0.16128 | pwf_PWF 0.01625 | right_gradX 0.00272 | pde 0.00969
Epoch 24/250  | lr 0.00996 | initial 0.16132 | pwf_PWF 0.03223 | right_gradX 0.00351 | pde 0.01178
Epoch 25/250  | lr 0.01 | initial 0.15929 | pwf_PWF 0.02211 | right_gradX 0.00303 | pde 0.00967
Epoch 26/250  | lr 0.01 | initial 0.15736 | pwf_PWF 0.017 | right_gradX 0.00326 | pde 0.00992
Epoch 27/250  | lr 0.01 | initial 0.15538 | pwf_PWF 0.01883 | right_gradX 0.00412 | pde 0.01189
Epoch 28/250  | lr 0.01 | initial 0.15286 | pwf_PWF 0.01245 | right_gradX 0.00373 | pde 0.0097
Epoch 29/250  | lr 0.00999 | initial 0.15031 | pwf_PWF 0.01702 | right_gradX 0.00435 | pde 0.01049
Epoch 30/250  | lr 0.00999 | initial 0.14901 | pwf_PWF 0.03216 | right_gradX 0.00488 | pde 0.01344
Epoch 31/250  | lr 0.00998 | initial 0.14724 | pwf_PWF 0.03192 | right_gradX 0.0048 | pde 0.01469
Epoch 32/250  | lr 0.00998 | initial 0.14037 | pwf_PWF 0.01207 | right_gradX 0.0056 | pde 0.01398
Epoch 33/250  | lr 0.00997 | initial 0.13764 | pwf_PWF 0.03604 | right_gradX 0.00517 | pde 0.01686
Epoch 34/250  | lr 0.00996 | initial 0.16231 | pwf_PWF 0.15226 | right_gradX 0.00579 | pde 0.03272
Epoch 35/250  | lr 0.00995 | initial 0.13276 | pwf_PWF 0.00549 | right_gradX 0.00441 | pde 0.01503
Epoch 36/250  | lr 0.00994 | initial 0.12596 | pwf_PWF 0.00446 | right_gradX 0.00458 | pde 0.01601
Epoch 37/250  | lr 0.00993 | initial 0.11833 | pwf_PWF 0.00309 | right_gradX 0.00417 | pde 0.01527
Epoch 38/250  | lr 0.00992 | initial 0.11394 | pwf_PWF 0.01878 | right_gradX 0.00464 | pde 0.02144
Epoch 39/250  | lr 0.0099 | initial 0.1103 | pwf_PWF 0.02646 | right_gradX 0.00396 | pde 0.02894
Epoch 40/250  | lr 0.00989 | initial 0.10685 | pwf_PWF 0.0248 | right_gradX 0.00333 | pde 0.02289
Epoch 41/250  | lr 0.00988 | initial 0.13623 | pwf_PWF 0.12675 | right_gradX 0.00516 | pde 0.04671
Epoch 42/250  | lr 0.00986 | initial 0.1146 | pwf_PWF 0.01114 | right_gradX 0.00321 | pde 0.03068
Epoch 43/250  | lr 0.00984 | initial 0.10128 | pwf_PWF 0.01007 | right_gradX 0.00278 | pde 0.02355
Epoch 44/250  | lr 0.00983 | initial 0.09542 | pwf_PWF 0.01453 | right_gradX 0.00233 | pde 0.02469
Epoch 45/250  | lr 0.00981 | initial 0.09657 | pwf_PWF 0.03549 | right_gradX 0.0021 | pde 0.03946
Epoch 46/250  | lr 0.00979 | initial 0.20719 | pwf_PWF 0.22434 | right_gradX 0.00315 | pde 0.03945
Epoch 47/250  | lr 0.00977 | initial 0.11974 | pwf_PWF 0.01029 | right_gradX 0.00298 | pde 0.03968
Epoch 48/250  | lr 0.00974 | initial 0.10238 | pwf_PWF 0.01263 | right_gradX 0.00227 | pde 0.04078
Epoch 49/250  | lr 0.00972 | initial 0.12054 | pwf_PWF 0.00865 | right_gradX 0.00151 | pde 0.02186
Epoch 50/250  | lr 0.0097 | initial 0.08841 | pwf_PWF 0.00754 | right_gradX 0.00141 | pde 0.02255
Epoch 51/250  | lr 0.00967 | initial 0.08814 | pwf_PWF 0.0169 | right_gradX 0.00145 | pde 0.02137
Epoch 52/250  | lr 0.00965 | initial 0.08579 | pwf_PWF 0.01507 | right_gradX 0.00146 | pde 0.02286
Epoch 53/250  | lr 0.00962 | initial 0.11226 | pwf_PWF 0.04188 | right_gradX 0.00268 | pde 0.04674
Epoch 54/250  | lr 0.0096 | initial 0.08304 | pwf_PWF 0.01502 | right_gradX 0.00095 | pde 0.02283
Epoch 55/250  | lr 0.00957 | initial 0.09191 | pwf_PWF 0.02476 | right_gradX 0.00146 | pde 0.02536
Epoch 56/250  | lr 0.00954 | initial 0.08406 | pwf_PWF 0.02117 | right_gradX 0.0011 | pde 0.02616
Epoch 57/250  | lr 0.00951 | initial 0.0847 | pwf_PWF 0.02479 | right_gradX 0.00136 | pde 0.03189
Epoch 58/250  | lr 0.00948 | initial 0.12115 | pwf_PWF 0.07433 | right_gradX 0.00307 | pde 0.06238
Epoch 59/250  | lr 0.00945 | initial 0.14424 | pwf_PWF 0.06825 | right_gradX 0.00132 | pde 0.04019
Epoch 60/250  | lr 0.00941 | initial 0.11308 | pwf_PWF 0.00177 | right_gradX 0.00177 | pde 0.00774
Epoch 61/250  | lr 0.00938 | initial 0.09607 | pwf_PWF 0.01236 | right_gradX 0.00087 | pde 0.02465
Epoch 62/250  | lr 0.00935 | initial 0.08926 | pwf_PWF 0.01063 | right_gradX 0.00089 | pde 0.01789
Epoch 63/250  | lr 0.00931 | initial 0.08453 | pwf_PWF 0.01615 | right_gradX 0.00074 | pde 0.02581
Epoch 64/250  | lr 0.00928 | initial 0.09818 | pwf_PWF 0.02685 | right_gradX 0.00129 | pde 0.04477
Epoch 65/250  | lr 0.00924 | initial 0.13963 | pwf_PWF 0.06031 | right_gradX 0.00154 | pde 0.04687
Epoch 66/250  | lr 0.0092 | initial 0.08219 | pwf_PWF 0.00399 | right_gradX 0.00056 | pde 0.01196
Epoch 67/250  | lr 0.00916 | initial 0.20675 | pwf_PWF 0.23545 | right_gradX 0.00331 | pde 0.07986
Epoch 68/250  | lr 0.00913 | initial 0.08989 | pwf_PWF 0.00794 | right_gradX 0.00047 | pde 0.01763
Epoch 69/250  | lr 0.00909 | initial 0.0888 | pwf_PWF 0.00962 | right_gradX 0.00053 | pde 0.02933
Epoch 70/250  | lr 0.00905 | initial 0.1961 | pwf_PWF 0.04692 | right_gradX 0.00196 | pde 0.07561
Epoch 71/250  | lr 0.009 | initial 0.14232 | pwf_PWF 0.01428 | right_gradX 0.00117 | pde 0.01137
Epoch 72/250  | lr 0.00896 | initial 0.10816 | pwf_PWF 0.01507 | right_gradX 0.00134 | pde 0.02008
Epoch 73/250  | lr 0.00892 | initial 0.088 | pwf_PWF 0.0111 | right_gradX 0.00045 | pde 0.02306
Epoch 74/250  | lr 0.00887 | initial 0.08105 | pwf_PWF 0.0095 | right_gradX 0.00055 | pde 0.01752
Epoch 75/250  | lr 0.00883 | initial 0.08078 | pwf_PWF 0.0107 | right_gradX 0.00054 | pde 0.01621
Epoch 76/250  | lr 0.00878 | initial 0.07756 | pwf_PWF 0.00779 | right_gradX 0.00053 | pde 0.02156
Epoch 77/250  | lr 0.00874 | initial 0.07884 | pwf_PWF 0.01958 | right_gradX 0.00082 | pde 0.02075
Epoch 78/250  | lr 0.00869 | initial 0.17277 | pwf_PWF 0.23968 | right_gradX 0.00097 | pde 0.05269
Epoch 79/250  | lr 0.00864 | initial 0.08689 | pwf_PWF 0.01078 | right_gradX 0.00117 | pde 0.03601
Epoch 80/250  | lr 0.0086 | initial 0.51186 | pwf_PWF 60.85821 | right_gradX 0.01286 | pde 0.57554
Epoch 81/250  | lr 0.00855 | initial 0.98623 | pwf_PWF 0.472 | right_gradX 0.00023 | pde 0.00794
Epoch 82/250  | lr 0.0085 | initial 0.98248 | pwf_PWF 0.00279 | right_gradX 5e-05 | pde 0.00281
Epoch 83/250  | lr 0.00845 | initial 0.97572 | pwf_PWF 0.00064 | right_gradX 6e-05 | pde 0.00162
Epoch 84/250  | lr 0.0084 | initial 0.96229 | pwf_PWF 0.00052 | right_gradX 0.00018 | pde 0.00191
Epoch 85/250  | lr 0.00835 | initial 0.92706 | pwf_PWF 0.00074 | right_gradX 0.00096 | pde 0.01007
Epoch 86/250  | lr 0.00829 | initial 0.87436 | pwf_PWF 0.00084 | right_gradX 0.00092 | pde 0.02078
Epoch 87/250  | lr 0.00824 | initial 0.77242 | pwf_PWF 0.00074 | right_gradX 0.00093 | pde 0.04199
Epoch 88/250  | lr 0.00819 | initial 0.61106 | pwf_PWF 0.00123 | right_gradX 0.0028 | pde 0.07662
Epoch 89/250  | lr 0.00813 | initial 0.40028 | pwf_PWF 0.00181 | right_gradX 0.00498 | pde 0.06438
Epoch 90/250  | lr 0.00808 | initial 0.19857 | pwf_PWF 0.00195 | right_gradX 0.00533 | pde 0.02752
Epoch 91/250  | lr 0.00802 | initial 0.1685 | pwf_PWF 0.00112 | right_gradX 0.00459 | pde 0.01819
Epoch 92/250  | lr 0.00797 | initial 0.16053 | pwf_PWF 0.00086 | right_gradX 0.00421 | pde 0.0139
Epoch 93/250  | lr 0.00791 | initial 0.15362 | pwf_PWF 0.00087 | right_gradX 0.00382 | pde 0.01246
Epoch 94/250  | lr 0.00785 | initial 0.14893 | pwf_PWF 0.00079 | right_gradX 0.00344 | pde 0.01117
Epoch 95/250  | lr 0.0078 | initial 0.14454 | pwf_PWF 0.00087 | right_gradX 0.00334 | pde 0.01155
Epoch 96/250  | lr 0.00774 | initial 0.14136 | pwf_PWF 0.00073 | right_gradX 0.00366 | pde 0.01077
Epoch 97/250  | lr 0.00768 | initial 0.13789 | pwf_PWF 0.0008 | right_gradX 0.00347 | pde 0.01105
Epoch 98/250  | lr 0.00762 | initial 0.1351 | pwf_PWF 0.00074 | right_gradX 0.00367 | pde 0.01065
Epoch 99/250  | lr 0.00756 | initial 0.13226 | pwf_PWF 0.00105 | right_gradX 0.00477 | pde 0.01243
Epoch 100/250  | lr 0.0075 | initial 0.12977 | pwf_PWF 0.00079 | right_gradX 0.00388 | pde 0.01055
Epoch 101/250  | lr 0.00744 | initial 0.12657 | pwf_PWF 0.0012 | right_gradX 0.00386 | pde 0.01157
Epoch 102/250  | lr 0.00738 | initial 0.12473 | pwf_PWF 0.00121 | right_gradX 0.00441 | pde 0.01179
Epoch 103/250  | lr 0.00732 | initial 0.12151 | pwf_PWF 0.00165 | right_gradX 0.00393 | pde 0.01206
Epoch 104/250  | lr 0.00725 | initial 0.12001 | pwf_PWF 0.00207 | right_gradX 0.00374 | pde 0.0122
Epoch 105/250  | lr 0.00719 | initial 0.11745 | pwf_PWF 0.00233 | right_gradX 0.00363 | pde 0.01127
Epoch 106/250  | lr 0.00713 | initial 0.11503 | pwf_PWF 0.00192 | right_gradX 0.00357 | pde 0.0122
Epoch 107/250  | lr 0.00707 | initial 0.11297 | pwf_PWF 0.00246 | right_gradX 0.0035 | pde 0.01129
Epoch 108/250  | lr 0.007 | initial 0.1104 | pwf_PWF 0.00285 | right_gradX 0.0038 | pde 0.01363
Epoch 109/250  | lr 0.00694 | initial 0.10882 | pwf_PWF 0.00222 | right_gradX 0.00378 | pde 0.01196
Epoch 110/250  | lr 0.00687 | initial 0.10645 | pwf_PWF 0.00318 | right_gradX 0.00337 | pde 0.01339
Epoch 111/250  | lr 0.00681 | initial 0.10399 | pwf_PWF 0.00404 | right_gradX 0.00304 | pde 0.01439
Epoch 112/250  | lr 0.00674 | initial 0.10327 | pwf_PWF 0.0035 | right_gradX 0.00305 | pde 0.01203
Epoch 113/250  | lr 0.00668 | initial 0.10102 | pwf_PWF 0.00396 | right_gradX 0.00248 | pde 0.01275
Epoch 114/250  | lr 0.00661 | initial 0.09898 | pwf_PWF 0.00538 | right_gradX 0.00229 | pde 0.01441
Epoch 115/250  | lr 0.00655 | initial 0.09922 | pwf_PWF 0.00502 | right_gradX 0.00314 | pde 0.02497
Epoch 116/250  | lr 0.00648 | initial 0.09699 | pwf_PWF 0.00398 | right_gradX 0.00282 | pde 0.01396
Epoch 117/250  | lr 0.00641 | initial 0.09527 | pwf_PWF 0.00421 | right_gradX 0.00185 | pde 0.01288
Epoch 118/250  | lr 0.00634 | initial 0.09444 | pwf_PWF 0.00355 | right_gradX 0.00155 | pde 0.01202
Epoch 119/250  | lr 0.00628 | initial 0.09243 | pwf_PWF 0.00551 | right_gradX 0.00121 | pde 0.01159
Epoch 120/250  | lr 0.00621 | initial 0.09241 | pwf_PWF 0.00606 | right_gradX 0.00156 | pde 0.01387
Epoch 121/250  | lr 0.00614 | initial 0.09121 | pwf_PWF 0.00516 | right_gradX 0.00246 | pde 0.01631
Epoch 122/250  | lr 0.00607 | initial 0.09053 | pwf_PWF 0.00517 | right_gradX 0.00128 | pde 0.01249
Epoch 123/250  | lr 0.00601 | initial 0.08878 | pwf_PWF 0.00466 | right_gradX 0.00074 | pde 0.01233
Epoch 124/250  | lr 0.00594 | initial 0.08932 | pwf_PWF 0.0084 | right_gradX 0.00233 | pde 0.01931
Epoch 125/250  | lr 0.00587 | initial 0.08717 | pwf_PWF 0.00448 | right_gradX 0.00074 | pde 0.01113
Epoch 126/250  | lr 0.0058 | initial 0.08702 | pwf_PWF 0.00543 | right_gradX 0.00072 | pde 0.01509
Epoch 127/250  | lr 0.00573 | initial 0.0863 | pwf_PWF 0.00536 | right_gradX 0.00094 | pde 0.01094
Epoch 128/250  | lr 0.00566 | initial 0.08907 | pwf_PWF 0.01096 | right_gradX 0.00311 | pde 0.02834
Epoch 129/250  | lr 0.00559 | initial 0.08605 | pwf_PWF 0.00459 | right_gradX 0.00075 | pde 0.01431
Epoch 130/250  | lr 0.00552 | initial 0.08495 | pwf_PWF 0.00411 | right_gradX 0.00096 | pde 0.01179
Epoch 131/250  | lr 0.00545 | initial 0.084 | pwf_PWF 0.00486 | right_gradX 0.00053 | pde 0.00962
Epoch 132/250  | lr 0.00538 | initial 0.08587 | pwf_PWF 0.00832 | right_gradX 0.00226 | pde 0.03027
Epoch 133/250  | lr 0.00531 | initial 0.08374 | pwf_PWF 0.00531 | right_gradX 0.00089 | pde 0.01323
Epoch 134/250  | lr 0.00524 | initial 0.08322 | pwf_PWF 0.00497 | right_gradX 0.00089 | pde 0.01446
Epoch 135/250  | lr 0.00517 | initial 0.08689 | pwf_PWF 0.01126 | right_gradX 0.00274 | pde 0.03399
Epoch 136/250  | lr 0.0051 | initial 0.08459 | pwf_PWF 0.00761 | right_gradX 0.00142 | pde 0.02246
Epoch 137/250  | lr 0.00503 | initial 0.08204 | pwf_PWF 0.00365 | right_gradX 0.0006 | pde 0.01064
Epoch 138/250  | lr 0.00497 | initial 0.08122 | pwf_PWF 0.00626 | right_gradX 0.00053 | pde 0.01008
Epoch 139/250  | lr 0.0049 | initial 0.08453 | pwf_PWF 0.00882 | right_gradX 0.00193 | pde 0.0237
Epoch 140/250  | lr 0.00483 | initial 0.08549 | pwf_PWF 0.00678 | right_gradX 0.00238 | pde 0.03628
Epoch 141/250  | lr 0.00476 | initial 0.08548 | pwf_PWF 0.02733 | right_gradX 0.00115 | pde 0.02618
Epoch 142/250  | lr 0.00469 | initial 0.08076 | pwf_PWF 0.00334 | right_gradX 0.00059 | pde 0.01154
Epoch 143/250  | lr 0.00462 | initial 0.08114 | pwf_PWF 0.00509 | right_gradX 0.00101 | pde 0.01388
Epoch 144/250  | lr 0.00455 | initial 0.08038 | pwf_PWF 0.00727 | right_gradX 0.00111 | pde 0.01778
Epoch 145/250  | lr 0.00448 | initial 0.0786 | pwf_PWF 0.00371 | right_gradX 0.0006 | pde 0.00886
Epoch 146/250  | lr 0.00441 | initial 0.07937 | pwf_PWF 0.00627 | right_gradX 0.0014 | pde 0.02231
Epoch 147/250  | lr 0.00434 | initial 0.07818 | pwf_PWF 0.00385 | right_gradX 0.00063 | pde 0.00881
Epoch 148/250  | lr 0.00427 | initial 0.08647 | pwf_PWF 0.01939 | right_gradX 0.0028 | pde 0.04685
Epoch 149/250  | lr 0.0042 | initial 0.08577 | pwf_PWF 0.00759 | right_gradX 0.00107 | pde 0.01423
Epoch 150/250  | lr 0.00413 | initial 0.07773 | pwf_PWF 0.0069 | right_gradX 0.00103 | pde 0.01133
Epoch 151/250  | lr 0.00406 | initial 0.07621 | pwf_PWF 0.00385 | right_gradX 0.00073 | pde 0.01014
Epoch 152/250  | lr 0.00399 | initial 0.07778 | pwf_PWF 0.01033 | right_gradX 0.0017 | pde 0.01908
Epoch 153/250  | lr 0.00393 | initial 0.07544 | pwf_PWF 0.00443 | right_gradX 0.00159 | pde 0.01342
Epoch 154/250  | lr 0.00386 | initial 0.07493 | pwf_PWF 0.00856 | right_gradX 0.00109 | pde 0.01881
Epoch 155/250  | lr 0.00379 | initial 0.07474 | pwf_PWF 0.00482 | right_gradX 0.00204 | pde 0.01958
Epoch 156/250  | lr 0.00372 | initial 0.07505 | pwf_PWF 0.00477 | right_gradX 0.00122 | pde 0.01369
Epoch 157/250  | lr 0.00366 | initial 0.07442 | pwf_PWF 0.00587 | right_gradX 0.00106 | pde 0.01375
Epoch 158/250  | lr 0.00359 | initial 0.07658 | pwf_PWF 0.01078 | right_gradX 0.00154 | pde 0.02818
Epoch 159/250  | lr 0.00352 | initial 0.07474 | pwf_PWF 0.0079 | right_gradX 0.00113 | pde 0.01827
Epoch 160/250  | lr 0.00345 | initial 0.07501 | pwf_PWF 0.00731 | right_gradX 0.00132 | pde 0.02025
Epoch 161/250  | lr 0.00339 | initial 0.07419 | pwf_PWF 0.0043 | right_gradX 0.00123 | pde 0.01501
Epoch 162/250  | lr 0.00332 | initial 0.07261 | pwf_PWF 0.00701 | right_gradX 0.00115 | pde 0.01623
Epoch 163/250  | lr 0.00326 | initial 0.07142 | pwf_PWF 0.00621 | right_gradX 0.00088 | pde 0.0142
Epoch 164/250  | lr 0.00319 | initial 0.06973 | pwf_PWF 0.00195 | right_gradX 0.00085 | pde 0.0083
Epoch 165/250  | lr 0.00313 | initial 0.06961 | pwf_PWF 0.0081 | right_gradX 0.00108 | pde 0.01095
Epoch 166/250  | lr 0.00306 | initial 0.06975 | pwf_PWF 0.00455 | right_gradX 0.00115 | pde 0.01321
Epoch 167/250  | lr 0.003 | initial 0.0684 | pwf_PWF 0.00432 | right_gradX 0.00094 | pde 0.01033
Epoch 168/250  | lr 0.00293 | initial 0.0707 | pwf_PWF 0.00943 | right_gradX 0.00146 | pde 0.02274
Epoch 169/250  | lr 0.00287 | initial 0.0725 | pwf_PWF 0.00825 | right_gradX 0.00125 | pde 0.01768
Epoch 170/250  | lr 0.00281 | initial 0.068 | pwf_PWF 0.00554 | right_gradX 0.00095 | pde 0.01893
Epoch 171/250  | lr 0.00275 | initial 0.07002 | pwf_PWF 0.00713 | right_gradX 0.00129 | pde 0.02452
Epoch 172/250  | lr 0.00268 | initial 0.07328 | pwf_PWF 0.01068 | right_gradX 0.00181 | pde 0.03872
Epoch 173/250  | lr 0.00262 | initial 0.07056 | pwf_PWF 0.00418 | right_gradX 0.00145 | pde 0.01507
Epoch 174/250  | lr 0.00256 | initial 0.06704 | pwf_PWF 0.00426 | right_gradX 0.00081 | pde 0.01818
Epoch 175/250  | lr 0.0025 | initial 0.0685 | pwf_PWF 0.00856 | right_gradX 0.00125 | pde 0.01459
Epoch 176/250  | lr 0.00244 | initial 0.07241 | pwf_PWF 0.00891 | right_gradX 0.00135 | pde 0.02712
Epoch 177/250  | lr 0.00238 | initial 0.06582 | pwf_PWF 0.00462 | right_gradX 0.0006 | pde 0.0089
Epoch 178/250  | lr 0.00232 | initial 0.0671 | pwf_PWF 0.00639 | right_gradX 0.00075 | pde 0.01917
Epoch 179/250  | lr 0.00226 | initial 0.06618 | pwf_PWF 0.00393 | right_gradX 0.00084 | pde 0.01489
Epoch 180/250  | lr 0.0022 | initial 0.065 | pwf_PWF 0.00522 | right_gradX 0.00076 | pde 0.0139
Epoch 181/250  | lr 0.00215 | initial 0.0638 | pwf_PWF 0.00253 | right_gradX 0.00058 | pde 0.00817
Epoch 182/250  | lr 0.00209 | initial 0.06306 | pwf_PWF 0.00438 | right_gradX 0.00065 | pde 0.012
Epoch 183/250  | lr 0.00203 | initial 0.06405 | pwf_PWF 0.00464 | right_gradX 0.00065 | pde 0.02006
Epoch 184/250  | lr 0.00198 | initial 0.06342 | pwf_PWF 0.00421 | right_gradX 0.00078 | pde 0.01056
Epoch 185/250  | lr 0.00192 | initial 0.06782 | pwf_PWF 0.00947 | right_gradX 0.00118 | pde 0.02693
Epoch 186/250  | lr 0.00187 | initial 0.06284 | pwf_PWF 0.00276 | right_gradX 0.00046 | pde 0.00731
Epoch 187/250  | lr 0.00181 | initial 0.0606 | pwf_PWF 0.0015 | right_gradX 0.00045 | pde 0.005
Epoch 188/250  | lr 0.00176 | initial 0.06075 | pwf_PWF 0.00204 | right_gradX 0.00062 | pde 0.0069
Epoch 189/250  | lr 0.00171 | initial 0.06299 | pwf_PWF 0.00415 | right_gradX 0.00108 | pde 0.02266
Epoch 190/250  | lr 0.00165 | initial 0.06128 | pwf_PWF 0.00441 | right_gradX 0.00075 | pde 0.01711
Epoch 191/250  | lr 0.0016 | initial 0.06259 | pwf_PWF 0.00421 | right_gradX 0.00115 | pde 0.01791
Epoch 192/250  | lr 0.00155 | initial 0.06198 | pwf_PWF 0.00439 | right_gradX 0.00102 | pde 0.01235
Epoch 193/250  | lr 0.0015 | initial 0.05988 | pwf_PWF 0.00212 | right_gradX 0.00091 | pde 0.01304
Epoch 194/250  | lr 0.00145 | initial 0.06174 | pwf_PWF 0.00476 | right_gradX 0.00099 | pde 0.01354
Epoch 195/250  | lr 0.0014 | initial 0.06006 | pwf_PWF 0.00349 | right_gradX 0.00063 | pde 0.01703
Epoch 196/250  | lr 0.00136 | initial 0.06126 | pwf_PWF 0.00395 | right_gradX 0.00132 | pde 0.01318
Epoch 197/250  | lr 0.00131 | initial 0.05839 | pwf_PWF 0.00173 | right_gradX 0.00048 | pde 0.00604
Epoch 198/250  | lr 0.00126 | initial 0.05767 | pwf_PWF 0.00148 | right_gradX 0.00049 | pde 0.00521
Epoch 199/250  | lr 0.00122 | initial 0.05685 | pwf_PWF 0.0017 | right_gradX 0.00043 | pde 0.00576
Epoch 200/250  | lr 0.00117 | initial 0.05689 | pwf_PWF 0.00175 | right_gradX 0.00033 | pde 0.00542
Epoch 201/250  | lr 0.00113 | initial 0.05664 | pwf_PWF 0.00161 | right_gradX 0.00044 | pde 0.00561
Epoch 202/250  | lr 0.00108 | initial 0.05613 | pwf_PWF 0.00182 | right_gradX 0.00036 | pde 0.00482
Epoch 203/250  | lr 0.00104 | initial 0.05535 | pwf_PWF 0.00146 | right_gradX 0.00032 | pde 0.0047
Epoch 204/250  | lr 0.001 | initial 0.05642 | pwf_PWF 0.00535 | right_gradX 0.00054 | pde 0.0149
Epoch 205/250  | lr 0.00095 | initial 0.05598 | pwf_PWF 0.00568 | right_gradX 0.00093 | pde 0.00953
Epoch 206/250  | lr 0.00091 | initial 0.05626 | pwf_PWF 0.00384 | right_gradX 0.00033 | pde 0.01775
Epoch 207/250  | lr 0.00087 | initial 0.05989 | pwf_PWF 0.00467 | right_gradX 0.00068 | pde 0.02026
Epoch 208/250  | lr 0.00084 | initial 0.05532 | pwf_PWF 0.0015 | right_gradX 0.00033 | pde 0.00483
Epoch 209/250  | lr 0.0008 | initial 0.05484 | pwf_PWF 0.00156 | right_gradX 0.00062 | pde 0.00928
Epoch 210/250  | lr 0.00076 | initial 0.05439 | pwf_PWF 0.00182 | right_gradX 0.00038 | pde 0.0074
Epoch 211/250  | lr 0.00072 | initial 0.05458 | pwf_PWF 0.00253 | right_gradX 0.00043 | pde 0.00555
Epoch 212/250  | lr 0.00069 | initial 0.05369 | pwf_PWF 0.00114 | right_gradX 0.00024 | pde 0.00397
Epoch 213/250  | lr 0.00065 | initial 0.05406 | pwf_PWF 0.00271 | right_gradX 0.00062 | pde 0.01169
Epoch 214/250  | lr 0.00062 | initial 0.0548 | pwf_PWF 0.00323 | right_gradX 0.00047 | pde 0.01319
Epoch 215/250  | lr 0.00059 | initial 0.0533 | pwf_PWF 0.00112 | right_gradX 0.00026 | pde 0.004
Epoch 216/250  | lr 0.00055 | initial 0.05284 | pwf_PWF 0.00095 | right_gradX 0.00024 | pde 0.00311
Epoch 217/250  | lr 0.00052 | initial 0.05247 | pwf_PWF 0.00079 | right_gradX 0.00025 | pde 0.00317
Epoch 218/250  | lr 0.00049 | initial 0.05252 | pwf_PWF 0.00083 | right_gradX 0.00022 | pde 0.00292
Epoch 219/250  | lr 0.00046 | initial 0.05251 | pwf_PWF 0.00086 | right_gradX 0.0003 | pde 0.0034
Epoch 220/250  | lr 0.00043 | initial 0.05212 | pwf_PWF 0.00117 | right_gradX 0.00023 | pde 0.00389
Epoch 221/250  | lr 0.0004 | initial 0.05207 | pwf_PWF 0.00096 | right_gradX 0.00019 | pde 0.00326
Epoch 222/250  | lr 0.00038 | initial 0.05151 | pwf_PWF 0.00087 | right_gradX 0.00021 | pde 0.00343
Epoch 223/250  | lr 0.00035 | initial 0.0514 | pwf_PWF 0.00083 | right_gradX 0.00021 | pde 0.00301
Epoch 224/250  | lr 0.00033 | initial 0.05124 | pwf_PWF 0.00127 | right_gradX 0.00019 | pde 0.0033
Epoch 225/250  | lr 0.0003 | initial 0.05125 | pwf_PWF 0.00082 | right_gradX 0.0002 | pde 0.00297
Epoch 226/250  | lr 0.00028 | initial 0.05096 | pwf_PWF 0.00099 | right_gradX 0.00019 | pde 0.00389
Epoch 227/250  | lr 0.00026 | initial 0.05115 | pwf_PWF 0.00103 | right_gradX 0.00025 | pde 0.004
Epoch 228/250  | lr 0.00023 | initial 0.05076 | pwf_PWF 0.00083 | right_gradX 0.0002 | pde 0.00294
Epoch 229/250  | lr 0.00021 | initial 0.05053 | pwf_PWF 0.00074 | right_gradX 0.00016 | pde 0.00285
Epoch 230/250  | lr 0.00019 | initial 0.05048 | pwf_PWF 0.00085 | right_gradX 0.00017 | pde 0.0028
Epoch 231/250  | lr 0.00017 | initial 0.05041 | pwf_PWF 0.00069 | right_gradX 0.00015 | pde 0.00233
Epoch 232/250  | lr 0.00016 | initial 0.05025 | pwf_PWF 0.00073 | right_gradX 0.00015 | pde 0.00227
Epoch 233/250  | lr 0.00014 | initial 0.05012 | pwf_PWF 0.00073 | right_gradX 0.00015 | pde 0.00226
Epoch 234/250  | lr 0.00012 | initial 0.05008 | pwf_PWF 0.00067 | right_gradX 0.00014 | pde 0.00233
Epoch 235/250  | lr 0.00011 | initial 0.05021 | pwf_PWF 0.00079 | right_gradX 0.00016 | pde 0.0025
Epoch 236/250  | lr 0.0001 | initial 0.04996 | pwf_PWF 0.00073 | right_gradX 0.00014 | pde 0.00215
Epoch 237/250  | lr 8e-05 | initial 0.0498 | pwf_PWF 0.00073 | right_gradX 0.00013 | pde 0.00246
Epoch 238/250  | lr 7e-05 | initial 0.04969 | pwf_PWF 0.00067 | right_gradX 0.00013 | pde 0.00235
Epoch 239/250  | lr 6e-05 | initial 0.04986 | pwf_PWF 0.00068 | right_gradX 0.00013 | pde 0.002
Epoch 240/250  | lr 5e-05 | initial 0.04986 | pwf_PWF 0.00064 | right_gradX 0.00013 | pde 0.00202
Epoch 241/250  | lr 4e-05 | initial 0.04971 | pwf_PWF 0.00064 | right_gradX 0.00013 | pde 0.00222
Epoch 242/250  | lr 3e-05 | initial 0.04965 | pwf_PWF 0.00067 | right_gradX 0.00013 | pde 0.00198
Epoch 243/250  | lr 2e-05 | initial 0.0496 | pwf_PWF 0.00065 | right_gradX 0.00012 | pde 0.00211
Epoch 244/250  | lr 2e-05 | initial 0.04954 | pwf_PWF 0.00064 | right_gradX 0.00013 | pde 0.00206
Epoch 245/250  | lr 1e-05 | initial 0.04963 | pwf_PWF 0.00063 | right_gradX 0.00012 | pde 0.0019
Epoch 246/250  | lr 1e-05 | initial 0.0496 | pwf_PWF 0.00063 | right_gradX 0.00012 | pde 0.00189
Epoch 247/250  | lr 0.0 | initial 0.04959 | pwf_PWF 0.00063 | right_gradX 0.00012 | pde 0.00186
Epoch 248/250  | lr 0.0 | initial 0.04959 | pwf_PWF 0.00063 | right_gradX 0.00012 | pde 0.00185
Epoch 249/250  | lr 0.0 | initial 0.04958 | pwf_PWF 0.00062 | right_gradX 0.00012 | pde 0.00185
Epoch 250/250  | lr 0.0 | initial 0.04958 | pwf_PWF 0.00062 | right_gradX 0.00012 | pde 0.00186

    CPU times: user 6min 51s, sys: 17.6 s, total: 7min 9s
    Wall time: 7min 2s

```python
t = 0.0
x = np.linspace(0,1,100)
#y = np.linspace(0,1,100)

_x, _t = np.meshgrid(x, t)
eval_mesh = Mesh({'x': x,  't':t}, device=device)
p = pde.eval(eval_mesh)
p = p.cpu().numpy()
#p5 = p.reshape((_y.shape[0],_x.shape[1]))
```

```python
ppp=[]
x = np.linspace(0.0,1,100)
for t in np.linspace(0.1,1.,11):
  #_x,_t = np.meshgrid(x,t)
  eval_mesh = Mesh({'x': x,  't':t}, device=device)
  pp = pde.eval(eval_mesh)
  pp = pp.cpu().numpy()
  ppp.append(pp)
```

```python
for p in ppp:
  plt.plot(p)
```

```python
from sympy import Sum, factorial, oo, IndexedBase, Function, sin,exp
from sympy.abc import n
```

```python
def diff_sol(x,t,alpha=0.1):
    sol= Sum(4/((2*n + 1)*np.pi)*sin((2*n+1)*np.pi/2*x)*exp(-((2*n + 1)*np.pi/2)**2*t*alpha),(n,0,50))

    return sol #,sol_n
```

```python
xxx=np.linspace(0,1,100)
for t in np.linspace(0.1,1,11):
    d=[]
    for xx in xxx:
        d.append(diff_sol(xx,t,alpha=1).doit())
    plt.plot(d)
    eval_mesh = Mesh({'x': x,  't':t}, device=device)
    pp = pde.eval(eval_mesh)
    pp = pp.cpu().numpy()
    plt.plot(p)
```

```python
xxx=np.linspace(0,1,10)
x = np.linspace(0,1,10)
for t in np.linspace(0.02,1,11):
    d=[]
    for xx in xxx:
        d.append(diff_sol(xx,t,alpha=1).doit())
    plt.plot(d)
    eval_mesh = Mesh({'x': x,  't':t}, device=device)
    pp = pde.eval(eval_mesh)
    pp = pp.cpu().numpy()
    plt.plot(pp,marker='o',linestyle='None')
```

```python
plt.plot(p,marker='.')
```

```python

```


```python
import holoviews as hv
hv.extension('bokeh')
hv.opts.defaults(hv.opts.Curve(width=500),
                 hv.opts.Scatter(width=500, size=4),
                 hv.opts.Histogram(width=500),
                 hv.opts.Slope(color='k', alpha=0.5, line_dash='dashed'),
                 hv.opts.HLine(color='k', alpha=0.5, line_dash='dashed'))
```

```python
import numpy as np
import pandas as pd
import scipy.stats
import statsmodels.api as sm
```

# Multivariate linear regression

In the previous lesson we introduce the topic of linear regression and studied the most simple linear model: the line.

In this lesson we will generalize this model to the multivariate case, i.e. when we want to predict an unidimensional (and continuous) variable $Y$ from a multidimensional (and continuous) variable $X$. You can interpret $X$ as a table where each column represents a particular attribute.

:::{admonition} Example
:class: tip

We want to predict a car's $Y=[\text{fuel consumption}]$ using its $X=[\text{weight}; \text{number of cylinders}; \text{average speed}; \ldots]$

:::

In what follows we will learn the mathematical formalism of the Ordinary Least Squares (OLS) method and how to implement it to fit regression models using Python

## Ordinary Least Squares (OLS)

### Mathematical derivation

Consider a dataset $\{x_i, y_i\}_{i=1,\ldots,N}$ of *i.i.d.* observations with $y_i \in \mathbb{R}$ and $x_i \in \mathbb{R}^D$, with $D>1$. We want to find $\theta$  such that

$$
y_i \approx \theta_0 + \sum_{j=1}^D \theta_j x_{ij}, \quad \forall i
$$

As before we start by writing the sum of squared errors (residuals)

$$
\min_\theta L = \sum_{i=1}^N (y_i - \theta_0 - \sum_{j=1}^D \theta_j x_{ij})^2
$$

but in this case we will express it in matrix form

$$
\min_\theta  L = \| Y - X \theta \|^2 = (Y - X \theta)^T (Y - X \theta)
$$

where

$$
X = \begin{pmatrix} 1 & x_{11} & x_{12} & \ldots & x_{1D} \\
1 & x_{21} & x_{22} & \ldots & x_{2D} \\
1 & \vdots & \vdots & \ddots & \vdots \\
1 & x_{N1} & x_{N2} & \ldots & x_{ND} \end{pmatrix},  Y = \begin{pmatrix} y_1 \\ y_2 \\ \vdots \\ y_N \end{pmatrix}, \theta =  \begin{pmatrix} \theta_0 \\ \theta_1 \\ \vdots \\ \theta_D \end{pmatrix}
$$

From here we can do

$$
\frac{dL}{d\theta} = -(Y - X \theta)^T X =  -X^T (Y - X \theta) = 0
$$

to obtain the **normal equations**

$$
X^T X \theta  = X^T Y
$$

whose solution is

$$
\hat \theta = (X^T X)^{-1} X^T Y
$$

which is known as the **least squares (LS) estimator** of $\theta$

:::{dropdown} Relation with the Moore-Penrose inverse

Matrix $X^{\dagger} = (X^T X)^{-1} X^T $ is known as the left [*Moore-Penrose*](https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse) pseudo-inverse. There is also the right pseudo inverse $X^T (X X^T)^{-1}$. Together they act  as a generalization of the inverse for non-squared matrices. Further note that if $X$ is squared and invertible then $X^{\dagger} = (X^T X)^{-1} X^T  = X^{-1} (X^T)^{-1} X^T = X^{-1}$

:::

:::{warning}

The OLS solution is only valid if $A=X^T X$ is invertible (non-singular). By construction $A \in \mathbb{R}^{D\times D}$ is a squared symmetric matrix. For $A$ to be invertible we require that its determinant is not zero or equivalently

- The rank of $A$, i.e. the number of linearly independent rows or columns, is equal to $D$
- The eigenvalues/singular values of $A$ are positive

:::

:::{note}

The solution we found for the univariate case in the previous lesson is a particular case of the OLS solution

:::

:::{dropdown} Proof

The solution for the univariate case was

$$
\begin{pmatrix} N & \sum_i x_i \\ \sum_i x_i & \sum_i x_i^2\\\end{pmatrix} \begin{pmatrix} \theta_0 \\ \theta_1 \end{pmatrix}  = \begin{pmatrix} \sum_i y_i \\ \sum_i x_i y_i \end{pmatrix}
$$

which can be rewritten as

$$
\begin{align}
\begin{pmatrix} 1 & 1 & \ldots & 1 \\ x_1 & x_2 & \ldots & x_N \end{pmatrix}
\begin{pmatrix} 1 & x_1 \\ 1 & x_2 \\ \vdots & \vdots \\ 1 & x_N \end{pmatrix}
\begin{pmatrix} \theta_0 \\ \theta_1 \end{pmatrix}  &=
\begin{pmatrix} 1 & 1 & \ldots & 1 \\ x_1 & x_2 & \ldots & x_N \end{pmatrix}
\begin{pmatrix} y_1 \\ y_2 \\ \vdots \\ y_N \end{pmatrix} \nonumber \\
X^T X \theta &= X^T Y \nonumber
\end{align}
$$

:::

### Fitting an hyperplane using `numpy`

The [`linalg`](https://numpy.org/doc/stable/reference/routines.linalg.html) submodule of the `numpy` library provides

```python
np.linalg.lstsq(X, # a (N, D) shaped ndarray
                Y, # a (N, ) shaped ndarray
                rcond='warn' # See note below
               )
```

which returns

- The OLS solution: $\hat \theta = (X^T X)^{-1} X^T Y$
- The sum of squared residuals
- The rank of matrix $X$
- The singular values of matrix $X$

:::{note}

For a near-singular $A=X^T X$ we might not be able to obtain the solution using numerical methods. Conditioning can help stabilize the solution. Singular values smaller than $\epsilon$ can be cut-off by setting `rcond=epsilon` when calling `lstsq`

:::

Let's test `lstsq` on the following database of ice-cream consumption from

```python
df = pd.read_csv('data/ice_cream.csv', header=0, index_col=0)
df.columns = ['Consumption', 'Income', 'Price', 'Temperature']
display(df.head())
```

The `corr` attribute of the `pandas` dataframe returns the pairwise correlations between the variables

```python
display(df.corr())
```

Observations:

- Temperature has a high positive correlation with consumption
- Price has a low negative correlation with consumption
- Income has an almost null correlation with consumption

Let's train a multivariate linear regressor for ice-cream consumption as a function of the other variables

```python
Y = df["Consumption"].values
X = df[["Income", "Price", "Temperature"]].values
```

- We will standardize the independent variables so that their scale is the same
- We will incorporate a column with ones to model the intercept ($\theta_0$) of the hyperplane

```python
X = (X - np.mean(X, axis=0, keepdims=True))/np.std(X, axis=0, keepdims=True)
X = np.concatenate((np.ones(shape=(X.shape[0], 1)), X), axis=1)

theta, mse, rank, singvals = np.linalg.lstsq(X, Y, rcond=None)
hatY = np.dot(X, theta) # Predicted Y
```

To assess the quality of the fitted model we can visualize the predicted consumption versus actual (real) consumption or the residuals as a function of the latter and/or the independent variables

```python
p1 = hv.Scatter((Y, hatY), 'Real', 'Predicted').opts(width=330) * hv.Slope(slope=1, y_intercept=0)
p2 = hv.Scatter((Y, Y - hatY), 'Real', 'Residuals').opts(width=330) * hv.HLine(0)
hv.Layout([p1, p2]).cols(2)
```

```python
p = []
for var_name in ["Income", "Price", "Temperature"]:
    p.append(hv.Scatter((df[var_name].values, Y - hatY), var_name, 'Residuals').opts(width=330) * hv.HLine(0))
hv.Layout(p).cols(3).opts(hv.opts.Scatter(width=280, height=250))
```

The predicted consumption follows the real consumption closely. There is also no apparent correlation in the residuals.

But some important questions remain

:::{important}

- How significant is the contribution of each of the independent variables to the prediction?
- How to measure in a quantitative way the quality of the fitted model?

:::

For this we need to view OLS from an statistical perspective

## Statistical perspective of OLS

Up to now we have viewed regression from a deterministic (optimization) perspective. To understand its properties and perform inference we seek an statistical interpretation.

Let's say that we have $\{x_i, y_i\}_{i=1,\ldots,N}$ *i.i.d.* observations from an unidimensional target variable $Y$ and a **D-dimensional** independent variable $X$. We will assume that our measurements of $Y$ consists of the **true model** plus **white Gaussian noise**, *i.e.*

$$
\begin{align}
y_i &= f_\theta(x_i) + \varepsilon_i \nonumber \\
&= \theta_0 + \sum_{j=1}^D \theta_j x_{ij} + \varepsilon_i
\end{align}
$$

where $\varepsilon_i \sim \mathcal{N}(0, \sigma^2)$. Then the log likelihood of $\theta$ is

$$
\begin{align}
\log L(\theta) &= \log \prod_{i=1}^N \mathcal{N}(y_i | f_\theta(x_i), \sigma^2) \nonumber \\
&= \sum_{i=1}^N \log \mathcal{N}(y_i | f_\theta(x_i), \sigma^2) \nonumber \\
&= -\frac{N}{2} \log(2\pi \sigma^2) - \frac{1}{2\sigma^2} \sum_{i=1}^N (y_i - f_\theta(x_i))^2\nonumber \\
&= -\frac{N}{2} \log(2\pi \sigma^2) - \frac{1}{2\sigma^2} (Y-X\theta)^T (Y - X\theta), \nonumber
\end{align}
$$

and the maximum likelihood solution for $\theta$ can by obtained from

$$
\max_\theta \log L(\theta) = - \frac{1}{2\sigma^2} (Y-X\theta)^T (Y - X\theta).
$$

Note that this is equivalent to

$$
\min_\theta \log L(\theta) =  \frac{1}{2\sigma^2} (Y-X\theta)^T (Y - X\theta),
$$

which yields

$$
\hat \theta = (X^T X)^{-1} X^T Y
$$

:::{important}

The least squares solution is equivalent to the maximum likelihood solution under iid samples and gaussian noise

:::

### Statistical properties of the OLS solution

Let $\varepsilon = (\varepsilon_1, \varepsilon_2, \ldots, \varepsilon_N)$, where $\varepsilon \sim \mathcal{N}(0, I \sigma^2) \quad \forall i$

Is the OLS estimator unbiased?

$$
\begin{align}
\mathbb{E}[\hat \theta] &= \mathbb{E}[(X^T X)^{-1} X^T Y] \nonumber \\
&= \mathbb{E}[(X^T X)^{-1} X^T (X \theta + \varepsilon)] \nonumber \\
&= \mathbb{E}[\theta] + (X^T X)^{-1} X^T \mathbb{E}[\varepsilon] \\
& = \mathbb{E}[\theta]
\end{align}
$$

> YES!

What is the variance of the estimator?

$$
\begin{align}
\mathbb{E}[(\hat \theta - \mathbb{E}[\hat\theta])(\hat \theta - \mathbb{E}[\hat\theta])^T] &= \mathbb{E}[((X^T X)^{-1} X^T \varepsilon) ((X^T X)^{-1} X^T \varepsilon)^T] \nonumber \\
&= (X^T X)^{-1} X^T  \mathbb{E}[\varepsilon \varepsilon^T] X ((X^T X)^{-1})^T  \nonumber \\
&= (X^T X)^{-1} X^T  \mathbb{E}[(\varepsilon-0) (\varepsilon-0)^T] X (X^T X)^{-1}  \nonumber \\
& =\sigma^2 (X^T X)^{-1}
\end{align}
$$

and typically we estimate the variance of the noise using the unbiased estimator

$$
\begin{align}
\hat \sigma^2 &= \frac{1}{N-D-1} \sum_{i=1}^N (y_i - \theta_0 - \sum_{j=1}^D \theta_j x_{ij})^2 \nonumber \\
& = \frac{1}{N-D-1} (Y-X\theta)^T (Y-X\theta)
\end{align}
$$

**The Gauss-Markov Theorem:** The least squares estimate of $\theta$ have the smallest variance among all unbiased estimators (Hastie, 3.2.2)

### Inference and hypothesis tests for OLS

We found the expected value and the variance of $\theta$. From the properties of MLE we know that

$$
\hat \theta \sim \mathcal{N}(\theta, \sigma^2 (X^T X)^{-1})
$$

and the estimator of the variance will be proportional to

$$
\hat \sigma^2 \sim  \frac{1}{(N-M)}\sigma^2 \chi_{N-M}^2
$$

With this we have all the ingredients to find confidence intervals and do hypothesis test on $\hat \theta$

To assess the significance of our model we might try to reject the following *hypotheses*

- One of the parameters (slopes) is zero (t-test)

    $\mathcal{H}_0: \theta_i = 0$

    $\mathcal{H}_A: \theta_i \neq 0$

- All parameters are zero (f-test)

    $\mathcal{H}_0: \theta_1 = \theta_2 = \ldots = \theta_D = 0$

    $\mathcal{H}_A:$ At least one parameter is not zero

- A subset of the parameters are zero (ANOVA)

    $\mathcal{H}_0: \theta_i = \theta_j =0 $

    $\mathcal{H}_A:$ $\theta_i \neq 0 $ or $\theta_j \neq 0 $

We can use the [`OLS`](https://www.statsmodels.org/stable/regression.html) function of the `statsmodels` Python library to perform all these tests

First we create the model by giving the target and independent variables. In `statsmodels` jargon these are called endogenous and exogenous, respectively. Then we call the `fit` attribute

The coefficients obtained are equivalent to those we found with `numpy`

```python
mod = sm.OLS(Y, X, hasconst=True)
res = mod.fit()
display(theta,
        res.params)
```

The `summary` attribute gives as

- the `R-squared` statistic of the model
- the `F-statistic` and its p-value
- A table with the values of `theta` their standard errors, `t-statistics`, p-values and confidence interval

```python
display(res.summary(yname="Consumption",
                    xname=["Intercept", "Income", "Price", "Temperature"],
                    alpha=0.05))
```

Observations from the results table:

- The f-test tells that we can reject the hypothesis that all coefficients are null
- The t-test tells us that we cannot reject the null hypothesis that the price coefficient is null

The $r^2$ statistic for the multivariate case is defined as

$$
\begin{align}
r^2 &= 1 - \frac{\sum_i (y_i - \hat y_i)^2}{\sum_i (y_i - \bar y_i)^2} \nonumber \\
&= 1 - \frac{Y^T(I-X(X^TX)^{-1}X^T)Y}{Y^T (I - \frac{1}{N} \mathbb{1}^T \mathbb{1} ) Y} \nonumber \\
&= 1 - \frac{SS_{res}}{SS_{total}} \nonumber
\end{align}
$$

where $\mathbb{1} = (1, 1, \ldots, 1)$. And it has the same interpretation that was given in the previous lecture

:::{important}

We can trust the test only if our assumptions are true. The assumptions in this case are

- Relation between X and Y is linear
- Errors/noise follows a multivariate normal with covariance $I\sigma^2$

:::

Verify this assumptions by

1. Checking the residuals for normality. Are there outliers that we should remove?
1. Checking for absence of correlation in the residuals
1. Do the errors have different variance?

If the variance of the error is not constant (heteroscedastic) we can use the  **Weighted Least Squares** estimator

## Extra: Weighted Least Squares (WLS)

Before we assumed that the noise was homoscedastic (constant variance). We will generalize to the heteroscedastic case.

We can write the multivariate linear regression model with observations subject to Gaussian noise with changing variance as

$$
y_i = \theta_0 + \sum_{j=1}^D \theta_j x_{ij} + \varepsilon_i, \forall i \quad \text{and} \quad \varepsilon_i \sim \mathcal{N}(0, \sigma_i^2)
$$

With respect to OLS the only difference is that $\sigma_i \neq \sigma$

In this case the maximum likelihood solution is

$$
\hat \theta = (X^T \Sigma^{-1}X)^{-1} X^T \Sigma^{-1} Y
$$

where

$$
\Sigma = \begin{pmatrix}
\sigma_1^2 & 0 &\ldots & 0 \\
0 & \sigma_2^2 &\ldots & 0 \\
\vdots & \vdots &\ddots & \vdots \\
0 & 0 &\ldots & \sigma_N^2 \\
\end{pmatrix}
$$

An the distribution of $\theta$ is

$$
\hat \theta \sim \mathcal{N}( \theta,  (X^T X)^{-1} X^T  \Sigma X (X^T X)^{-1} )
$$


# ForestDML, ForestDRLearner, OrthoForest and CausalForest: Basic Example

We depict the performance of our `ForestDML`, `ForestDRLearner`, `OrthoForest` and `CausalForest` estimators on the same data generating process as the one used in the tutorial page of the grf package (see https://github.com/grf-labs/grf#usage-examples). This is mostly for qualitative comparison and verification purposes among our implementation of variants of Causal Forests and the implementation in the grf R package.

```python
%load_ext autoreload
%autoreload 2
```

```python
## Ignore warnings
import warnings
warnings.filterwarnings('ignore')
```

```python
# Helper imports
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
%matplotlib inline
```

# DGP

We use the following data generating process (DGP) from [here](https://github.com/grf-labs/grf#usage-examples):

\begin{align}
X \sim& \text{Normal}(0,\, I_{p})\\
T =& \text{Binomial}(1, .4 + .2 \cdot 1\{X[0] > 0\})\\
Y =& (X[0] \cdot 1\{X[0] > 0\}) \cdot T + X[1] + X[2] \cdot 1\{X[2] < 0\} + \epsilon, &\; \epsilon \sim \text{Normal}(0, 1)\\
\end{align}

We use $p=10$ and draw $n=2000$ samples from this DGP.

```python
import numpy as np
import scipy.special
np.random.seed(123)
n = 2000
p = 10
X = np.random.normal(size=(n, p))
true_propensity = lambda x: .4 + .2 * (x[:, 0] > 0)
true_effect = lambda x: (x[:, 0] * (x[:, 0] > 0))
true_conf = lambda x: x[:, 1] + np.clip(x[:, 2], - np.inf, 0)
T = np.random.binomial(1, true_propensity(X))
Y =  true_effect(X) * T + true_conf(X) + np.random.normal(size=(n,))
```

### Cross-Validated Forest Nuisance Models

We use forest based estimators (Gradient Boosted Forests or Random Forests) as nuisance models. For the meta-learner versions of our forest based estimators, we also use a generic forest estimator even as a final model. The hyperparameters of the forest models (e.g. number of estimators, max depth, min leaf size) is chosen via cross validation. We also choose among Gradient or Random Forests via cross validation

```python
from econml.sklearn_extensions.model_selection import GridSearchCVList
from sklearn.linear_model import Lasso, LogisticRegression
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.base import clone
from econml.sklearn_extensions.linear_model import WeightedLasso

def first_stage_reg():
    return GridSearchCVList([Lasso(),
                             RandomForestRegressor(n_estimators=100, random_state=123),
                             GradientBoostingRegressor(random_state=123)],
                             param_grid_list=[{'alpha': [.001, .01, .1, 1, 10]},
                                               {'max_depth': [3, None],
                                               'min_samples_leaf': [10, 50]},
                                              {'n_estimators': [50, 100],
                                               'max_depth': [3],
                                               'min_samples_leaf': [10, 30]}],
                             cv=5,
                             scoring='neg_mean_squared_error')

def first_stage_clf():
    return GridSearchCVList([LogisticRegression(),
                             RandomForestClassifier(n_estimators=100, random_state=123),
                             GradientBoostingClassifier(random_state=123)],
                             param_grid_list=[{'C': [0.01, .1, 1, 10, 100]},
                                              {'max_depth': [3, 5],
                                               'min_samples_leaf': [10, 50]},
                                              {'n_estimators': [50, 100],
                                               'max_depth': [3],
                                               'min_samples_leaf': [10, 30]}],
                             cv=5,
                             scoring='neg_mean_squared_error')

def final_stage():
    return GridSearchCVList([WeightedLasso(),
                             RandomForestRegressor(n_estimators=100, random_state=123)],
                             param_grid_list=[{'alpha': [.001, .01, .1, 1, 10]},
                                              {'max_depth': [3, 5],
                                               'min_samples_leaf': [10, 50]}],
                             cv=5,
                             scoring='neg_mean_squared_error')
```

```python
model_y = clone(first_stage_reg().fit(X, Y).best_estimator_)
model_y
```

    GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
                              init=None, learning_rate=0.1, loss='ls', max_depth=3,
                              max_features=None, max_leaf_nodes=None,
                              min_impurity_decrease=0.0, min_impurity_split=None,
                              min_samples_leaf=30, min_samples_split=2,
                              min_weight_fraction_leaf=0.0, n_estimators=50,
                              n_iter_no_change=None, presort='deprecated',
                              random_state=123, subsample=1.0, tol=0.0001,
                              validation_fraction=0.1, verbose=0, warm_start=False)

```python
model_t = clone(first_stage_clf().fit(X, T).best_estimator_)
model_t
```

    RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
                           criterion='gini', max_depth=3, max_features='auto',
                           max_leaf_nodes=None, max_samples=None,
                           min_impurity_decrease=0.0, min_impurity_split=None,
                           min_samples_leaf=10, min_samples_split=2,
                           min_weight_fraction_leaf=0.0, n_estimators=100,
                           n_jobs=None, oob_score=False, random_state=123,
                           verbose=0, warm_start=False)

# DML Estimators

```python
from econml.dml import ForestDML
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier
from sklearn.dummy import DummyRegressor, DummyClassifier

est = ForestDML(model_y=model_y,
                model_t=model_t,
                discrete_treatment=True,
                n_crossfit_splits=3,
                n_estimators=4000,
                subsample_fr='auto',
                min_samples_leaf=10,
                verbose=0,
                min_weight_fraction_leaf=.005)
est.fit(Y, T, X=X)
```

    <econml.dml.ForestDML at 0x2340e3c1c88>

```python
from econml.dml import NonParamDML
est2 = NonParamDML(model_y=model_y,
                   model_t=model_t,
                   n_splits=3,
                   discrete_treatment=True,
                   model_final=final_stage())
est2.fit(Y, T, X=X)
```

    <econml.dml.NonParamDML at 0x2342e7aac88>

```python
X_test = np.zeros((100, p))
X_test[:, 0] = np.linspace(-2, 2, 100)
```

```python
pred = est.effect(X_test)
lb, ub = est.effect_interval(X_test, alpha=0.01)
```

```python
pred2 = est2.effect(X_test)
```

```python
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(X_test[:, 0], true_effect(X_test), '--')
plt.plot(X_test[:, 0], pred2, label='nonparamdml')
plt.plot(X_test[:, 0], pred, label='forestdml (causal forest)')
plt.fill_between(X_test[:, 0], lb, ub, alpha=.4, label='honestrf_ci')
plt.legend()
plt.show()
```

### First Stage Learned Models

```python
# Model T
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.title('honestrf')
for mdl in est.models_t:
    plt.plot(X_test[:, 0], mdl.predict_proba(X_test)[:, 1])
plt.plot(X_test[:, 0], true_propensity(X_test), '--', label='truth')
plt.legend()

plt.subplot(1, 2, 2)
plt.title('rf')
for mdl in est2.models_t:
    plt.plot(X_test[:, 0], mdl.predict_proba(X_test)[:, 1])
plt.plot(X_test[:, 0], true_propensity(X_test), '--', label='truth')
plt.legend()
plt.show()
```

```python
# Model Y
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.title('honestrf')
for mdl in est.models_y:
    plt.plot(X_test[:, 0], mdl.predict(X_test))
plt.plot(X_test[:, 0], true_effect(X_test) * true_propensity(X_test) + true_conf(X_test), '--', label='truth')
plt.legend()

plt.subplot(1, 2, 2)
plt.title('rf')
for mdl in est2.models_y:
    plt.plot(X_test[:, 0], mdl.predict(X_test))
plt.plot(X_test[:, 0], true_effect(X_test) * true_propensity(X_test) + true_conf(X_test), '--', label='truth')
plt.legend()
plt.show()
```

### Interpretability of CATE Model of NonParamDML with SHAP

```python
import shap
import string
feature_names=list(string.ascii_lowercase)[:X.shape[1]]
# explain the model's predictions using SHAP values
shap_values = est.shap_values(X[:100],feature_names=feature_names)
```

     99%|===================| 99/100 [00:20<00:00]

```python
# visualize the first prediction's explanation (use matplotlib=True to avoid Javascript)
shap.force_plot(shap_values["Y0"]["T0"][0], matplotlib=True)
```

```python
shap.summary_plot(shap_values["Y0"]["T0"])
```

# DRLearner

```python
model_regression = clone(first_stage_reg().fit(np.hstack([T.reshape(-1, 1), X]), Y).best_estimator_)
model_regression
```

    GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
                              init=None, learning_rate=0.1, loss='ls', max_depth=3,
                              max_features=None, max_leaf_nodes=None,
                              min_impurity_decrease=0.0, min_impurity_split=None,
                              min_samples_leaf=30, min_samples_split=2,
                              min_weight_fraction_leaf=0.0, n_estimators=50,
                              n_iter_no_change=None, presort='deprecated',
                              random_state=123, subsample=1.0, tol=0.0001,
                              validation_fraction=0.1, verbose=0, warm_start=False)

```python
from econml.drlearner import ForestDRLearner
from sklearn.dummy import DummyRegressor, DummyClassifier

est = ForestDRLearner(model_regression=model_regression,
                      model_propensity=model_t,
                      n_crossfit_splits=3,
                      n_estimators=4000,
                      subsample_fr='auto',
                      min_samples_leaf=10,
                      verbose=0,
                      min_weight_fraction_leaf=.005)
est.fit(Y, T, X=X)
```

    <econml.drlearner.ForestDRLearner at 0x2342ee35548>

```python
from econml.drlearner import DRLearner
est2 = DRLearner(model_regression=model_regression,
                 model_propensity=model_t,
                 model_final=final_stage(),
                 n_splits=3)
est2.fit(Y, T, X=X)
```

    <econml.drlearner.DRLearner at 0x2342ee0fe88>

```python
X_test = np.zeros((100, p))
X_test[:, 0] = np.linspace(-2, 2, 100)
```

```python
pred = est.effect(X_test)
lb, ub = est.effect_interval(X_test, alpha=0.01)
```

```python
pred2 = est2.effect(X_test)
```

```python
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(X_test[:, 0], true_effect(X_test), '--')
plt.plot(X_test[:, 0], pred2, label='nonparamdml')
plt.plot(X_test[:, 0], pred, label='forestdml (causal forest)')
plt.fill_between(X_test[:, 0], lb, ub, alpha=.4, label='honestrf_ci')
plt.legend()
plt.show()
```

### First stage nuisance models

```python
# Model T
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.title('honestrf')
for mdl in est.models_propensity:
    plt.plot(X_test[:, 0], mdl.predict_proba(X_test)[:, 1])
plt.plot(X_test[:, 0], true_propensity(X_test), '--', label='truth')
plt.legend()

plt.subplot(1, 2, 2)
plt.title('rf')
for mdl in est2.models_propensity:
    plt.plot(X_test[:, 0], mdl.predict_proba(X_test)[:, 1])
plt.plot(X_test[:, 0], true_propensity(X_test), '--', label='truth')
plt.legend()
plt.show()
```

```python
# Model Y
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.title('honestrf')
for mdl in est.models_regression:
    plt.plot(X_test[:, 0], mdl.predict(np.hstack([X_test, np.ones((X_test.shape[0], 1))])))
plt.plot(X_test[:, 0], true_effect(X_test) + true_conf(X_test), '--', label='truth')
plt.legend()

plt.subplot(1, 2, 2)
plt.title('rf')
for mdl in est2.models_regression:
    plt.plot(X_test[:, 0], mdl.predict(np.hstack([X_test, np.ones((X_test.shape[0], 1))])))
plt.plot(X_test[:, 0], true_effect(X_test) + true_conf(X_test), '--', label='truth')
plt.legend()
plt.show()
```

### Interpretability of CATE Model of DRLearner with SHAP

```python
# explain the model's predictions using SHAP values
shap_values = est.shap_values(X[:100],feature_names=feature_names)
```

     95%|=================== | 95/100 [00:16<00:00]

```python
# visualize the first prediction's explanation (use matplotlib=True to avoid Javascript)
shap.force_plot(shap_values["Y0"]["T0"][0], matplotlib=True)
```

```python
shap.summary_plot(shap_values["Y0"]["T0"])
```

# OrthoForest and CausalForest

```python
from econml.ortho_forest import DROrthoForest
from sklearn.linear_model import Lasso, LassoCV, LogisticRegression, LogisticRegressionCV
from econml.sklearn_extensions.linear_model import WeightedLassoCV

est3 = DROrthoForest(model_Y=Lasso(alpha=0.01),
                     propensity_model=LogisticRegression(C=1),
                     model_Y_final=WeightedLassoCV(cv=3),
                     propensity_model_final=LogisticRegressionCV(cv=3),
                     n_trees=1000, min_leaf_size=10)
est3.fit(Y, T, X=X)
```

    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   16.0s
    [Parallel(n_jobs=-1)]: Done 112 tasks      | elapsed:   17.9s
    [Parallel(n_jobs=-1)]: Done 272 tasks      | elapsed:   21.1s
    [Parallel(n_jobs=-1)]: Done 496 tasks      | elapsed:   25.6s
    [Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed:   32.0s
    [Parallel(n_jobs=-1)]: Done 1000 out of 1000 | elapsed:   37.4s finished
    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:    0.4s
    [Parallel(n_jobs=-1)]: Done 208 tasks      | elapsed:    4.2s
    [Parallel(n_jobs=-1)]: Done 528 tasks      | elapsed:   10.0s
    [Parallel(n_jobs=-1)]: Done 1000 out of 1000 | elapsed:   18.4s finished

    <econml.ortho_forest.DROrthoForest at 0x2342e8ac588>

```python
pred3 = est3.effect(X_test)
```

    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   15.4s
    [Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   20.5s finished

```python
from econml.causal_forest import CausalForest
from sklearn.linear_model import Lasso, LassoCV, LogisticRegression, LogisticRegressionCV
from econml.sklearn_extensions.linear_model import WeightedLassoCV

est4 = CausalForest(model_Y=model_y,
                    model_T=model_t,
                    n_trees=1000, min_leaf_size=10,
                    discrete_treatment=True,
                    cv=5)
est4.fit(Y, T, X=X)
```

    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   11.6s
    [Parallel(n_jobs=-1)]: Done 233 tasks      | elapsed:   13.2s
    [Parallel(n_jobs=-1)]: Done 873 tasks      | elapsed:   16.8s
    [Parallel(n_jobs=-1)]: Done 985 out of 1000 | elapsed:   17.3s remaining:    0.2s
    [Parallel(n_jobs=-1)]: Done 1000 out of 1000 | elapsed:   17.4s finished
    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:    0.2s
    [Parallel(n_jobs=-1)]: Done 368 tasks      | elapsed:    2.5s
    [Parallel(n_jobs=-1)]: Done 1000 out of 1000 | elapsed:    5.9s finished

    <econml.causal_forest.CausalForest at 0x2343661c088>

```python
pred4 = est4.effect(X_test)
```

    [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
    [Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   12.4s
    [Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   15.1s finished

```python
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(X_test[:, 0], true_effect(X_test), '--')
plt.plot(X_test[:, 0], pred, label='forestdr')
plt.plot(X_test[:, 0], pred2, label='nonparamdr')
plt.plot(X_test[:, 0], pred3, label='discreteorf')
plt.plot(X_test[:, 0], pred4, label='causalforest')
plt.fill_between(X_test[:, 0], lb, ub, alpha=.4, label='forest_dr_ci')
plt.legend()
plt.show()
```

```python

```


# CME Session

### Goals
1. Search and download some coronagraph images
2. Load into Maps
3. Basic CME front enhancement
4. Extract CME front positions
6. Convert positions to height
5. Fit some simple models to height-time data

```python
%matplotlib notebook

import warnings
warnings.filterwarnings("ignore")

import astropy.units as u
import numpy as np

from astropy.time import Time
from astropy.coordinates import SkyCoord
from astropy.visualization import time_support, quantity_support

from matplotlib import pyplot as plt
from matplotlib.colors import LogNorm, SymLogNorm

from scipy import ndimage
from scipy.optimize import curve_fit
from sunpy.net import Fido, attrs as a
from sunpy.map import Map
from sunpy.coordinates.frames import Heliocentric, Helioprojective
```

# LASCO C2

## Data Search and Download

```python
c2_query = Fido.search(a.Time('2017-09-10T15:10', '2017-09-10T18:00'),
                       a.Instrument.lasco, a.Detector.c2)
```

```python
c2_query
```

```python
c2_results = Fido.fetch(c2_query);
c2_results
```

```python
c2_results
```

## Load into maps and plot

```python
# results = ['/Users/shane/sunpy/data/22650296.fts', '/Users/shane/sunpy/data/22650294.fts', '/Users/shane/sunpy/data/22650292.fts', '/Users/shane/sunpy/data/22650297.fts', '/Users/shane/sunpy/data/22650295.fts', '/Users/shane/sunpy/data/22650290.fts', '/Users/shane/sunpy/data/22650293.fts', '/Users/shane/sunpy/data/22650291.fts', '/Users/shane/sunpy/data/22650298.fts', '/Users/shane/sunpy/data/22650289.fts']
c2_maps = Map(c2_results, sequence=True);
```

```python
c2_maps.plot();
```

Check the polarisation and filter to make sure they don't change

```python
[(m.exposure_time, m.meta.get('polar'), m.meta.get('filter')) for m in c2_maps]
```

Rotate the map so standard orintation and pixel are aligned with wcs axes

```python
c2_maps = [m.rotate() for m in c2_maps];
```

# Running and Base Difference

The corona above $\sim 2 R_{Sun}$ is dominated by the F-corona (Fraunhofer corona) which is composed of photospheric radiation Rayleigh scattered off dust particles It forms a continuous spectrum with the Fraunhofer absorption lines superimposed. The radiation has a very low degree of polarisation.

There are a number of approaches to remove this the most straight forward are
* Running Difference $I(x,y)=I_i(x,y) - I_{i-1}(x,y)$
* Base Difference $I(x,y)=I_i(x,y) - I_{B}(x,y)$
* Background Subtraction $I(x,y)=I_i(x,y) - I_{BG}(x,y)$$

Can create new map using data and meta from other maps

```python
c2_bdiff_maps = Map([(c2_maps[i+1].data/c2_maps[i+1].exposure_time
                      - c2_maps[0].data/c2_maps[0].exposure_time, c2_maps[i+1].meta)
                     for i in range(len(c2_maps)-1)], sequence=True)
```

In jupyter notebook sunpy has very nice preview functionality

```python
c2_bdiff_maps
```

## CME front

One technique to help extract CME front is to create a space-time plot or j-plot we can define a region of interest and then sum ovber the region to increase signal to noise.

```python
fig, ax = plt.subplots(subplot_kw={'projection': c2_bdiff_maps[3]})

c2_bdiff_maps[3].plot(clip_interval=[1,99]*u.percent, axes=ax)
bottom_left = SkyCoord(0*u.arcsec, -200*u.arcsec, frame=c2_bdiff_maps[3].coordinate_frame)
top_right = SkyCoord(6500*u.arcsec, 200*u.arcsec, frame=c2_bdiff_maps[3].coordinate_frame)

c2_bdiff_maps[9].draw_quadrangle(bottom_left, top_right=top_right, axes=ax)
```

Going to extract the data in the region above and then sum over the y-direction to a 1-d plot of intensity vs pixel coordinate.

```python
c2_submaps = []
for m in c2_bdiff_maps:
    # define the coordinates of the bottom left and top right for each map should really define once and then transform
    bottom_left = SkyCoord(0*u.arcsec, -200*u.arcsec, frame=m.coordinate_frame)
    top_right = SkyCoord(6500*u.arcsec, 200*u.arcsec, frame=m.coordinate_frame)
    c2_submaps.append(m.submap(bottom_left, top_right=top_right))
```

```python
c2_submaps[0].data.shape
```

No we can create a space-time diagram by stack these slices one after another,

```python
c2_front_pix = []
def onclick(event):
    global coords
    ax.plot(event.xdata, event.ydata, 'o', color='r')
    c2_front_pix.append((event.ydata, event.xdata))

fig, ax = plt.subplots()

ax.imshow(np.stack([m.data.mean(axis=0)/m.data.mean(axis=0).max() for m in c2_submaps]).T,
           aspect='auto', origin='lower', interpolation='none', norm=SymLogNorm(0.1,vmax=2))
cid = fig.canvas.mpl_connect('button_press_event', onclick)
```

```python
if not c2_front_pix:
    c2_front_pix = [(209.40188156061873, 3.0291329045449533),
                    (391.58261749135465, 3.9464716142223724)]
```

```python
pix, index = c2_front_pix[0]
index = round(index)

fig, ax = plt.subplots(subplot_kw={'projection': c2_bdiff_maps[index]})
c2_bdiff_maps[index].plot(clip_interval=[1,99]*u.percent, axes=ax)
pp = c2_submaps[index].pixel_to_world(*[pix,34/4]*u.pix)

ax.plot_coord(pp, marker='x', ms=20, color='k');
```

Extract the times an coordinates for later

```python
c2_times = [m.date for m in c2_submaps[3:5] ]
c2_coords = [c2_submaps[i].pixel_to_world(*[c2_front_pix[i][0],34/4]*u.pix) for i, m in enumerate(c2_submaps[3:5])]
c2_times, c2_coords
```

# Lasco C3
## Data seach and download

```python
c3_query = Fido.search(a.Time('2017-09-10T15:10', '2017-09-10T19:00'),
                       a.Instrument.lasco, a.Detector.c3)
c3_query
```

Download

```python
c3_results = Fido.fetch(c3_query);
```

Load into maps

```python
c3_maps = Map(c3_results, sequence=True);
```

```python
c3_maps
```

Rotate

```python
c3_maps = [m.rotate() for m in c3_maps]
```

## Create Base Differnce maps

```python
c3_bdiff_maps = Map([(c3_maps[i+1].data/c3_maps[i+1].exposure_time.to_value('s')
                      - c3_maps[0].data/c3_maps[0].exposure_time.to_value('s'),
                      c3_maps[i+1].meta)
                     for i in range(0, len(c3_maps)-1)], sequence=True)
```

```python
c3_bdiff_maps
```

Can use a median filter to reduce some of the noise and make front easier to identify

```python
c3_bdiff_maps = Map([(ndimage.median_filter(c3_maps[i+1].data/c3_maps[i+1].exposure_time.to_value('s')
                      - c3_maps[0].data/c3_maps[0].exposure_time.to_value('s'), size=5),
                      c3_maps[i+1].meta)
                     for i in range(0, len(c3_maps)-1)], sequence=True)
```

## CME front

```python
fig, ax = plt.subplots(subplot_kw={'projection': c3_bdiff_maps[9]})
c3_bdiff_maps[9].plot(clip_interval=[1,99]*u.percent, axes=ax)

bottom_left = SkyCoord(0*u.arcsec, -2000*u.arcsec, frame=c3_bdiff_maps[9].coordinate_frame)
top_right = SkyCoord(29500*u.arcsec, 2000*u.arcsec, frame=c3_bdiff_maps[9].coordinate_frame)

c3_bdiff_maps[9].draw_quadrangle(bottom_left, top_right=top_right,
                        axes=ax)
```

Extract region of data

```python
c3_submaps = []
for m in c3_bdiff_maps:
    bottom_left = SkyCoord(0*u.arcsec, -2000*u.arcsec, frame=m.coordinate_frame)
    top_right = SkyCoord(29500*u.arcsec, 2000*u.arcsec, frame=m.coordinate_frame)
    c3_submaps.append(m.submap(bottom_left, top_right=top_right))
```

```python
c3_front_pix = []
def onclick(event):
    global coords
    ax.plot(event.xdata, event.ydata, 'o', color='r')
    c3_front_pix.append((event.ydata, event.xdata))

fig, ax = plt.subplots()

ax.imshow(np.stack([m.data.mean(axis=0)/m.data.mean(axis=0).max() for m in c3_submaps]).T,
           aspect='auto', origin='lower', interpolation='none', norm=SymLogNorm(0.1,vmax=2))
cid = fig.canvas.mpl_connect('button_press_event', onclick)
```

```python
if not c3_front_pix:
    c3_front_pix = [(75.84577056752656, 3.007459455920803),
                    (124.04923377098979, 3.9872981655982223),
                    (173.6704458922019, 5.039717520436931),
                    (216.20291342466945, 5.874394939791771),
                    (248.81113853289455, 6.854233649469189),
                    (287.0903593121153, 7.797782036565963),
                    (328.20507792683395, 8.995362681727254),
                    (369.3197965415526, 9.866330423662738),
                    (401.92802164977775, 10.991330423662738)]
```

```python
pix, index = c3_front_pix[5]
index = round(index)

fig, ax = plt.subplots(subplot_kw={'projection': c3_bdiff_maps[index]})
c3_bdiff_maps[index].plot(clip_interval=[1,99]*u.percent, axes=ax)
pp = c3_submaps[index].pixel_to_world(*[pix,37/4]*u.pix)
ax.plot_coord(pp, marker='x', ms=20, color='r');
```

Extract times and coordinates for later

```python
c3_times = [m.date for m in c3_submaps[3:12] ]
c3_coords = [c3_submaps[i].pixel_to_world(*[c3_front_pix[i][0],37/4]*u.pix) for i, m in enumerate(c3_submaps[3:12])]
```

# Coordintes to Heights

```python
times = Time(np.concatenate([c2_times, c3_times]))
coords = c2_coords + c3_coords
```

```python
heights_pos = np.hstack([(c.observer.radius * np.tan(c.Tx)) for c in coords])
heights_pos

heights_pos_error = np.hstack([(c.observer.radius * np.tan(c.Tx + 56*u.arcsecond*5)) for c in coords])
height_err = heights_pos_error - heights_pos
heights_pos, height_err
```

```python
times.shape, height_err.shape
```

```python
with Helioprojective.assume_spherical_screen(center=coords[0].observer):
    heights_sph = np.hstack([np.sqrt(c.transform_to('heliocentric').x**2
                + c.transform_to('heliocentric').y**2
                + c.transform_to('heliocentric').z**2) for c in coords])
heights_sph
```

```python
fig, axs = plt.subplots()
axs.errorbar(times.datetime, heights_pos.to_value(u.Rsun), yerr=height_err.to_value(u.Rsun), fmt='.')
axs.plot(times.datetime, heights_sph.to(u.Rsun), '+')
```

C2 data points look off not uncommon as different telescope different sensetivity we'll just drop these for the moment

```python
times = Time(np.hstack([c2_times, c3_times]))
heights_pos = heights_pos
height_err = height_err
```

# Model Fitting

### Models

Constant velocity model
\begin{align}
a = \frac{dv}{dt} = 0 \\
h(t) = h_0 + v_0 t  \\
\end{align}

Constant acceleration model
\begin{align}
a = a_{0} \\
v(t) = v_0 + a_0 t  \\
h(t) = h_0 + v_0 t + \frac{1}{2}a_0 t^{2}
\end{align}

```python
def const_vel(t0, h0, v0):
    return h0 + v0*t0

def const_accel(t0, h0, v0, a0):
    return h0  + v0*t0 + 0.5 * a0*t0**2
```

```python
t0 = (times-times[0]).to(u.s)
```

```python
const_vel_fit = curve_fit(const_vel, t0, heights_pos, sigma=height_err,
                          p0=[heights_pos[0].to_value(u.m), 350000])

h0, v0 = const_vel_fit[0]
delta_h0, delta_v0 = np.sqrt(const_vel_fit[1].diagonal())
h0 = h0*u.m
v0 = v0*u.m/u.s
delta_h0 = delta_h0*u.m
delta_v0 = delta_v0*(u.m/u.s)
print(f'h0: {h0.to(u.Rsun).round(2)} +/- {delta_h0.to(u.Rsun).round(2)}')
print(f'v0: {v0.to(u.km/u.s).round(2)} +/- {delta_v0.to(u.km/u.s).round(2)}')
```

```python
const_accel_fit = curve_fit(const_accel, t0, heights_pos, p0=[heights_pos[0].to_value(u.m), 600000, -5])

h0, v0, a0 = const_accel_fit[0]
delta_h0, delta_v0, delta_a0 = np.sqrt(const_accel_fit[1].diagonal())
h0 = h0*u.m
v0 = v0*u.m/u.s
a0 = a0*u.m/u.s**2
delta_h0 = delta_h0*u.m
delta_v0 = delta_v0*(u.m/u.s)
delta_a0 = delta_a0*(u.m/u.s**2)
print(f'h0: {h0.to(u.Rsun).round(2)} +/- {delta_h0.to(u.Rsun).round(2)}')
print(f'v0: {v0.to(u.km/u.s).round(2)} +/- {delta_v0.to(u.km/u.s).round(2)}')
print(f'a0: {a0.to(u.m/u.s**2).round(2)} +/- {delta_a0.to(u.m/u.s**2).round(2)}')
```

# Check against CDAW CME list
* https://cdaw.gsfc.nasa.gov/CME_list/UNIVERSAL/2017_09/univ2017_09.html

```python
with quantity_support():
    fig, axes = plt.subplots()
    axes.errorbar(times.datetime, heights_pos.to(u.Rsun), fmt='.', yerr=height_err)
    axes.plot(times.datetime, const_vel(t0.value, *const_vel_fit[0])*u.m, 'r-')
    axes.plot(times.datetime, const_accel(t0.value, *const_accel_fit[0])*u.m, 'r-')
    fig.autofmt_xdate()
```

```python
with quantity_support() and time_support(format='isot'):
    fig, axes = plt.subplots()
    axes.plot(times, heights_pos.to(u.Rsun), 'x')
    axes.plot(times, const_vel(t0.value, *const_vel_fit[0])*u.m, 'r-')
    axes.plot(times, const_accel(t0.value, *const_accel_fit[0])*u.m, 'r-')
    fig.autofmt_xdate()
```

Estimate the arrival time at Earth like distance for constant velocity model

```python
(((1*u.AU) - const_vel_fit[0][0] * u.m) / (const_vel_fit[0][1] * u.m/u.s)).decompose().to(u.hour)
```

```python
roots = np.roots([((1*u.AU) - const_accel_fit[0][0] * u.m).to_value(u.m),
          const_accel_fit[0][1], 0.5*const_accel_fit[0][2]][::-1])
```

```python
(roots*u.s).to(u.hour)
```

```python

```


Minimal example: Investigate the dispatch of two power plants
during a single time slot in dependence on decreasing residual load
(increasing variable renewable energy production).

Initialize a SymEnergy model instance:
```python
from symenergy.core import model
m = model.Model()
```
Add a time slot `'t0'` with 3000 load and 1 VRE production. VRE production
is varied later; any finite value of this parameter is acceptable at this point.

::

    m.add_slot('t0', load=3000, vre=1)

Add two power plants: Cheap baseload power `'n'` with limited capacity, expensive peaker plants with unconstrained power production:

::

    m.add_plant('n', vc0=1, capacity=1000)
    m.add_plant('g', vc0=2)

All Lagrange problems corresponding to this model are generated and solved.

::

    m.generate_solve()

All results of this process are stored in the model attribute ``df_comb``, indexed by the *constraint combinations* (see below)

```python
from symenergy.core import model
from sympy import init_printing
init_printing(pretty_print=False)
m = model.Model()
m.add_slot('t0', load=3000, vre=1)
m.add_plant('n', vc0=1, capacity=1000)
m.add_plant('g', vc0=2)
m.cache.delete()
m.generate_solve()

```

    > 11:12:49 - WARNING - symenergy.core.model - !!! Monkey-patching sympy.linsolve !!!
    > 11:12:50 - WARNING - symenergy.auxiliary.constrcomb - Aborting gen_col_combs: missing constraints ('p_cap_C',)
    > 11:12:51 - WARNING - symenergy.core.model - Number of solutions with linear dependencies: Key 1: 0 (0.0%), Key 2: 0 (0.0%), Key 3: 0 (0.0%)

```python
m.tc
```

    g_p_t0*vc0_g_none*w_none + n_p_t0*vc0_n_none*w_none

```python
import tabulate

```

```python
from importlib import reload
from symenergy.evaluator import evaluator
from symenergy import logger
import numpy as np
reload(evaluator)

x_vals_vre = {m.slots['t0'].vre: np.linspace(0, 3000, 31)}
ev = evaluator.Evaluator(m, x_vals_vre, drop_non_optimum=False)
```

```python
ev.cache_lambd.delete()
ev.get_evaluated_lambdas_parallel()

import inspect
print(inspect.getsource(
ev.df_lam_func.set_index(['idx', 'func']).loc[(1, 'g_p_t0'), 'lambd_func']
))
```

    def _6f5e357ced970205c772e7346c30689d(vre_t0):
        return(3000 - vre_t0)

```python
ev.cache_eval.delete()
ev.expand_to_x_vals_parallel()

ev.df_exp
```

    > 11:13:00 - WARNING - symenergy.evaluator.evaluator - _call_eval: Generating dataframe with length 372

      func
      idx
      vre_t0
      lambd
      act_lb_n_pos_p_t0
      act_lb_g_pos_p_t0
      act_lb_n_p_cap_C_t0
      is_positive
      map_sanitize
      mask_valid
      is_optimum
      slot
      func_no_slot

      0
      g_p_t0
      1
      0.0
      3000.0
      True
      False
      False
      1
      False
      True
      False
      t0
      g_p

      31
      n_p_t0
      1
      0.0
      0.0
      True
      False
      False
      1
      False
      True
      False
      t0
      n_p

      62
      pi_supply_t0
      1
      0.0
      2.0
      True
      False
      False
      0
      False
      True
      False
      t0
      pi_supply

      93
      tc
      1
      0.0
      6000.0
      True
      False
      False
      0
      False
      True
      False
      global
      tc

      124
      g_p_t0
      3
      0.0
      2000.0
      False
      False
      True
      1
      False
      True
      True
      t0
      g_p

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      247
      tc
      3
      3000.0
      -1000.0
      False
      False
      True
      0
      False
      False
      False
      global
      tc

      278
      g_p_t0
      4
      3000.0
      0.0
      False
      True
      False
      1
      False
      True
      False
      t0
      g_p

      309
      n_p_t0
      4
      3000.0
      0.0
      False
      True
      False
      1
      False
      True
      False
      t0
      n_p

      340
      pi_supply_t0
      4
      3000.0
      1.0
      False
      True
      False
      0
      False
      True
      False
      t0
      pi_supply

      371
      tc
      4
      3000.0
      0.0
      False
      True
      False
      0
      False
      True
      False
      global
      tc

372 rows × 13 columns

```python
(ev.df_bal.query('slot not in ["global"]')
   .pivot_table(index='vre_t0', columns='func',
                values='lambd')[['n_p_t0', 'g_p_t0', 'vre_t0', 'l_t0']].plot.area())
```

    <matplotlib.axes._subplots.AxesSubplot at 0x7fc825154e80>

```python
import tabulate
    tb = (ev.df_exp.query('is_optimum and func == "tc"')
            .groupby(['idx'] + m.constrs_cols_neq)
            .vre_t0.apply(lambda x: (x.min(), x.max()))
            .reset_index())
print(
tabulate.tabulate(tb, tb.columns, tablefmt='rst', showindex=False))
```

    =====  ===================  =====================  ===================  ================
      idx  act_lb_n_pos_p_t0    act_lb_n_p_cap_C_t0    act_lb_g_pos_p_t0    vre_t0
    =====  ===================  =====================  ===================  ================
        1  True                 False                  False                (3000.0, 3000.0)
        3  False                True                   False                (0.0, 2000.0)
        4  False                False                  True                 (2100.0, 2900.0)
    =====  ===================  =====================  ===================  ================