```
# default_exp foldnet
```

```
#export
from wong.imports import *
from wong.core import *
from wong.config import cfg, assert_cfg

```

```
from fastcore.all import *  # test_eq
```

# FoldNet
> a folded ResNet

Aggregate to enough units for folded net:
1. Unit : unit operator
2. ni : number of input channels for `Unit`
3. fold : folding length
4. stride : across stage or not
5. **kwargs : arguments to `Unit`

```
# #export
# class FoldBlock(nn.Module):
#     "Basic block of folded ResNet"
#     def __init__(self, Unit:nn.Module, ni:int, fold:int, stride:int=1, **kwargs):
#         super(FoldBlock, self).__init__()
#         self.ni, self.fold, self.stride = ni, fold, stride
#         units = []
#         for i in range(max(1,fold-1)):
#             units += [Unit(ni, stride=1, **kwargs)]
#         self.units = nn.ModuleList(units)

#     def forward(self, *xs):
#         xs = list(xs)
#         if self.fold==1:
#             xs[0] = xs[0] + self.units[0](xs[0])
#             return xs
#         for i in range(self.fold-1):
#             xs[i+1] = xs[i] + self.units[i](xs[i+1])
#         xs.reverse()
#         return xs
```

```
#export
class FoldBlock(nn.Module):
    "Basic block of folded ResNet"
    def __init__(self, Unit:nn.Module, ni:int, fold:int, stride:int=1, **kwargs):
        super(FoldBlock, self).__init__()
        self.ni, self.fold, self.stride = ni, fold, stride
        units = []
        for i in range(max(1,fold-1)):
            units += [Unit(ni, stride=1, **kwargs)]
        self.units = nn.ModuleList(units)

    def forward(self, *xs):
        xs = list(xs)
        if self.fold==1:
            xs[0] = xs[0] + self.units[0](xs[0])
            return xs
        for i in range(self.fold-1):
            xs[i+1] = xs[i+1] + self.units[i](xs[i])
        xs.reverse()
        return xs
```

```
m = FoldBlock(mbconv, 16, 4, nh=32)

xs = [torch.randn(2,16,32,32)] * 4

xs2 = m(*xs)

len(xs2), [x.shape for x in xs2]
```

    (4,
     [torch.Size([2, 16, 32, 32]),
      torch.Size([2, 16, 32, 32]),
      torch.Size([2, 16, 32, 32]),
      torch.Size([2, 16, 32, 32])])

```
#export
class ExpandBlock(nn.Module):
    "Expand block of folded ResNet"
    def __init__(self, Unit:nn.Module, ni:int, fold1:int, fold2:int, stride:int=1, **kwargs):
        super(ExpandBlock, self).__init__()
        self.ni, self.fold1, self.fold2, self.stride = ni, fold1, fold2, stride
        units = []
        for i in range(fold2 - fold1):
            units += [Unit(ni, stride=1, **kwargs)]
        self.units = nn.ModuleList(units)
        if stride == 2:
            self.pool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

    def forward(self, *xs):
        xs = list(xs)
        if self.stride == 2:
            for i in range(len(xs)):
                xs[i] = self.pool(xs[i])
        if self.fold2 <= self.fold1:
            return xs[:self.fold2]
        xs.reverse()
        for i in range(self.fold2 - self.fold1):
            xs.append(self.units[i](xs[-1]) + xs[-1])
        xs.reverse()
        return xs
```

```
m = ExpandBlock(mbconv, 16, fold1=4, fold2=3, stride=2, nh=32)
```

```
xs2 = m(*xs)
```

```
len(xs2), [x.shape for x in xs2]
```

    (3,
     [torch.Size([2, 16, 16, 16]),
      torch.Size([2, 16, 16, 16]),
      torch.Size([2, 16, 16, 16])])

```
#export
class FoldNet(nn.Module):
    "A folded resnet, using Expand."
    def __init__(self, Stem, Unit, folds:tuple, ni:int, num_nodes:tuple,
                 bottle_scale:int=1, first_downsample:bool=False, tail_all:bool=True,
                 c_in:int=3, c_out:int=10, **kwargs):
        super(FoldNet, self).__init__()
        num_stages = len(num_nodes)
        nh = int(ni * bottle_scale)
        strides = [1 if i==0 and not first_downsample else 2 for i in range(num_stages)]
        folds = [1] + folds #[fold*exp**i for i in range(num_stages)]

        self.stem = Stem(c_in, no=ni) # , deep_stem

        units = []
        for i, (nu, stride) in enumerate(zip(num_nodes, strides)):
            for j in range(nu):
                if j == 0: # the first node(layer) of each stage
                    units += [ExpandBlock(Unit, ni, fold1 = folds[i], fold2=folds[i+1], stride=stride, nh=nh, **kwargs)]
                else:
                    units += [FoldBlock(Unit, ni, fold=folds[i+1], stride=1, nh=nh, **kwargs)]

        self.units = nn.ModuleList(units)

        if tail_all:
            self.classifier = Classifier(ni*folds[-1], c_out) #
        else:
            self.classifier = Classifier(ni, c_out)
        self.folds = folds
        self.num_nodes = num_nodes
        self.tail_all = tail_all
        init_cnn(self)

    def forward(self, x):
        x = self.stem(x)
        xs = [x] #self.init(x)
        for unit in self.units:
            xs = unit(*xs)
        if self.tail_all:
            x = torch.cat(xs,1)
        else:
            x = xs[0]

        x = self.classifier(x)
        return x

```

```
num_nodes = [8,9,9,9]
folds = [4,4,4,4]
model = FoldNet(Stem=conv_bn, Unit=mbconv, folds=folds, ni=64, num_nodes=num_nodes, bottle_scale=1, tail_all=True, ks=3, c_out=100)
num_params(model)
```

    tensor(906148)

```
x = torch.randn(2,3,64,64)
```

```
with torch.autograd.set_detect_anomaly(True):
    out = model(x)
    out.mean().backward()
```

## Calculate number of params in FoldNet

```
#export
def num_units(folds, nodes):
    "calculate the number of all units in the backbone of FoldNet."
    num_units = (folds[0] - 1) * nodes[0]
    for i in range(len(folds)-1):
#         print(num_units)
        num_units += (folds[i+1]-1)*(nodes[i+1]-1) + max(0, folds[i+1]-folds[i])
    return(num_units)
```

Suppose the folding length of stage $i$ is $d_i$, then the number of units per FoldNet block is $d_i-1$.Suppose the number of blocks per stage is $b_i$, then the number of units of stage $i$ equal to $(d_i-1) * (b_i-1)$ for all the stages except the first stage, since a `ExpandBlock` with none unit start at each stage except the first stage.

Suppose $n$ stages exist, then the number of all units in the backbone of FoldNet is:
\begin{equation}
(d_0-1) * b_0 + \sum_1^{n-1} (d_i-1) * (b_i-1)
\end{equation}

```
#export
def cal_num_params(Stem, Unit, folds, nodes, ni, bottle_scale, tail_all, c_out):
    "calcuate the number of all params of FoldNet, according to hyper-params."
    m0 = Stem(3, no=ni)
    m1 = Unit(ni, nh=ni*bottle_scale)
    m2 = Classifier(ni*folds[-1] if tail_all else ni, c_out)
    return num_params(m0) + num_params(m1) * num_units(folds, nodes) + num_params(m2)
```

```
#folds = [fold]*4
cal_num_params(Stem=conv_bn, Unit=mbconv, folds=folds, nodes=num_nodes, ni=64, bottle_scale=1, tail_all=True, c_out=100)
```

    tensor(906148)

## Misc

```
#hide
class InitBlock(nn.Module):
    "Init block of folded ResNet"
    def __init__(self, Unit:nn.Module, ni:int, fold:int, stride:int=1, **kwargs):
        super(InitBlock, self).__init__()
        self.ni, self.fold = ni, fold
        units = []
        for i in range(fold-1):
            units += [Unit(ni, stride=stride, **kwargs)]
        self.units = nn.ModuleList(units)

    def forward(self, x):
        xs = [x]
        for i in range(self.fold-1):
            xs += [xs[i] + self.units[i](xs[i])]
        xs.reverse()
        return xs
```

```
#hide
m = InitBlock(mbconv, 16, 4, stride=1, nh=32)
x = torch.randn(2,16,32,32)
xs = m(x)
len(xs), xs[0].shape
```

    (4, torch.Size([2, 16, 32, 32]))

```
#hide
# try inner fold (fold before BN), may fail
class FoldBlock2(nn.Module):
    "Basic block of folded ResNet"
    def __init__(self, Unit:nn.Module, ni:int, fold:int, stride:int=1, **kwargs):
        super(FoldBlock2, self).__init__()
        self.ni, self.fold, self.stride = ni, fold, stride
        units = []
        aggregates = []
        for i in range(fold-1):
            units += [Unit(ni, stride=1, **kwargs)]
            aggregates += [conv_bn(ni, ks=1, zero_bn=False)]
        self.units = nn.ModuleList(units)
        self.aggregates = nn.ModuleList(aggregates)

    def forward(self, *xs):
        xs = list(xs)
        for i in range(self.fold-1):
            xs[i+1] = xs[i+1] + self.units[i](xs[i])
        for i in range(self.fold-1):
            xs[i+1] = self.aggregates[i](xs[i+1])
        xs.reverse()
        return xs
```

```
#hide
# try particular transition block, may fail
class TransitionBlock(nn.Module):
    "Transition block of folded ResNet"
    def __init__(self, Unit:nn.Module, ni:int, no:int, fold:int, stride:int=1, **kwargs):
        super(TransitionBlock, self).__init__()
        self.ni, self.no, self.fold, self.stride = ni, no, fold, stride
        units = []
        idmappings = []
        for i in range(fold-1):
            if i==0:
                units += [Unit(ni, no=no, stride=stride, **kwargs)]
            else:
                units += [Unit(ni=no, no=no, stride=1, **kwargs)]
            idmappings += [IdentityMappingMaxPool(ni, no=no, stride=stride)]
        self.units = nn.ModuleList(units)
        self.idmappings = nn.ModuleList(idmappings)
        self.idmapping0 = IdentityMappingMaxPool(ni, no=no, stride=stride)

    def forward(self, *xs):
        xs = list(xs)
        for i in range(self.fold-1):
            xs[i+1] = self.idmappings[i](xs[i+1]) + self.units[i](xs[i])
        xs[0] = self.idmapping0(xs[0])
        xs.reverse()
        return xs
```

```
#hide
m = TransitionBlock(mbconv, ni=16, no=32, fold=4, stride=2, nh=32)

[x.shape for x in xs]

xs2 = m(*xs)

len(xs2), [x.shape for x in xs2]

isinstance(m, TransitionBlock), isinstance(m, ExpandBlock)
```

    (True, False)

```
#hide
# try particular transition block, may fail
class ResNetXTransition(nn.Module):
    "A folded resnet using Transition."
    def __init__(self, Stem, Unit, fold:int, nis:tuple, num_nodes:tuple,
                 bottle_scale:int=1, first_downsample:bool=False, tail_all:bool=True,
                 c_in:int=3, c_out:int=10, **kwargs):
        super(ResNetXTransition, self).__init__()
        num_stages = len(num_nodes)
        nhs = [int(ni * bottle_scale) for ni in nis]
        strides = [1 if i==0 and not first_downsample else 2 for i in range(num_stages)]

        self.stem = Stem(c_in, no=nis[0]) # , deep_stem
        self.expand = ExpandBlock(Unit, ni=nis[0], fold1=1, fold2=fold, stride=strides[0], nh=nhs[0], **kwargs)

        units = []
        for i, (nu, stride) in enumerate(zip(num_nodes, strides)):
            if i != 0:
                units += [TransitionBlock(Unit, ni=nis[i-1], no=nis[i], fold=fold, stride=stride, nh=nhs[i-1], **kwargs)]
            for j in range(nu):
#                 if j == 0: # the first node(layer) of each stage
#                     units += [ExpandBlock(Unit, ni, fold1 = folds[i], fold2=folds[i+1], stride=stride, nh=nh, **kwargs)]
#                 else:
                units += [FoldBlock(Unit, ni=nis[i], fold=fold, stride=1, nh=nhs[i], **kwargs)]

        self.units = nn.ModuleList(units)

        if tail_all:
            self.classifier = Classifier(nis[-1]*fold, c_out) #
        else:
            self.classifier = Classifier(nis[-1], c_out)
        self.fold = fold
        self.num_nodes = num_nodes
        self.tail_all = tail_all
        init_cnn(self)

    def forward(self, x):
        x = self.stem(x)
        xs = self.expand(*[x])
        for unit in self.units:
            xs = unit(*xs)
        if self.tail_all:
            x = torch.cat(xs,1)
        else:
            x = xs[0]

        x = self.classifier(x)
        return x

```

```
#hide
num_nodes = [4,4,4,4]
fold = 4
nis = [32,64,96,128]
bottle_scale = 6
model = ResNetXTransition(Stem=conv_bn, Unit=mbconv, fold=fold, nis=nis, num_nodes=num_nodes,
                          bottle_scale=bottle_scale, tail_all=False, ks=3, c_out=100)
num_params(model)
```

    tensor(5620548)


##### Iмпортую бiблiотеки.

```python
import keras
import matplotlib.pyplot as plt
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Flatten
from keras.utils.np_utils import to_categorical
from keras import metrics
import numpy as np
import sympy as sp
from fractions import Fraction
import math
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report,confusion_matrix
```

    Using TensorFlow backend.

##### Завантажую данi про картинки.

```python
(trainData, trainAnswer), (testData, testAnswer) = mnist.load_data()
print("Train data : {}".format(trainData.shape))
print("Test data : {}".format(testData.shape))
```

    Train data : (60000, 28, 28)
    Test data : (10000, 28, 28)

##### Реалiзував свою модель нейронної мережi.

```python
class Perceptron:
    def __init__ (self, activFunc = (lambda x: x), inputSize = 1):
        self.activFunc = activFunc
        self.inputSize = inputSize
        self.weights = np.random.sample(inputSize)
        self.outputValue = 0
        self.lastInput = []
        self.lastOutput = 0

    def output(self, inputData):
        if len(inputData)!=self.inputSize:
            raise ValueError('Invalid size! Received: {}, should be: {}.'.format(len(inputData), self.inputSize))
        else:
            self.lastInput = inputData
            self.lastOutput = self.activFunc(self.wsm())
            return self.activFunc(self.wsm())

    def wsm(self):
        ws = 0
        for i in range(self.inputSize):
            ws+=self.weights[i]*self.lastInput[i]
        return ws

class mlpNeurallNetwork:
    def __init__ (self, size = (1,), activFunc = (lambda x: x), inputSize = 1, errorFunc = (lambda x, y: (x-y)**2)):
        self.inputSize = inputSize
        self.size = size+(1,)
        self.perceptrons = []
        self.errorFunc = errorFunc
        for i in self.size:
            self.perceptrons.append(i*[0])
        presize = inputSize
        for i in range(len(self.size)):
            for j in range(self.size[i]):
                self.perceptrons[i][j] = Perceptron(activFunc = activFunc, inputSize = presize)
            presize = self.size[i]

    def predict(self, inputData):
        self.lastInput = inputData
        oldInput = inputData
        for layer in self.perceptrons:
            newInput = []
            for pc in layer:
                newInput.append(pc.output(oldInput))
            oldInput = newInput
        return oldInput[0]

    def limit(self, func, x):
        return 1

    def dfs(self, lossdif, layer, r):
        if layer == -1:
            return
        for p in self.perceptrons[layer]:
            lm = self.limit(p.activFunc, p.wsm())
            for i in range(p.inputSize):
                self.dfs(lm*lossdif*p.weights[i], layer-1, r)
                p.weights[i]-=r*lm*self.lastInput[i]*lossdif

    def fit(self, inputData, answerData, r = 0.01):
        for i in range(len(inputData)):
            lossdif = 2*(self.predict(inputData[i]) - answerData[i])
            self.dfs(lossdif, len(self.size)-1, r)

```

##### Кодую данi в клас розмiрностi 10.

```python
trainAnswer = to_categorical(trainAnswer, num_classes = 10)
testAnswer = to_categorical(testAnswer, num_classes = 10)
```

```python
trainData = trainData/255.
testData = testData/255.
```

```python
n_train, h, w = trainData.shape
n_test = testData.shape[0]

trainData = trainData.reshape(n_train, h * w)
testData = testData.reshape(n_test, h * w)
print("Train data : {}".format(trainData.shape))
print("Test data : {}".format(testData.shape))
```

    Train data : (60000, 784)
    Test data : (10000, 784)

##### Створюю бiблiотечну модель, додаю до неї один прихований шар з 784 нейронiв, та один вихiдний шар - на 10.

```python
model = Sequential()
model.add(Dense(784, input_dim = 784,  activation='relu', kernel_initializer="normal"))
model.add(Dense(10,  activation='softmax'))
model.summary()
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=[metrics.categorical_accuracy])
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_1 (Dense)              (None, 784)               615440
    _________________________________________________________________
    dense_2 (Dense)              (None, 10)                7850
    =================================================================
    Total params: 623,290
    Trainable params: 623,290
    Non-trainable params: 0
    _________________________________________________________________

##### Треную модель на ранiше завантажених даних.

```python
history = model.fit(trainData, trainAnswer, batch_size=200, epochs=10)
```

    Epoch 1/10
    60000/60000 [==============================] - 10s 161us/step - loss: 0.2805 - categorical_accuracy: 0.9199
    Epoch 2/10
    60000/60000 [==============================] - 9s 154us/step - loss: 0.1123 - categorical_accuracy: 0.9675
    Epoch 3/10
    60000/60000 [==============================] - 9s 152us/step - loss: 0.0716 - categorical_accuracy: 0.9798
    Epoch 4/10
    60000/60000 [==============================] - 9s 147us/step - loss: 0.0497 - categorical_accuracy: 0.9858
    Epoch 5/10
    60000/60000 [==============================] - 9s 149us/step - loss: 0.0347 - categorical_accuracy: 0.9902
    Epoch 6/10
    60000/60000 [==============================] - 9s 149us/step - loss: 0.0263 - categorical_accuracy: 0.9928
    Epoch 7/10
    60000/60000 [==============================] - 9s 146us/step - loss: 0.0202 - categorical_accuracy: 0.9948
    Epoch 8/10
    60000/60000 [==============================] - 9s 147us/step - loss: 0.0139 - categorical_accuracy: 0.9967
    Epoch 9/10
    60000/60000 [==============================] - 9s 150us/step - loss: 0.0105 - categorical_accuracy: 0.9976
    Epoch 10/10
    60000/60000 [==============================] - 9s 150us/step - loss: 0.0080 - categorical_accuracy: 0.9984

##### Будую графiки втрат та точоcтей пiд час тренування.

```python
fig, (loss_graph, acc_graph) = plt.subplots(nrows=1, ncols=2, figsize=(15, 5))

acc_list, loss_list = history.history['categorical_accuracy'], history.history['loss']

listX = range(1, 11)
loss_graph.plot(listX, loss_list);
loss_graph.set_title('LOSS GRAPH');
acc_graph.plot(listX, acc_list);
acc_graph.set_title('ACCURACY GRAPH');
```

##### Результати тренування.

```python
loss, accuracy = model.evaluate(testData, testAnswer)
print('Loss on tests: ', loss)
print('Accuracy: ', accuracy)
```

    10000/10000 [==============================] - 1s 101us/step
    Loss on tests:  0.07000552951387362
    Accuracy:  0.9811

##### Завантажую данi з сifar10.

```python
from keras.datasets import cifar10
```

```python
(trainData, trainAnswer), (testData, testAnswer)  = cifar10.load_data()
print("Train data : {}".format(trainData.shape))
print("Test data : {}".format(testData.shape))
```

    Train data : (50000, 32, 32, 3)
    Test data : (10000, 32, 32, 3)

```python
n_train, height, width, channels = trainData.shape
n_test = testData.shape[0]

trainData = trainData.reshape(n_train, height*width*channels)
testData = testData.reshape(n_test, height*width*channels)
print("Train data : {}".format(trainData.shape))
print("Test data : {}".format(testData.shape))
```

    Train data : (50000, 3072)
    Test data : (10000, 3072)

```python
trainData = trainData/255.
testData = testData/255.

trainAnswer = to_categorical(trainAnswer, num_classes = 10)
testAnswer = to_categorical(testAnswer, num_classes = 10)
```

##### Створюю нову модель, та додаю декiлька шарiв нейронiв.

```python
newModel = Sequential()
newModel.add(Dense(300, input_dim = 3072,  activation='relu', kernel_initializer="normal"))
newModel.add(Dense(300, input_dim = 3072,  activation='relu', kernel_initializer="normal"))
newModel.add(Dense(150, input_dim = 3072,  activation='relu', kernel_initializer="normal"))
newModel.add(Dense(10,  activation='softmax'))
newModel.summary()
newModel.compile(loss="categorical_crossentropy", optimizer="adam", metrics=[metrics.categorical_accuracy])
```

    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_14 (Dense)             (None, 300)               921900
    _________________________________________________________________
    dense_15 (Dense)             (None, 300)               90300
    _________________________________________________________________
    dense_16 (Dense)             (None, 150)               45150
    _________________________________________________________________
    dense_17 (Dense)             (None, 10)                1510
    =================================================================
    Total params: 1,058,860
    Trainable params: 1,058,860
    Non-trainable params: 0
    _________________________________________________________________

##### Тренування моделi та побудова графiкiв.

```python
history = newModel.fit(trainData, trainAnswer, batch_size=300, epochs=10)
```

    Epoch 1/10
    50000/50000 [==============================] - 11s 211us/step - loss: 1.9026 - categorical_accuracy: 0.3061
    Epoch 2/10
    50000/50000 [==============================] - 10s 199us/step - loss: 1.6691 - categorical_accuracy: 0.4033
    Epoch 3/10
    50000/50000 [==============================] - 10s 200us/step - loss: 1.5905 - categorical_accuracy: 0.4322
    Epoch 4/10
    50000/50000 [==============================] - 10s 200us/step - loss: 1.5278 - categorical_accuracy: 0.4522
    Epoch 5/10
    50000/50000 [==============================] - 10s 207us/step - loss: 1.4763 - categorical_accuracy: 0.4729
    Epoch 6/10
    50000/50000 [==============================] - 13s 253us/step - loss: 1.4335 - categorical_accuracy: 0.4895
    Epoch 7/10
    50000/50000 [==============================] - 11s 211us/step - loss: 1.3975 - categorical_accuracy: 0.5000
    Epoch 8/10
    50000/50000 [==============================] - 10s 205us/step - loss: 1.3691 - categorical_accuracy: 0.5122
    Epoch 9/10
    50000/50000 [==============================] - 10s 204us/step - loss: 1.3420 - categorical_accuracy: 0.5228
    Epoch 10/10
    50000/50000 [==============================] - 10s 203us/step - loss: 1.3128 - categorical_accuracy: 0.5310

```python
fig, (loss_graph, acc_graph) = plt.subplots(nrows=1, ncols=2, figsize=(15, 5))

acc_list, loss_list = history.history['categorical_accuracy'], history.history['loss']

listX = range(1, 11)
loss_graph.plot(listX, loss_list);
loss_graph.set_title('LOSS GRAPH');
acc_graph.plot(listX, acc_list);
acc_graph.set_title('ACCURACY GRAPH');
```

##### Результати при проходженнi тестових даних.

```python
cftest_loss, cftest_accuracy = newModel.evaluate(testData, testAnswer)
print('Loss on tests: ', cftest_loss)
print('Accuracy: ', cftest_accuracy)
```

    10000/10000 [==============================] - 2s 164us/step
    Loss on tests:  1.38714591255188
    Accuracy:  0.5113


# 03 - Full-Waveform Inversion (FWI)

This notebook is the third in a series of tutorial highlighting various aspects of seismic inversion based on Devito operators. In this second example we aim to highlight the core ideas behind seismic inversion, where we create an image of the subsurface from field recorded data. This tutorial follows on the modelling tutorial and will reuse the modelling and velocity model.

## Inversion requirement

Seismic inversion relies on two known parameters:

- **Field data** - or also called **recorded data**. This is a shot record corresponding to the true velocity model. In practice this data is acquired as described in the first tutorial. In order to simplify this tutorial we will fake field data by modelling it with the true velocity model.

- **Initial velocity model**. This is a velocity model that has been obtained by processing the field data. This model is a rough and very smooth estimate of the velocity as an initial estimate for the inversion. This is a necessary requirement for any optimization (method).

## Inversion computational setup

In this tutorial, we will introduce the gradient operator. This operator corresponds to the imaging condition introduced in the previous tutorial with some minor modifications that are defined by the objective function (also referred to in the tutorial series as the *functional*, *f*) and its gradient, *g*. We will define this two terms in the tutorial too.

## Notes on the operators

As we already describe the creation of a forward modelling operator, we will only call an wrapped function here. This wrappers already contains all the necessary operator for seismic modeling, imaging and inversion, however any new operator will be fully described and only used from the wrapper in the next tutorials.

```python
import numpy as np
%matplotlib inline

from devito import configuration
configuration['log-level'] = 'WARNING'
```

## Computational considerations

As we will see in this tutorial, FWI is again very computationally demanding, even more so than RTM. To keep this tutorial as light-wight as possible we therefore again use a very small demonstration model. We also define here a few parameters for the final example runs that can be changed to modify the overall runtime of the tutorial.

```python
nshots = 9  # Number of shots to create gradient from
nreceivers = 101  # Number of receiver locations per shot
fwi_iterations = 5  # Number of outer FWI iterations
```

# True and smooth velocity models

As before, we will again use a very simple model domain, consisting of a circle within a 2D domain. We will again use the "true" model to generate our synthetic shot data and use a "smooth" model as our initial guess. In this case the smooth model is very smooth indeed - it is simply a constant background velocity without any features.

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import demo_model, plot_velocity, plot_perturbation

# Define true and initial model
shape = (101, 101)  # Number of grid point (nx, nz)
spacing = (10., 10.)  # Grid spacing in m. The domain size is now 1km by 1km
origin = (0., 0.)  # Need origin to define relative source and receiver locations

model = demo_model('circle-isotropic', vp=3.0, vp_background=2.5,
                    origin=origin, shape=shape, spacing=spacing, nbpml=40)

model0 = demo_model('circle-isotropic', vp=2.5, vp_background=2.5,
                     origin=origin, shape=shape, spacing=spacing, nbpml=40,
                    grid = model.grid)

plot_velocity(model)
plot_velocity(model0)
plot_perturbation(model0, model)
```

## Acquisition geometry

In this tutorial, we will use the easiest case for inversion, namely a transmission experiment. The sources are located on one side of the model and the receivers on the other side. This allow to record most of the information necessary for inversion, as reflections usually lead to poor inversion results.

```python
#NBVAL_IGNORE_OUTPUT
# Define acquisition geometry: source
from examples.seismic import AcquisitionGeometry

t0 = 0.
tn = 1000.
f0 = 0.010
# First, position source centrally in all dimensions, then set depth
src_coordinates = np.empty((1, 2))
src_coordinates[0, :] = np.array(model.domain_size) * .5
src_coordinates[0, 0] = 20.  # Depth is 20m

# Define acquisition geometry: receivers

# Initialize receivers for synthetic and imaging data
rec_coordinates = np.empty((nreceivers, 2))
rec_coordinates[:, 1] = np.linspace(0, model.domain_size[0], num=nreceivers)
rec_coordinates[:, 0] = 980.

# Geometry

geometry = AcquisitionGeometry(model, rec_coordinates, src_coordinates, t0, tn, f0=f0, src_type='Ricker')
# We can plot the time signature to see the wavelet
geometry.src.show()
```

```python
# Plot acquisition geometry
plot_velocity(model, source=geometry.src_positions,
              receiver=geometry.rec_positions[::4, :])
```

## True and smooth data

We can generate shot records for the true and smoothed initial velocity models, since the difference between them will again form the basis of our imaging procedure.

```python
# Compute synthetic data with forward operator
from examples.seismic.acoustic import AcousticWaveSolver

solver = AcousticWaveSolver(model, geometry, space_order=4)
true_d, _, _ = solver.forward(m=model.m)
```

```python
# Compute initial data with forward operator
smooth_d, _, _ = solver.forward(m=model0.m)
```

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import plot_shotrecord

# Plot shot record for true and smooth velocity model and the difference
plot_shotrecord(true_d.data, model, t0, tn)
plot_shotrecord(smooth_d.data, model, t0, tn)
plot_shotrecord(smooth_d.data - true_d.data, model, t0, tn)
```

# Full-Waveform Inversion

## Formulation

Full-waveform inversion (FWI) aims to invert an accurate model of the discrete wave velocity, $\mathbf{c}$, or equivalently the square slowness of the wave, $\mathbf{m} = \frac{1}{\mathbf{c}^2}$, from a given set of measurements of the pressure wavefield $\mathbf{u}$. This can be expressed as the following optimization problem [1, 2]:

\begin{aligned}
	\mathop{\hbox{minimize}}_{\mathbf{m}} \Phi_s(\mathbf{m})&=\frac{1}{2}\left\lVert\mathbf{P}_r
	\mathbf{u} - \mathbf{d}\right\rVert_2^2 \\
	\mathbf{u} &= \mathbf{A}(\mathbf{m})^{-1} \mathbf{P}_s^T \mathbf{q}_s,
\end{aligned}

where $\mathbf{P}_r$ is the sampling operator at the receiver locations, $\mathbf{P}_s^T$ is the injection operator at the source locations, $\mathbf{A}(\mathbf{m})$ is the operator representing the discretized wave equation matrix, $\mathbf{u}$ is the discrete synthetic pressure wavefield, $\mathbf{q}_s$ is the corresponding pressure source and $\mathbf{d}$ is the measured data. It is worth noting that $\mathbf{m}$ is the unknown in this formulation and that multiple implementations of the wave equation operator $\mathbf{A}(\mathbf{m})$ are possible.

We have already defined a concrete solver scheme for $\mathbf{A}(\mathbf{m})$ in the first tutorial, including appropriate implementations of the sampling operator $\mathbf{P}_r$ and source term $\mathbf{q}_s$.

To solve this optimization problem using a gradient-based method, we use the
adjoint-state method to evaluate the gradient $\nabla\Phi_s(\mathbf{m})$:

\begin{align}
 \nabla\Phi_s(\mathbf{m})=\sum_{\mathbf{t} =1}^{n_t}\mathbf{u}[\mathbf{t}] \mathbf{v}_{tt}[\mathbf{t}] =\mathbf{J}^T\delta\mathbf{d}_s,
\end{align}

where $n_t$ is the number of computational time steps, $\delta\mathbf{d}_s = \left(\mathbf{P}_r \mathbf{u} - \mathbf{d} \right)$ is the data residual (difference between the measured data and the modelled data), $\mathbf{J}$ is the Jacobian operator and $\mathbf{v}_{tt}$ is the second-order time derivative of the adjoint wavefield solving:

\begin{align}
 \mathbf{A}^T(\mathbf{m}) \mathbf{v} = \mathbf{P}_r^T \delta\mathbf{d}.
\end{align}

We see that the gradient of the FWI function is the previously defined imaging condition with an extra second-order time derivative. We will therefore reuse the operators defined previously inside a Devito wrapper.

## FWI gradient operator

To compute a single gradient $\nabla\Phi_s(\mathbf{m})$ in our optimization workflow we again use `solver.forward` to compute the entire forward wavefield $\mathbf{u}$ and a similar pre-defined gradient operator to compute the adjoint wavefield `v`. The gradient operator provided by our `solver` utility also computes the correlation between the wavefields, allowing us to encode a similar procedure to the previous imaging tutorial as our gradient calculation:

- Simulate the forward wavefield with the background velocity model to get the synthetic data and save the full wavefield $\mathbf{u}$
- Compute the data residual
- Back-propagate the data residual and compute on the fly the gradient contribution at each time step.

This procedure is applied to multiple source positions and summed to obtain a gradient image of the subsurface. We again prepare the source locations for each shot and visualize them, before defining a single gradient computation over a number of shots as a single function.

```python
#NBVAL_IGNORE_OUTPUT

# Prepare the varying source locations sources
source_locations = np.empty((nshots, 2), dtype=np.float32)
source_locations[:, 0] = 30.
source_locations[:, 1] = np.linspace(0., 1000, num=nshots)

plot_velocity(model, source=source_locations)
```

```python
# Create FWI gradient kernel
from devito import Function, clear_cache, TimeFunction
from examples.seismic import Receiver

import scipy
def fwi_gradient(m_in):
    # Create symbols to hold the gradient and residual
    grad = Function(name="grad", grid=model.grid)
    residual = Receiver(name='rec', grid=model.grid,
                        time_range=geometry.time_axis,
                        coordinates=geometry.rec_positions)
    objective = 0.

    # Creat forward wavefield to reuse to avoid memory overload
    u0 = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=4,
                      save=geometry.nt)
    for i in range(nshots):
        # Important: We force previous wavefields to be destroyed,
        # so that we may reuse the memory.
        clear_cache()

        # Update source location
        geometry.src_positions[0, :] = source_locations[i, :]

        # Generate synthetic data from true model
        true_d, _, _ = solver.forward(m=model.m)

        # Compute smooth data and full forward wavefield u0
        u0.data.fill(0.)
        smooth_d, _, _ = solver.forward(m=m_in, save=True, u=u0)

        # Compute gradient from data residual and update objective function
        residual.data[:] = smooth_d.data[:] - true_d.data[:]

        objective += .5*np.linalg.norm(residual.data.flatten())**2
        solver.gradient(rec=residual, u=u0, m=m_in, grad=grad)

    return objective, grad.data
```

Having defined our FWI gradient procedure we can compute the initial iteration from our starting model. This allows us to visualize the gradient alongside the model perturbation and the effect of the gradient update on the model.

```python
#NBVAL_IGNORE_OUTPUT

# Compute gradient of initial model
ff, update = fwi_gradient(model0.m)
print('Objective value is %f ' % ff)
```

    Objective value is 56971.650325

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import plot_image

# Plot the FWI gradient
plot_image(update, vmin=-1e4, vmax=1e4, cmap="jet")

# Plot the difference between the true and initial model.
# This is not known in practice as only the initial model is provided.
plot_image(model0.m.data - model.m.data, vmin=-1e-1, vmax=1e-1, cmap="jet")

# Show what the update does to the model
alpha = .005 / np.abs(update).max()
plot_image(model0.m.data - alpha*update, vmin=.1, vmax=.2, cmap="jet")
```

We see that the gradient and the true perturbation have the same sign, therefore, with an appropriate scaling factor, we will update the model in the correct direction.

```python
# Define bounding box constraints on the solution.
def apply_box_constraint(m):
    # Maximum possible 'realistic' velocity is 3.5 km/sec
    # Minimum possible 'realistic' velocity is 2 km/sec
    return np.clip(m, 1/3.5**2, 1/2**2)
```

```python
#NBVAL_SKIP

# Run FWI with gradient descent
history = np.zeros((fwi_iterations, 1))
for i in range(0, fwi_iterations):
    # Compute the functional value and gradient for the current
    # model estimate
    phi, direction = fwi_gradient(model0.m)

    # Store the history of the functional values
    history[i] = phi

    # Artificial Step length for gradient descent
    # In practice this would be replaced by a Linesearch (Wolfe, ...)
    # that would guarantee functional decrease Phi(m-alpha g) <= epsilon Phi(m)
    # where epsilon is a minimum decrease constant
    alpha = .005 / np.abs(direction).max()

    # Update the model estimate and inforce minimum/maximum values
    model0.m.data[:] = apply_box_constraint(model0.m.data - alpha * direction)

    # Log the progress made
    print('Objective value is %f at iteration %d' % (phi, i+1))
```

    Objective value is 56971.650325 at iteration 1
    Objective value is 39713.780163 at iteration 2
    Objective value is 26134.103267 at iteration 3
    Objective value is 16177.646433 at iteration 4
    Objective value is 9279.193459 at iteration 5

```python
#NBVAL_IGNORE_OUTPUT

# First, update velocity from computed square slowness
nbpml = model.nbpml
model0.vp = np.sqrt(1. / model0.m.data[nbpml:-nbpml, nbpml:-nbpml])

# Plot inverted velocity model
plot_velocity(model0)
```

```python
#NBVAL_SKIP
import matplotlib.pyplot as plt

# Plot objective function decrease
plt.figure()
plt.loglog(history)
plt.xlabel('Iteration number')
plt.ylabel('Misift value Phi')
plt.title('Convergence')
plt.show()
```

## References

[1] _Virieux, J. and Operto, S.: An overview of full-waveform inversion in exploration geophysics, GEOPHYSICS, 74, WCC1–WCC26, doi:10.1190/1.3238367, http://library.seg.org/doi/abs/10.1190/1.3238367, 2009._

[2] _Haber, E., Chung, M., and Herrmann, F. J.: An effective method for parameter estimation with PDE constraints with multiple right hand sides, SIAM Journal on Optimization, 22, http://dx.doi.org/10.1137/11081126X, 2012._

This notebook is part of the tutorial "Optimised Symbolic Finite Difference Computation with Devito" presented at the Intel® HPC Developer Conference 2017.


```python
DATASET_PATH = {
    'train': 'workdir/datasets/iris-train-2-s1.csv',
    'test': 'workdir/datasets/iris-test-2-s1.csv'
}
SAVE_RESULTS_PATH = "workdir/results/res1-new.results"
N_ESTIMATORS = 5
MIN_SAMPLES_SPLIT = 2
N_JOBS = 1
MAX_DEPTH = 5
SUBSPACES = 5
CV = 5
CV_REPEATS = 10
RF_TYPE = 'extraTrees' # 'randomForest'
SELECTION_METHODS = ['balanced_accuracy', 'accuracy', 'rf_accuracy', 'rf_balanced_accuracy', 'accuracy/accuracy_stddev'] # 'balanced_accuracy' # 'accuracy', 'f1_weighted'
PARALLELISM = "loky"
```

```python
import pickle
import numpy as np
import pandas as pd
from reduction.api import Instance, AdjacentOrNot, Relation, Rule, Features, Statement
from reduction.extract_rules import extract_rules
from reduction.measure_adjacencies import measure_rules
from toolz.curried import pipe, filter, map, reduce
from reduction.utils import bound_rule, join_consecutive_statements
from joblib import Parallel, delayed
from sklearn.metrics import recall_score, make_scorer, confusion_matrix
from sklearn.metrics import accuracy_score, balanced_accuracy_score, precision_score, recall_score, f1_score, make_scorer, confusion_matrix
from imblearn.metrics import geometric_mean_score
```

```python
from dask.distributed import Client
import joblib

if PARALLELISM == 'dask':
    parallel_backend = 'dask'
    client = Client("localhost:56436")
else:
    parallel_backend = 'loky'
```

## LOAD

```python
train_data = pd.read_csv(DATASET_PATH['train'])
test_data = pd.read_csv(DATASET_PATH['test'])
```

```python
x_train = train_data.drop('TARGET', axis=1).values
y_train = train_data['TARGET'].values
x_test = test_data.drop('TARGET', axis=1).values
y_test = test_data['TARGET'].values
```

```python
meta = {
        **DATASET_PATH,
       }
```

```python
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from reduction.classification import measure_acc, to_instance, RulesClassifier, measure_metric
```

```python
if RF_TYPE == 'randomForest':
    clf_rf = RandomForestClassifier(max_depth=MAX_DEPTH, n_estimators=N_ESTIMATORS, min_samples_split=MIN_SAMPLES_SPLIT, random_state=42)
elif RF_TYPE == 'extraTrees':
    clf_rf = ExtraTreesClassifier(max_depth=MAX_DEPTH, n_estimators=N_ESTIMATORS, min_samples_split=MIN_SAMPLES_SPLIT, random_state=42)
clf_rf.fit(x_train, y_train)
```

    ExtraTreesClassifier(max_depth=5, n_estimators=5, random_state=42)

```python
clf_dt = DecisionTreeClassifier(random_state=42, min_samples_split=MIN_SAMPLES_SPLIT, max_depth=MAX_DEPTH)
clf_dt.fit(x_train, y_train)
```

    DecisionTreeClassifier(max_depth=5, random_state=42)

```python
rf_test_predict = clf_rf.predict(x_test)
dt_test_predict = clf_dt.predict(x_test)
```

```python
meta = {
    **meta,
    "DT_test_acc": accuracy_score(y_test, dt_test_predict),
    "RF_test_acc": accuracy_score(y_test, rf_test_predict),
    "DT_test_confusion_matrix": confusion_matrix(y_test, dt_test_predict),
    "RF_test_confusion_matrix": confusion_matrix(y_test, rf_test_predict),
    "RF_test_f1": f1_score(y_test, rf_test_predict, average='weighted'),
    "DT_test_f1": f1_score(y_test, dt_test_predict, average='weighted'),
    "DT_test_gmean": geometric_mean_score(y_test, dt_test_predict, average='weighted'),
    "RF_test_gmean": geometric_mean_score(y_test, rf_test_predict, average='weighted'),
    "DT_test_balanced_accuracy": balanced_accuracy_score(y_test, dt_test_predict),
    "RF_test_balanced_accuracy": balanced_accuracy_score(y_test, rf_test_predict),
    "x_test": x_test,
    "RF_test_predictions": rf_test_predict,
    "DT_test_predictions": dt_test_predict,
    "SELECTION_METHODS": SELECTION_METHODS,
    "CV": CV,
    "RF_TYPE": RF_TYPE,
    "CV_REPEATS": CV_REPEATS
}
```

```python
%%time

all_rules = pipe(
    clf_rf.estimators_,
    map(lambda estimator: extract_rules(estimator)),
    reduce(set.union),
    map(lambda r: join_consecutive_statements(r)),
#     map(lambda r: bound_rule(r, x_train)),
    list
)

```

    CPU times: user 39.1 ms, sys: 0 ns, total: 39.1 ms
    Wall time: 37.9 ms

```python
all_statements = pipe(
    all_rules,
    set,
    map(lambda r: list(r.statements)),
    reduce(list.__add__),
)
```

# Find all non-overlapping rule cliques

```python
import networkx as nx
from networkx.algorithms.clique import enumerate_all_cliques, find_cliques
```

```python
g = nx.Graph()
```

```python
%%time

with joblib.parallel_backend(parallel_backend):
    all_rule_measurements = measure_rules(all_rules, n_jobs=N_JOBS)
```

    CPU times: user 69 ms, sys: 1.3 ms, total: 70.3 ms
    Wall time: 68.6 ms

```python
def add_to_graph(measurements_tuple, measurement):

    if measurement == AdjacentOrNot.NOT_ADJACENT:
        rule_1, rule_2 = measurements_tuple
        rule_idx_1 = all_rules.index(rule_1)
        rule_idx_2 = all_rules.index(rule_2)

        g.add_node(rule_idx_1)

        g.add_node(rule_idx_2)

        g.add_edge(rule_idx_1, rule_idx_2)
```

```python
%%time
for measurements_tuple, measurement in all_rule_measurements.items():

    add_to_graph(measurements_tuple, measurement)
```

    CPU times: user 35.3 ms, sys: 904 µs, total: 36.2 ms
    Wall time: 34.7 ms

```python
%%time
all_subspaces = list(filter(lambda s: len(s) <= SUBSPACES)(find_cliques(g)))

subspaces_to_check = all_subspaces

for clique_size in reversed(range(1, SUBSPACES + 1)):
    subspaces_by_size_of_param = list(filter(lambda s: len(s) == SUBSPACES)(all_subspaces))
    print(f"clique = {clique_size}")
    if not len(subspaces_by_size_of_param) == 0:
        subspaces_to_check = subspaces_by_size_of_param
        break
```

    clique = 5
    CPU times: user 4.6 ms, sys: 0 ns, total: 4.6 ms
    Wall time: 4.47 ms

```python
if not subspaces_to_check:
    subspaces_to_check = [[r_idx] for r_idx in list(range(len(all_rules)))]
```

```python
len(subspaces_to_check)
```

    24

```python
meta = {
    **meta,
    'total_rules' : len(all_rules),
    'total_subspaces_to_check': len(subspaces_to_check),
#     'clique_size': clique_size
}
```

```python
meta
```

    {'train': 'workdir/datasets/iris-train-2-s1.csv',
     'test': 'workdir/datasets/iris-test-2-s1.csv',
     'DT_test_acc': 0.94,
     'RF_test_acc': 0.96,
     'DT_test_confusion_matrix': array([[17,  0,  0],
            [ 0, 15,  2],
            [ 0,  1, 15]]),
     'RF_test_confusion_matrix': array([[17,  0,  0],
            [ 0, 15,  2],
            [ 0,  0, 16]]),
     'RF_test_f1': 0.9599264705882353,
     'DT_test_f1': 0.94,
     'DT_test_gmean': 0.9553120086485319,
     'RF_test_gmean': 0.970530479565019,
     'DT_test_balanced_accuracy': 0.9399509803921569,
     'RF_test_balanced_accuracy': 0.9607843137254902,
     'x_test': array([[5.1, 3.5, 1.4, 0.2],
            [4.8, 3. , 1.4, 0.1],
            [4.3, 3. , 1.1, 0.1],
            [5.4, 3.9, 1.3, 0.4],
            [5.1, 3.5, 1.4, 0.3],
            [5.7, 3.8, 1.7, 0.3],
            [4.6, 3.6, 1. , 0.2],
            [5.1, 3.3, 1.7, 0.5],
            [4.8, 3.4, 1.9, 0.2],
            [5.2, 3.4, 1.4, 0.2],
            [5.2, 4.1, 1.5, 0.1],
            [4.9, 3.1, 1.5, 0.1],
            [5.5, 3.5, 1.3, 0.2],
            [4.4, 3. , 1.3, 0.2],
            [4.8, 3. , 1.4, 0.3],
            [6.9, 3.1, 4.9, 1.5],
            [5. , 2. , 3.5, 1. ],
            [5.9, 3. , 4.2, 1.5],
            [5.8, 2.7, 4.1, 1. ],
            [5.9, 3.2, 4.8, 1.8],
            [6.7, 3. , 5. , 1.7],
            [5.7, 2.6, 3.5, 1. ],
            [5.8, 2.7, 3.9, 1.2],
            [5.4, 3. , 4.5, 1.5],
            [5.5, 2.6, 4.4, 1.2],
            [5.8, 2.6, 4. , 1.2],
            [5.6, 2.7, 4.2, 1.3],
            [5.7, 3. , 4.2, 1.2],
            [6.2, 2.9, 4.3, 1.3],
            [5.7, 2.8, 4.1, 1.3],
            [6.3, 3.3, 6. , 2.5],
            [7.1, 3. , 5.9, 2.1],
            [7.3, 2.9, 6.3, 1.8],
            [7.2, 3.6, 6.1, 2.5],
            [6.4, 2.7, 5.3, 1.9],
            [6.8, 3. , 5.5, 2.1],
            [5.8, 2.8, 5.1, 2.4],
            [6.7, 3.3, 5.7, 2.1],
            [7.2, 3.2, 6. , 1.8],
            [7.2, 3. , 5.8, 1.6],
            [7.7, 3. , 6.1, 2.3],
            [6.9, 3.1, 5.1, 2.3],
            [6.8, 3.2, 5.9, 2.3],
            [6.7, 3.3, 5.7, 2.5],
            [5.9, 3. , 5.1, 1.8],
            [4.7, 3.2, 1.3, 0.2],
            [5.8, 4. , 1.2, 0.2],
            [6.8, 2.8, 4.8, 1.4],
            [6. , 3.4, 4.5, 1.6],
            [6.9, 3.2, 5.7, 2.3]]),
     'RF_test_predictions': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 1,
            1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
            2, 0, 0, 1, 1, 2]),
     'DT_test_predictions': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 1,
            1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2,
            2, 0, 0, 1, 1, 2]),
     'SELECTION_METHODS': ['balanced_accuracy',
      'accuracy',
      'rf_accuracy',
      'rf_balanced_accuracy',
      'accuracy/accuracy_stddev'],
     'CV': 5,
     'RF_TYPE': 'extraTrees',
     'CV_REPEATS': 10,
     'total_rules': 50,
     'total_subspaces_to_check': 24}

```python
from copy import deepcopy
from collections import defaultdict
from reduction.classification import SubspaceRulesClassifier
from sklearn.model_selection import cross_val_score, cross_validate
from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, RepeatedKFold, RepeatedStratifiedKFold, StratifiedShuffleSplit
from imblearn.metrics import geometric_mean_score, sensitivity_score, specificity_score
from sklearn.metrics import recall_score, make_scorer, confusion_matrix, f1_score, precision_score, balanced_accuracy_score, cohen_kappa_score
from reduction.rule_measures import BayesianRuleMeasures, covered_by_statements
from sympy import parse_expr, evalf
```

```python
from scipy.stats import entropy
classes_count = len(np.unique(y_train))

def calculate_entropy(y, classes_count):
    counts = np.unique(y, return_counts=True)[1]
    return entropy(counts, base=classes_count)
```

```python
def accuracy_with_rf(estimator, X, y):
    y_rf = clf_rf.predict(X)
    y_model = estimator.predict(X)

    return accuracy_score(y_rf, y_model)
```

```python
def bal_accuracy_with_rf(estimator, X, y):
    y_rf = clf_rf.predict(X)
    y_model = estimator.predict(X)

    return balanced_accuracy_score(y_rf, y_model)
```

```python
def rf_kohen_cappa(estimator, X, y):
    y_rf = clf_rf.predict(X)
    y_model = estimator.predict(X)

    return cohen_kappa_score(y_rf, y_model)
```

```python
def get_val(clique, x_train, y_train, x_test, y_test):

    rules = np.array(all_rules)[clique]
    clf = SubspaceRulesClassifier(rules=rules, max_depth=MAX_DEPTH, random_state=42)

    all_scores = defaultdict(list)
    all_additional_scores = defaultdict(list)
    skf = ShuffleSplit(n_splits=CV, test_size=0.5,random_state=42)
    with joblib.parallel_backend('threading'):
        scores = cross_validate(clf, x_train, y_train, n_jobs=1, scoring={
            'balanced_accuracy': 'balanced_accuracy',
            'f1': 'f1_weighted',
            'accuracy': 'accuracy',
            'g_mean': make_scorer(geometric_mean_score, average='weighted'),
            'recall': 'recall_weighted',
            'precision': 'precision_weighted',
            'rf_accuracy': accuracy_with_rf,
            'rf_balanced_accuracy': bal_accuracy_with_rf,
            'rf_cohen_kappa': rf_kohen_cappa
        }, cv=skf)

    additional_scores = defaultdict(list)

    skf = ShuffleSplit(n_splits=CV, test_size=0.5, random_state=42)
    for train_index, test_index in skf.split(x_train, y_train):
        x_train_split = x_train[train_index]
        y_train_split = y_train[train_index]

        mean_rule_scores = defaultdict(list)
        for rule in rules:
            covered_indicies = list(covered_by_statements(rule, x_train_split))

            rule_measurements = BayesianRuleMeasures.create(rule, x_train_split, y_train_split)
            mean_rule_scores['a'].append(rule_measurements.a())
            mean_rule_scores['b'].append(rule_measurements.b())
            mean_rule_scores['c'].append(rule_measurements.c())
            mean_rule_scores['d'].append(rule_measurements.d())
            mean_rule_scores['s'].append(rule_measurements.s_measure())
            mean_rule_scores['n'].append(rule_measurements.n_measure())
            mean_rule_scores['entropy'].append(calculate_entropy(y_train_split[covered_indicies], classes_count))

        for score_name, this_score in dict(mean_rule_scores).items():
            additional_scores[score_name].append(np.mean(this_score))

    final_clf = SubspaceRulesClassifier(rules=rules, max_depth=MAX_DEPTH, random_state=42)
    final_clf.fit(x_train, y_train)
    final_clf_y_pred = final_clf.predict(x_test)
    final_model_test_accuracy = accuracy_score(y_test, final_clf_y_pred)
    final_model_confusion_matrix = confusion_matrix(y_test, final_clf_y_pred)

    scores = {
        **scores,
        **dict(additional_scores),
    }
    scores_without_test_preffix = {
        **{k[5:] if k.startswith('test_') else k: np.mean(v) for k, v in scores.items()},
        **{f"{k[5:]}_stddev" if k.startswith('test_') else f"{k}_stddev": np.std(v) for k, v in scores.items()},
    }

    score_by_selection_method = {
        'score ' + method: float(parse_expr(method).evalf(subs=scores_without_test_preffix)) for method in SELECTION_METHODS
    }

    return {
        **scores_without_test_preffix,
        **score_by_selection_method,
        'final_model_test_accuracy': final_model_test_accuracy,
        'final_model_test_predictions': final_clf_y_pred,
        'final_model_confusion_matrix': final_model_confusion_matrix,
        'final_model_used_trees': len(final_clf._clf_by_rule),
        'final_model_test_f1_score': f1_score(y_test, final_clf_y_pred, average='weighted'),
        'final_model_test_g_mean': geometric_mean_score(y_test, final_clf_y_pred, average='weighted'),
        'final_model_test_recall_score': recall_score(y_test, final_clf_y_pred, average='weighted'),
        'final_model_test_precision_score': precision_score(y_test, final_clf_y_pred, average='weighted'),
        'final_model_test_balanced_accuracy_score': balanced_accuracy_score(y_test, final_clf_y_pred),
        'final_model_original_rf_fidelity': accuracy_score(rf_test_predict, final_clf_y_pred)
    }
```

```python
import warnings
warnings.filterwarnings('ignore')
```

```python
# import contextlib
# import joblib
# from tqdm.notebook import tqdm
# from joblib import Parallel, delayed

# @contextlib.contextmanager
# def tqdm_joblib(tqdm_object):
#     """Context manager to patch joblib to report into tqdm progress bar given as argument"""
#     class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
#         def __init__(self, *args, **kwargs):
#             super().__init__(*args, **kwargs)

#         def __call__(self, *args, **kwargs):
#             tqdm_object.update(n=self.batch_size)
#             return super().__call__(*args, **kwargs)

#     old_batch_callback = joblib.parallel.BatchCompletionCallBack
#     joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
#     try:
#         yield tqdm_object
#     finally:
#         joblib.parallel.BatchCompletionCallBack = old_batch_callback
#         tqdm_object.close()

```

```python
import time
scoring_start = time.time()
```

```python
%%time
with joblib.parallel_backend(parallel_backend):
#     with tqdm_joblib(tqdm(desc="My calculation", total=len(subspaces_to_check))) as progress_bar:
    score_by_subspace = \
        dict(zip(
            map(tuple)(subspaces_to_check),
            Parallel(n_jobs=N_JOBS)(delayed(lambda subspace: get_val(subspace, x_train, y_train, x_test, y_test))(subspace) for subspace in subspaces_to_check)
        ))

```

    CPU times: user 7.8 s, sys: 602 ms, total: 8.4 s
    Wall time: 7.91 s

```python
# score_by_subspace_as_list = list(score_by_subspace.items())
```

```python
# vectorized_results = np.array([
#     [
#         scores['balanced_accuracy'],
#         scores['f1'],
#         scores['accuracy'],
#         scores['g_mean'],
#         scores['recall'],
#         scores['precision'],
#         scores['a'],
#         scores['b'],
#         scores['c'],
#         scores['d'],
#         scores['s'],
#         scores['n']

#     ] for scores in score_by_subspace.values()
# ])
```

```python
# def closest_node(node, nodes):
#     nodes = np.asarray(nodes)
#     dist_2 = np.sum((nodes - node)**2, axis=1)
#     return np.argmin(dist_2)
```

```python
# closest_node(np.array(current_value), vectorized_results)
```

```python
# current_value = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 5, 5, 10, 10, 0.5, 0.5]
# for rules, scores in score_by_subspace.items():

```

```python
# import numpy as np
# from pymoo.model.problem import Problem

# class MyProblem(Problem):

#     def __init__(self):
#         super().__init__(n_var=12,
#                          n_obj=4,
#                          n_constr=0,
#                          xl=np.array([0,0,0,0,0,0,0,0,0,0,0,0]),
#                          xu=np.array([1,1,1,1,1,1, len(x_train), len(x_train), len(x_train), len(x_train), 1, 1]))

#     def _evaluate(self, X, out, *args, **kwargs):
#         closest_points = [
#             closest_node(x, vectorized_results) for x in X
#         ]

#         f1 = np.array([
#             1 - score_by_subspace_as_list[point][1]['recall'] for point in closest_points
#         ])
#         f2 = np.array([
#             1 - score_by_subspace_as_list[point][1]['precision'] for point in closest_points
#         ])
#         f3 = np.array([
#             1 - score_by_subspace_as_list[point][1]['g_mean'] for point in closest_points
#         ])
#         f4 = np.array([
#             1 - score_by_subspace_as_list[point][1]['balanced_accuracy'] for point in closest_points
#         ])

#         out["F"] = np.column_stack([f1, f2, f3, f4])
# #         out["G"] = np.column_stack([g1, g2])

# vectorized_problem = MyProblem()

```

```python
# from pymoo.algorithms.nsga2 import NSGA2
# from pymoo.factory import get_sampling, get_crossover, get_mutation
# from pymoo.factory import get_termination
# from pymoo.optimize import minimize

# algorithm = NSGA2(
#     pop_size=100,
#     n_offsprings=20,
#     sampling=get_sampling("real_random"),
#     crossover=get_crossover("real_sbx", prob=0.9, eta=15),
#     mutation=get_mutation("real_pm", eta=20),
#     eliminate_duplicates=True
# )

# termination = get_termination("n_gen", 120)
```

```python
# res = minimize(vectorized_problem,
#                algorithm,
#                termination,
#                seed=1,
#                save_history=True,
#                verbose=True)
```

```python
# %%time
# scoring_start = time.time()
# score_by_subspace = {
#     tuple(subspace): get_val(subspace, x_train, y_train, x_test, y_test) for subspace in tqdm(subspaces_to_check)
# }

```

```python
scoring_end = time.time()
scoring_time = scoring_end - scoring_start
```

```python
top_rules = max(score_by_subspace, key=lambda s: score_by_subspace[s]['final_model_test_accuracy'])
```

```python
top = score_by_subspace[top_rules]
```

```python
scores_by_selection_method = {}
for selection_method in SELECTION_METHODS:
    best_score = max([s[f'score {selection_method}'] for s in score_by_subspace.values()])
    best_score_rules = [rules for rules, val in score_by_subspace.items() if val[f'score {selection_method}'] == best_score]

    best_score_rules_with_scoring = {
        rules: score_by_subspace[rules] for rules in best_score_rules
    }
    worst = best_score_rules_with_scoring[min(best_score_rules_with_scoring, key=lambda v: best_score_rules_with_scoring[v]['accuracy'])]
    best = best_score_rules_with_scoring[max(best_score_rules_with_scoring, key=lambda v: best_score_rules_with_scoring[v]['accuracy'])]

    scores_by_selection_method[selection_method] = {
        **{f'worst_{k}': v for k, v in worst.items()},
        **{f'best_{k}': v for k, v in worst.items()},
        'found': len(best_score_rules)
    }

```

```python
results = {
    **meta,
    'scoring_time': scoring_time,
    **scores_by_selection_method,
    **{f'top_{k}': v for k, v in top.items()},
    'found_rules': len(best_score_rules_with_scoring),
    'all_subspaces_with_score': score_by_subspace
}
```

```python
if PARALLELISM == 'dask':
    client.close()
```

```python
with open(SAVE_RESULTS_PATH, 'wb') as file:
    pickle.dump(results, file)
```

```python
from joblib.externals.loky import get_reusable_executor
get_reusable_executor().shutdown(wait=True)
```