# Model Application

## 1. Importing Libraries

- Importing the necessary libraries to deal with the data. We are going to need **NumPy**, **Pandas** and **Pathlib** to import the data. The **Tensorflow** library is going to be used to apply the model and to plot the data, we are going to use **Matplotlib** and **Seaborn**.

```python
# Libraries to deal with the data
import numpy as np
import pandas as pd
import os
import tempfile
import pathlib
import random

# Libraries to work neural networks
import tensorflow as tf
from tensorflow import keras

# Libraries to plot the results
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
sns.set()
```

```python
mpl.rcParams['figure.figsize'] = (12, 10)
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
```

## 2. Importing Preprocessed Data

- Defining the **file parameters** to import the preprocessed data.

```python
# File parameters

path = str(pathlib.Path().absolute())
path_dataset = path + '\\Datasets\\'
path_figures = path + '\\Figures\\'
path_result = path + '\\Result\\'
```

- Importing the preprocessed **down-sampled 50/50** dataset.

```python
# Train dataset
npz_ds = np.load(path_result + 'DS_5050_backorder_data_train.npz')

train_inputs_ds_5050 = npz_ds['inputs'].astype(np.float)
train_targets_ds_5050 = npz_ds['targets'].astype(np.int)

# Validation dataset
npz_ds = np.load(path_result + 'DS_5050_backorder_data_validation.npz')

validation_inputs_ds_5050 = npz_ds['inputs'].astype(np.float)
validation_targets_ds_5050 = npz_ds['targets'].astype(np.int)

# Single test dataset
npz_ds = np.load(path_result + 'DS_5050_backorder_data_test.npz')

test_inputs_ds_5050 = npz_ds['inputs'].astype(np.float)
test_targets_ds_5050 = npz_ds['targets'].astype(np.int)
```

- Importing the preprocessed **down-sampled 05/95** dataset.

```python
# Train dataset
npz_ds = np.load(path_result + 'DS_0595_backorder_data_train.npz')

train_inputs_ds_0595 = npz_ds['inputs'].astype(np.float)
train_targets_ds_0595 = npz_ds['targets'].astype(np.int)

# Validation dataset
npz_ds = np.load(path_result + 'DS_0595_backorder_data_validation.npz')

validation_inputs_ds_0595 = npz_ds['inputs'].astype(np.float)
validation_targets_ds_0595 = npz_ds['targets'].astype(np.int)

# Single test dataset
npz_ds = np.load(path_result + 'DS_0595_backorder_data_test.npz')

test_inputs_ds_0595 = npz_ds['inputs'].astype(np.float)
test_targets_ds_0595 = npz_ds['targets'].astype(np.int)
```

- Importing the preprocessed **up-sampled** dataset.

```python
# Train dataset
npz_us = np.load(path_result + 'US_backorder_data_train.npz')

train_inputs_us = npz_us['inputs'].astype(np.float)
train_targets_us = npz_us['targets'].astype(np.int)

# Validation dataset
npz_us = np.load(path_result + 'US_backorder_data_validation.npz')

validation_inputs_us = npz_us['inputs'].astype(np.float)
validation_targets_us = npz_us['targets'].astype(np.int)

# Single test dataset
npz_us = np.load(path_result + 'US_backorder_data_test.npz')

test_inputs_us = npz_us['inputs'].astype(np.float)
test_targets_us = npz_us['targets'].astype(np.int)
```

- Reimporting the **test_for_all** dataset.

```python
# Test for all dataset will be evaluated by all trained models.
# This dataset was separated before the preprocessing phase.

npz_tfa = np.load(path_result + 'TFA_backorder_data_gral_test.npz')

test_inputs_tfa = npz_tfa['inputs'].astype(np.float)
test_targets_tfa = npz_tfa['targets'].astype(np.int)
```

## 3. Defining Model

- Defining **hyperparameters** for the model. These parameters are going to be used by all trained models.

```python
# Number of features
input_size = 29

# Number of targets
# output_size = 2 # Softmax activation function
output_size = 1 # Sigmoid activation function

# Size of Hidden Layers
hidden_layer_size = 15
hidden_layer_size_2 = 15
```

- Defining the ANN model to train using the 3 sampled datasets (**Down-sampled 50/50**, **Down-sampled 05/95** and **Upsampled**).

```python
# Defining the Sequential model with 2 Hidden layers.
# Using "relu" activation funtion of Hidden layers.
# Using "sigmoid" activation function for output layer.

METRICS = [
      keras.metrics.TruePositives(name='tp'),
      keras.metrics.FalsePositives(name='fp'),
      keras.metrics.TrueNegatives(name='tn'),
      keras.metrics.FalseNegatives(name='fn'),
      keras.metrics.BinaryAccuracy(name='accuracy'),
      keras.metrics.Precision(name='precision'),
      keras.metrics.Recall(name='recall'),
      keras.metrics.AUC(name='auc'),
]

def make_model(metrics = METRICS, output_bias=None):
  if output_bias is not None:
    output_bias = tf.keras.initializers.Constant(output_bias)
  model = keras.Sequential([
      keras.layers.Dense(
          hidden_layer_size, activation='relu',
          input_shape=(train_inputs_ds_0595.shape[-1],)),  # all datasets have the same number of inputs
      keras.layers.Dense(
          hidden_layer_size_2, activation='relu'),
      # keras.layers.Dropout(0.5),
      keras.layers.Dense(output_size, activation='sigmoid',
                         bias_initializer=output_bias),
  ])

# Defining the optimizer and the loss-function for the model.

# Using ADAM which is an enhancement of Stocatic Gradient Descendent (SGD).
# Using metrics relevant for this kind of classification problems

  model.compile(
      optimizer=keras.optimizers.Adam(lr=1e-3),
      loss=keras.losses.BinaryCrossentropy(),
      metrics=metrics)

  return model

```

- Defining **hyperparameters** to fit the models with train data.

```python
# set the batch size, but we might not need this
BATCH_SIZE = 4000

# set a maximum number of training epochs
EPOCHS = 150
```

- Applying a early stopping mechanism to avoid **overfitting** of the model.

```python
# Set an early stopping mechanism
# Set patience=10, to be a bit tolerant against random validation loss increases
early_stopping = tf.keras.callbacks.EarlyStopping(
    monitor='val_auc',
    verbose=1,
    patience=10,
    mode='max',
    restore_best_weights=True)
```

```python
model = make_model()
model.summary()
```

    Model: "sequential"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense (Dense)                (None, 15)                450
    _________________________________________________________________
    dense_1 (Dense)              (None, 15)                240
    _________________________________________________________________
    dense_2 (Dense)              (None, 1)                 16
    =================================================================
    Total params: 706
    Trainable params: 706
    Non-trainable params: 0
    _________________________________________________________________

## 4. Training and Testing Model

- Training the model using the **down-sampled 50/50** dataset.

```python
model = make_model()
baseline_history = model.fit(
    train_inputs_ds_5050,  # train inputs
    train_targets_ds_5050, # train targets
    # batch_size=BATCH_SIZE, # batch size
    epochs=EPOCHS,         # epochs that we will train for (assuming early stopping doesn't kick in)
    callbacks = [early_stopping],  # early stopping
    validation_data=(validation_inputs_ds_5050, validation_targets_ds_5050) # validation data
)

```

    Epoch 1/150
    400/400 [==============================] - 3s 7ms/step - loss: 0.6480 - tp: 3424.0000 - fp: 1609.0000 - tn: 4717.0000 - fn: 3050.0000 - accuracy: 0.6360 - precision: 0.6803 - recall: 0.5289 - auc: 0.6902 - val_loss: 0.6278 - val_tp: 515.0000 - val_fp: 300.0000 - val_tn: 543.0000 - val_fn: 242.0000 - val_accuracy: 0.6612 - val_precision: 0.6319 - val_recall: 0.6803 - val_auc: 0.7105
    Epoch 2/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.6111 - tp: 4542.0000 - fp: 2169.0000 - tn: 4157.0000 - fn: 1932.0000 - accuracy: 0.6796 - precision: 0.6768 - recall: 0.7016 - auc: 0.7309 - val_loss: 0.6084 - val_tp: 510.0000 - val_fp: 254.0000 - val_tn: 589.0000 - val_fn: 247.0000 - val_accuracy: 0.6869 - val_precision: 0.6675 - val_recall: 0.6737 - val_auc: 0.7339
    Epoch 3/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5934 - tp: 4580.0000 - fp: 1992.0000 - tn: 4334.0000 - fn: 1894.0000 - accuracy: 0.6964 - precision: 0.6969 - recall: 0.7074 - auc: 0.7516 - val_loss: 0.6001 - val_tp: 537.0000 - val_fp: 279.0000 - val_tn: 564.0000 - val_fn: 220.0000 - val_accuracy: 0.6881 - val_precision: 0.6581 - val_recall: 0.7094 - val_auc: 0.7445
    Epoch 4/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5812 - tp: 4635.0000 - fp: 1922.0000 - tn: 4404.0000 - fn: 1839.0000 - accuracy: 0.7062 - precision: 0.7069 - recall: 0.7159 - auc: 0.7646 - val_loss: 0.5890 - val_tp: 519.0000 - val_fp: 236.0000 - val_tn: 607.0000 - val_fn: 238.0000 - val_accuracy: 0.7038 - val_precision: 0.6874 - val_recall: 0.6856 - val_auc: 0.7577
    Epoch 5/150
    400/400 [==============================] - 2s 5ms/step - loss: 0.5735 - tp: 4692.0000 - fp: 1889.0000 - tn: 4437.0000 - fn: 1782.0000 - accuracy: 0.7132 - precision: 0.7130 - recall: 0.7247 - auc: 0.7730 - val_loss: 0.5871 - val_tp: 486.0000 - val_fp: 189.0000 - val_tn: 654.0000 - val_fn: 271.0000 - val_accuracy: 0.7125 - val_precision: 0.7200 - val_recall: 0.6420 - val_auc: 0.7714
    Epoch 6/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5673 - tp: 4685.0000 - fp: 1804.0000 - tn: 4522.0000 - fn: 1789.0000 - accuracy: 0.7193 - precision: 0.7220 - recall: 0.7237 - auc: 0.7794 - val_loss: 0.5811 - val_tp: 525.0000 - val_fp: 227.0000 - val_tn: 616.0000 - val_fn: 232.0000 - val_accuracy: 0.7131 - val_precision: 0.6981 - val_recall: 0.6935 - val_auc: 0.7767
    Epoch 7/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5639 - tp: 4680.0000 - fp: 1817.0000 - tn: 4509.0000 - fn: 1794.0000 - accuracy: 0.7179 - precision: 0.7203 - recall: 0.7229 - auc: 0.7820 - val_loss: 0.5861 - val_tp: 517.0000 - val_fp: 214.0000 - val_tn: 629.0000 - val_fn: 240.0000 - val_accuracy: 0.7163 - val_precision: 0.7073 - val_recall: 0.6830 - val_auc: 0.7792
    Epoch 8/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5598 - tp: 4730.0000 - fp: 1800.0000 - tn: 4526.0000 - fn: 1744.0000 - accuracy: 0.7231 - precision: 0.7243 - recall: 0.7306 - auc: 0.7862 - val_loss: 0.5869 - val_tp: 535.0000 - val_fp: 231.0000 - val_tn: 612.0000 - val_fn: 222.0000 - val_accuracy: 0.7169 - val_precision: 0.6984 - val_recall: 0.7067 - val_auc: 0.7752
    Epoch 9/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5563 - tp: 4695.0000 - fp: 1749.0000 - tn: 4577.0000 - fn: 1779.0000 - accuracy: 0.7244 - precision: 0.7286 - recall: 0.7252 - auc: 0.7894 - val_loss: 0.5830 - val_tp: 531.0000 - val_fp: 214.0000 - val_tn: 629.0000 - val_fn: 226.0000 - val_accuracy: 0.7250 - val_precision: 0.7128 - val_recall: 0.7015 - val_auc: 0.7860
    Epoch 10/150
    400/400 [==============================] - 1s 4ms/step - loss: 0.5543 - tp: 4715.0000 - fp: 1730.0000 - tn: 4596.0000 - fn: 1759.0000 - accuracy: 0.7274 - precision: 0.7316 - recall: 0.7283 - auc: 0.7914 - val_loss: 0.5840 - val_tp: 566.0000 - val_fp: 265.0000 - val_tn: 578.0000 - val_fn: 191.0000 - val_accuracy: 0.7150 - val_precision: 0.6811 - val_recall: 0.7477 - val_auc: 0.7848
    Epoch 11/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5520 - tp: 4758.0000 - fp: 1784.0000 - tn: 4542.0000 - fn: 1716.0000 - accuracy: 0.7266 - precision: 0.7273 - recall: 0.7349 - auc: 0.7936 - val_loss: 0.5794 - val_tp: 497.0000 - val_fp: 176.0000 - val_tn: 667.0000 - val_fn: 260.0000 - val_accuracy: 0.7275 - val_precision: 0.7385 - val_recall: 0.6565 - val_auc: 0.7903
    Epoch 12/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5494 - tp: 4722.0000 - fp: 1712.0000 - tn: 4614.0000 - fn: 1752.0000 - accuracy: 0.7294 - precision: 0.7339 - recall: 0.7294 - auc: 0.7957 - val_loss: 0.5782 - val_tp: 511.0000 - val_fp: 193.0000 - val_tn: 650.0000 - val_fn: 246.0000 - val_accuracy: 0.7256 - val_precision: 0.7259 - val_recall: 0.6750 - val_auc: 0.7938
    Epoch 13/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5475 - tp: 4748.0000 - fp: 1672.0000 - tn: 4654.0000 - fn: 1726.0000 - accuracy: 0.7345 - precision: 0.7396 - recall: 0.7334 - auc: 0.7974 - val_loss: 0.5811 - val_tp: 527.0000 - val_fp: 211.0000 - val_tn: 632.0000 - val_fn: 230.0000 - val_accuracy: 0.7244 - val_precision: 0.7141 - val_recall: 0.6962 - val_auc: 0.7932
    Epoch 14/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5455 - tp: 4748.0000 - fp: 1673.0000 - tn: 4653.0000 - fn: 1726.0000 - accuracy: 0.7345 - precision: 0.7394 - recall: 0.7334 - auc: 0.7997 - val_loss: 0.5769 - val_tp: 540.0000 - val_fp: 211.0000 - val_tn: 632.0000 - val_fn: 217.0000 - val_accuracy: 0.7325 - val_precision: 0.7190 - val_recall: 0.7133 - val_auc: 0.7947
    Epoch 15/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5423 - tp: 4757.0000 - fp: 1666.0000 - tn: 4660.0000 - fn: 1717.0000 - accuracy: 0.7357 - precision: 0.7406 - recall: 0.7348 - auc: 0.8028 - val_loss: 0.5785 - val_tp: 559.0000 - val_fp: 231.0000 - val_tn: 612.0000 - val_fn: 198.0000 - val_accuracy: 0.7319 - val_precision: 0.7076 - val_recall: 0.7384 - val_auc: 0.7963
    Epoch 16/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5411 - tp: 4784.0000 - fp: 1673.0000 - tn: 4653.0000 - fn: 1690.0000 - accuracy: 0.7373 - precision: 0.7409 - recall: 0.7390 - auc: 0.8033 - val_loss: 0.5753 - val_tp: 522.0000 - val_fp: 182.0000 - val_tn: 661.0000 - val_fn: 235.0000 - val_accuracy: 0.7394 - val_precision: 0.7415 - val_recall: 0.6896 - val_auc: 0.7991
    Epoch 17/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5400 - tp: 4752.0000 - fp: 1646.0000 - tn: 4680.0000 - fn: 1722.0000 - accuracy: 0.7369 - precision: 0.7427 - recall: 0.7340 - auc: 0.8049 - val_loss: 0.5800 - val_tp: 563.0000 - val_fp: 233.0000 - val_tn: 610.0000 - val_fn: 194.0000 - val_accuracy: 0.7331 - val_precision: 0.7073 - val_recall: 0.7437 - val_auc: 0.8007
    Epoch 18/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5382 - tp: 4827.0000 - fp: 1655.0000 - tn: 4671.0000 - fn: 1647.0000 - accuracy: 0.7420 - precision: 0.7447 - recall: 0.7456 - auc: 0.8062 - val_loss: 0.5783 - val_tp: 534.0000 - val_fp: 198.0000 - val_tn: 645.0000 - val_fn: 223.0000 - val_accuracy: 0.7369 - val_precision: 0.7295 - val_recall: 0.7054 - val_auc: 0.7967
    Epoch 19/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5366 - tp: 4816.0000 - fp: 1658.0000 - tn: 4668.0000 - fn: 1658.0000 - accuracy: 0.7409 - precision: 0.7439 - recall: 0.7439 - auc: 0.8078 - val_loss: 0.5769 - val_tp: 525.0000 - val_fp: 188.0000 - val_tn: 655.0000 - val_fn: 232.0000 - val_accuracy: 0.7375 - val_precision: 0.7363 - val_recall: 0.6935 - val_auc: 0.8003
    Epoch 20/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5354 - tp: 4773.0000 - fp: 1632.0000 - tn: 4694.0000 - fn: 1701.0000 - accuracy: 0.7396 - precision: 0.7452 - recall: 0.7373 - auc: 0.8085 - val_loss: 0.5739 - val_tp: 520.0000 - val_fp: 180.0000 - val_tn: 663.0000 - val_fn: 237.0000 - val_accuracy: 0.7394 - val_precision: 0.7429 - val_recall: 0.6869 - val_auc: 0.8048
    Epoch 21/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5337 - tp: 4831.0000 - fp: 1631.0000 - tn: 4695.0000 - fn: 1643.0000 - accuracy: 0.7442 - precision: 0.7476 - recall: 0.7462 - auc: 0.8103 - val_loss: 0.5757 - val_tp: 544.0000 - val_fp: 192.0000 - val_tn: 651.0000 - val_fn: 213.0000 - val_accuracy: 0.7469 - val_precision: 0.7391 - val_recall: 0.7186 - val_auc: 0.8031
    Epoch 22/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5321 - tp: 4843.0000 - fp: 1635.0000 - tn: 4691.0000 - fn: 1631.0000 - accuracy: 0.7448 - precision: 0.7476 - recall: 0.7481 - auc: 0.8117 - val_loss: 0.5733 - val_tp: 561.0000 - val_fp: 218.0000 - val_tn: 625.0000 - val_fn: 196.0000 - val_accuracy: 0.7412 - val_precision: 0.7202 - val_recall: 0.7411 - val_auc: 0.8039
    Epoch 23/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5313 - tp: 4856.0000 - fp: 1629.0000 - tn: 4697.0000 - fn: 1618.0000 - accuracy: 0.7463 - precision: 0.7488 - recall: 0.7501 - auc: 0.8120 - val_loss: 0.5731 - val_tp: 549.0000 - val_fp: 207.0000 - val_tn: 636.0000 - val_fn: 208.0000 - val_accuracy: 0.7406 - val_precision: 0.7262 - val_recall: 0.7252 - val_auc: 0.8006
    Epoch 24/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5284 - tp: 4842.0000 - fp: 1598.0000 - tn: 4728.0000 - fn: 1632.0000 - accuracy: 0.7477 - precision: 0.7519 - recall: 0.7479 - auc: 0.8147 - val_loss: 0.5692 - val_tp: 543.0000 - val_fp: 187.0000 - val_tn: 656.0000 - val_fn: 214.0000 - val_accuracy: 0.7494 - val_precision: 0.7438 - val_recall: 0.7173 - val_auc: 0.8060
    Epoch 25/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5284 - tp: 4849.0000 - fp: 1594.0000 - tn: 4732.0000 - fn: 1625.0000 - accuracy: 0.7485 - precision: 0.7526 - recall: 0.7490 - auc: 0.8144 - val_loss: 0.5676 - val_tp: 533.0000 - val_fp: 178.0000 - val_tn: 665.0000 - val_fn: 224.0000 - val_accuracy: 0.7487 - val_precision: 0.7496 - val_recall: 0.7041 - val_auc: 0.8072
    Epoch 26/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5277 - tp: 4862.0000 - fp: 1594.0000 - tn: 4732.0000 - fn: 1612.0000 - accuracy: 0.7495 - precision: 0.7531 - recall: 0.7510 - auc: 0.8148 - val_loss: 0.5663 - val_tp: 535.0000 - val_fp: 184.0000 - val_tn: 659.0000 - val_fn: 222.0000 - val_accuracy: 0.7462 - val_precision: 0.7441 - val_recall: 0.7067 - val_auc: 0.8103
    Epoch 27/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5253 - tp: 4836.0000 - fp: 1579.0000 - tn: 4747.0000 - fn: 1638.0000 - accuracy: 0.7487 - precision: 0.7539 - recall: 0.7470 - auc: 0.8169 - val_loss: 0.5796 - val_tp: 576.0000 - val_fp: 241.0000 - val_tn: 602.0000 - val_fn: 181.0000 - val_accuracy: 0.7362 - val_precision: 0.7050 - val_recall: 0.7609 - val_auc: 0.8040
    Epoch 28/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5241 - tp: 4858.0000 - fp: 1603.0000 - tn: 4723.0000 - fn: 1616.0000 - accuracy: 0.7485 - precision: 0.7519 - recall: 0.7504 - auc: 0.8180 - val_loss: 0.5823 - val_tp: 584.0000 - val_fp: 279.0000 - val_tn: 564.0000 - val_fn: 173.0000 - val_accuracy: 0.7175 - val_precision: 0.6767 - val_recall: 0.7715 - val_auc: 0.8026
    Epoch 29/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5240 - tp: 4891.0000 - fp: 1601.0000 - tn: 4725.0000 - fn: 1583.0000 - accuracy: 0.7513 - precision: 0.7534 - recall: 0.7555 - auc: 0.8177 - val_loss: 0.5668 - val_tp: 566.0000 - val_fp: 242.0000 - val_tn: 601.0000 - val_fn: 191.0000 - val_accuracy: 0.7294 - val_precision: 0.7005 - val_recall: 0.7477 - val_auc: 0.8090
    Epoch 30/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5213 - tp: 4898.0000 - fp: 1595.0000 - tn: 4731.0000 - fn: 1576.0000 - accuracy: 0.7523 - precision: 0.7544 - recall: 0.7566 - auc: 0.8203 - val_loss: 0.5718 - val_tp: 549.0000 - val_fp: 216.0000 - val_tn: 627.0000 - val_fn: 208.0000 - val_accuracy: 0.7350 - val_precision: 0.7176 - val_recall: 0.7252 - val_auc: 0.8055
    Epoch 31/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5205 - tp: 4869.0000 - fp: 1603.0000 - tn: 4723.0000 - fn: 1605.0000 - accuracy: 0.7494 - precision: 0.7523 - recall: 0.7521 - auc: 0.8206 - val_loss: 0.5681 - val_tp: 550.0000 - val_fp: 216.0000 - val_tn: 627.0000 - val_fn: 207.0000 - val_accuracy: 0.7356 - val_precision: 0.7180 - val_recall: 0.7266 - val_auc: 0.8084
    Epoch 32/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5183 - tp: 4916.0000 - fp: 1575.0000 - tn: 4751.0000 - fn: 1558.0000 - accuracy: 0.7552 - precision: 0.7574 - recall: 0.7593 - auc: 0.8227 - val_loss: 0.5646 - val_tp: 511.0000 - val_fp: 157.0000 - val_tn: 686.0000 - val_fn: 246.0000 - val_accuracy: 0.7481 - val_precision: 0.7650 - val_recall: 0.6750 - val_auc: 0.8108
    Epoch 33/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5170 - tp: 4901.0000 - fp: 1551.0000 - tn: 4775.0000 - fn: 1573.0000 - accuracy: 0.7559 - precision: 0.7596 - recall: 0.7570 - auc: 0.8235 - val_loss: 0.5781 - val_tp: 583.0000 - val_fp: 263.0000 - val_tn: 580.0000 - val_fn: 174.0000 - val_accuracy: 0.7269 - val_precision: 0.6891 - val_recall: 0.7701 - val_auc: 0.8064
    Epoch 34/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5167 - tp: 4896.0000 - fp: 1538.0000 - tn: 4788.0000 - fn: 1578.0000 - accuracy: 0.7566 - precision: 0.7610 - recall: 0.7563 - auc: 0.8236 - val_loss: 0.5594 - val_tp: 517.0000 - val_fp: 162.0000 - val_tn: 681.0000 - val_fn: 240.0000 - val_accuracy: 0.7487 - val_precision: 0.7614 - val_recall: 0.6830 - val_auc: 0.8104
    Epoch 35/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5160 - tp: 4898.0000 - fp: 1564.0000 - tn: 4762.0000 - fn: 1576.0000 - accuracy: 0.7547 - precision: 0.7580 - recall: 0.7566 - auc: 0.8243 - val_loss: 0.5576 - val_tp: 554.0000 - val_fp: 207.0000 - val_tn: 636.0000 - val_fn: 203.0000 - val_accuracy: 0.7437 - val_precision: 0.7280 - val_recall: 0.7318 - val_auc: 0.8132
    Epoch 36/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5153 - tp: 4931.0000 - fp: 1561.0000 - tn: 4765.0000 - fn: 1543.0000 - accuracy: 0.7575 - precision: 0.7596 - recall: 0.7617 - auc: 0.8247 - val_loss: 0.5541 - val_tp: 539.0000 - val_fp: 183.0000 - val_tn: 660.0000 - val_fn: 218.0000 - val_accuracy: 0.7494 - val_precision: 0.7465 - val_recall: 0.7120 - val_auc: 0.8131
    Epoch 37/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5130 - tp: 4934.0000 - fp: 1543.0000 - tn: 4783.0000 - fn: 1540.0000 - accuracy: 0.7591 - precision: 0.7618 - recall: 0.7621 - auc: 0.8268 - val_loss: 0.5569 - val_tp: 565.0000 - val_fp: 219.0000 - val_tn: 624.0000 - val_fn: 192.0000 - val_accuracy: 0.7431 - val_precision: 0.7207 - val_recall: 0.7464 - val_auc: 0.8144
    Epoch 38/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5132 - tp: 4908.0000 - fp: 1556.0000 - tn: 4770.0000 - fn: 1566.0000 - accuracy: 0.7561 - precision: 0.7593 - recall: 0.7581 - auc: 0.8267 - val_loss: 0.5565 - val_tp: 567.0000 - val_fp: 220.0000 - val_tn: 623.0000 - val_fn: 190.0000 - val_accuracy: 0.7437 - val_precision: 0.7205 - val_recall: 0.7490 - val_auc: 0.8172
    Epoch 39/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5122 - tp: 4912.0000 - fp: 1531.0000 - tn: 4795.0000 - fn: 1562.0000 - accuracy: 0.7584 - precision: 0.7624 - recall: 0.7587 - auc: 0.8270 - val_loss: 0.5491 - val_tp: 552.0000 - val_fp: 180.0000 - val_tn: 663.0000 - val_fn: 205.0000 - val_accuracy: 0.7594 - val_precision: 0.7541 - val_recall: 0.7292 - val_auc: 0.8207
    Epoch 40/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5109 - tp: 4931.0000 - fp: 1561.0000 - tn: 4765.0000 - fn: 1543.0000 - accuracy: 0.7575 - precision: 0.7596 - recall: 0.7617 - auc: 0.8278 - val_loss: 0.5558 - val_tp: 495.0000 - val_fp: 145.0000 - val_tn: 698.0000 - val_fn: 262.0000 - val_accuracy: 0.7456 - val_precision: 0.7734 - val_recall: 0.6539 - val_auc: 0.8191
    Epoch 41/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5101 - tp: 4921.0000 - fp: 1495.0000 - tn: 4831.0000 - fn: 1553.0000 - accuracy: 0.7619 - precision: 0.7670 - recall: 0.7601 - auc: 0.8286 - val_loss: 0.5485 - val_tp: 553.0000 - val_fp: 194.0000 - val_tn: 649.0000 - val_fn: 204.0000 - val_accuracy: 0.7513 - val_precision: 0.7403 - val_recall: 0.7305 - val_auc: 0.8180
    Epoch 42/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5085 - tp: 4905.0000 - fp: 1522.0000 - tn: 4804.0000 - fn: 1569.0000 - accuracy: 0.7585 - precision: 0.7632 - recall: 0.7576 - auc: 0.8300 - val_loss: 0.5481 - val_tp: 528.0000 - val_fp: 169.0000 - val_tn: 674.0000 - val_fn: 229.0000 - val_accuracy: 0.7513 - val_precision: 0.7575 - val_recall: 0.6975 - val_auc: 0.8224
    Epoch 43/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5081 - tp: 4985.0000 - fp: 1541.0000 - tn: 4785.0000 - fn: 1489.0000 - accuracy: 0.7633 - precision: 0.7639 - recall: 0.7700 - auc: 0.8300 - val_loss: 0.5505 - val_tp: 575.0000 - val_fp: 194.0000 - val_tn: 649.0000 - val_fn: 182.0000 - val_accuracy: 0.7650 - val_precision: 0.7477 - val_recall: 0.7596 - val_auc: 0.8186
    Epoch 44/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5063 - tp: 4947.0000 - fp: 1529.0000 - tn: 4797.0000 - fn: 1527.0000 - accuracy: 0.7613 - precision: 0.7639 - recall: 0.7641 - auc: 0.8315 - val_loss: 0.5565 - val_tp: 547.0000 - val_fp: 180.0000 - val_tn: 663.0000 - val_fn: 210.0000 - val_accuracy: 0.7563 - val_precision: 0.7524 - val_recall: 0.7226 - val_auc: 0.8188
    Epoch 45/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5066 - tp: 4941.0000 - fp: 1495.0000 - tn: 4831.0000 - fn: 1533.0000 - accuracy: 0.7634 - precision: 0.7677 - recall: 0.7632 - auc: 0.8307 - val_loss: 0.5505 - val_tp: 561.0000 - val_fp: 214.0000 - val_tn: 629.0000 - val_fn: 196.0000 - val_accuracy: 0.7437 - val_precision: 0.7239 - val_recall: 0.7411 - val_auc: 0.8203
    Epoch 46/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5041 - tp: 4971.0000 - fp: 1536.0000 - tn: 4790.0000 - fn: 1503.0000 - accuracy: 0.7626 - precision: 0.7639 - recall: 0.7678 - auc: 0.8334 - val_loss: 0.5490 - val_tp: 548.0000 - val_fp: 194.0000 - val_tn: 649.0000 - val_fn: 209.0000 - val_accuracy: 0.7481 - val_precision: 0.7385 - val_recall: 0.7239 - val_auc: 0.8191
    Epoch 47/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.5035 - tp: 4951.0000 - fp: 1492.0000 - tn: 4834.0000 - fn: 1523.0000 - accuracy: 0.7645 - precision: 0.7684 - recall: 0.7648 - auc: 0.8336 - val_loss: 0.5447 - val_tp: 589.0000 - val_fp: 225.0000 - val_tn: 618.0000 - val_fn: 168.0000 - val_accuracy: 0.7544 - val_precision: 0.7236 - val_recall: 0.7781 - val_auc: 0.8247
    Epoch 48/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5028 - tp: 4968.0000 - fp: 1507.0000 - tn: 4819.0000 - fn: 1506.0000 - accuracy: 0.7646 - precision: 0.7673 - recall: 0.7674 - auc: 0.8343 - val_loss: 0.5378 - val_tp: 540.0000 - val_fp: 167.0000 - val_tn: 676.0000 - val_fn: 217.0000 - val_accuracy: 0.7600 - val_precision: 0.7638 - val_recall: 0.7133 - val_auc: 0.8251
    Epoch 49/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5032 - tp: 4960.0000 - fp: 1516.0000 - tn: 4810.0000 - fn: 1514.0000 - accuracy: 0.7633 - precision: 0.7659 - recall: 0.7661 - auc: 0.8337 - val_loss: 0.5465 - val_tp: 565.0000 - val_fp: 208.0000 - val_tn: 635.0000 - val_fn: 192.0000 - val_accuracy: 0.7500 - val_precision: 0.7309 - val_recall: 0.7464 - val_auc: 0.8228
    Epoch 50/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.5016 - tp: 4958.0000 - fp: 1497.0000 - tn: 4829.0000 - fn: 1516.0000 - accuracy: 0.7646 - precision: 0.7681 - recall: 0.7658 - auc: 0.8352 - val_loss: 0.5426 - val_tp: 597.0000 - val_fp: 240.0000 - val_tn: 603.0000 - val_fn: 160.0000 - val_accuracy: 0.7500 - val_precision: 0.7133 - val_recall: 0.7886 - val_auc: 0.8240
    Epoch 51/150
    400/400 [==============================] - 2s 4ms/step - loss: 0.4992 - tp: 4978.0000 - fp: 1491.0000 - tn: 4835.0000 - fn: 1496.0000 - accuracy: 0.7666 - precision: 0.7695 - recall: 0.7689 - auc: 0.8370 - val_loss: 0.5488 - val_tp: 569.0000 - val_fp: 224.0000 - val_tn: 619.0000 - val_fn: 188.0000 - val_accuracy: 0.7425 - val_precision: 0.7175 - val_recall: 0.7517 - val_auc: 0.8189
    Epoch 52/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4998 - tp: 4962.0000 - fp: 1485.0000 - tn: 4841.0000 - fn: 1512.0000 - accuracy: 0.7659 - precision: 0.7697 - recall: 0.7665 - auc: 0.8365 - val_loss: 0.5434 - val_tp: 575.0000 - val_fp: 219.0000 - val_tn: 624.0000 - val_fn: 182.0000 - val_accuracy: 0.7494 - val_precision: 0.7242 - val_recall: 0.7596 - val_auc: 0.8257
    Epoch 53/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4981 - tp: 4992.0000 - fp: 1481.0000 - tn: 4845.0000 - fn: 1482.0000 - accuracy: 0.7685 - precision: 0.7712 - recall: 0.7711 - auc: 0.8380 - val_loss: 0.5409 - val_tp: 594.0000 - val_fp: 247.0000 - val_tn: 596.0000 - val_fn: 163.0000 - val_accuracy: 0.7437 - val_precision: 0.7063 - val_recall: 0.7847 - val_auc: 0.8243
    Epoch 54/150
    400/400 [==============================] - 2s 4ms/step - loss: 0.4972 - tp: 4972.0000 - fp: 1500.0000 - tn: 4826.0000 - fn: 1502.0000 - accuracy: 0.7655 - precision: 0.7682 - recall: 0.7680 - auc: 0.8389 - val_loss: 0.5376 - val_tp: 555.0000 - val_fp: 182.0000 - val_tn: 661.0000 - val_fn: 202.0000 - val_accuracy: 0.7600 - val_precision: 0.7531 - val_recall: 0.7332 - val_auc: 0.8258
    Epoch 55/150
    400/400 [==============================] - 2s 6ms/step - loss: 0.4968 - tp: 4975.0000 - fp: 1484.0000 - tn: 4842.0000 - fn: 1499.0000 - accuracy: 0.7670 - precision: 0.7702 - recall: 0.7685 - auc: 0.8391 - val_loss: 0.5331 - val_tp: 592.0000 - val_fp: 256.0000 - val_tn: 587.0000 - val_fn: 165.0000 - val_accuracy: 0.7369 - val_precision: 0.6981 - val_recall: 0.7820 - val_auc: 0.8254
    Epoch 56/150
    400/400 [==============================] - 2s 5ms/step - loss: 0.4977 - tp: 4976.0000 - fp: 1467.0000 - tn: 4859.0000 - fn: 1498.0000 - accuracy: 0.7684 - precision: 0.7723 - recall: 0.7686 - auc: 0.8381 - val_loss: 0.5419 - val_tp: 609.0000 - val_fp: 255.0000 - val_tn: 588.0000 - val_fn: 148.0000 - val_accuracy: 0.7481 - val_precision: 0.7049 - val_recall: 0.8045 - val_auc: 0.8288
    Epoch 57/150
    400/400 [==============================] - 2s 4ms/step - loss: 0.4962 - tp: 4999.0000 - fp: 1481.0000 - tn: 4845.0000 - fn: 1475.0000 - accuracy: 0.7691 - precision: 0.7715 - recall: 0.7722 - auc: 0.8395 - val_loss: 0.5303 - val_tp: 576.0000 - val_fp: 194.0000 - val_tn: 649.0000 - val_fn: 181.0000 - val_accuracy: 0.7656 - val_precision: 0.7481 - val_recall: 0.7609 - val_auc: 0.8305
    Epoch 58/150
    400/400 [==============================] - 2s 5ms/step - loss: 0.4956 - tp: 4988.0000 - fp: 1462.0000 - tn: 4864.0000 - fn: 1486.0000 - accuracy: 0.7697 - precision: 0.7733 - recall: 0.7705 - auc: 0.8398 - val_loss: 0.5315 - val_tp: 567.0000 - val_fp: 191.0000 - val_tn: 652.0000 - val_fn: 190.0000 - val_accuracy: 0.7619 - val_precision: 0.7480 - val_recall: 0.7490 - val_auc: 0.8309
    Epoch 59/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4942 - tp: 4966.0000 - fp: 1473.0000 - tn: 4853.0000 - fn: 1508.0000 - accuracy: 0.7671 - precision: 0.7712 - recall: 0.7671 - auc: 0.8409 - val_loss: 0.5241 - val_tp: 577.0000 - val_fp: 192.0000 - val_tn: 651.0000 - val_fn: 180.0000 - val_accuracy: 0.7675 - val_precision: 0.7503 - val_recall: 0.7622 - val_auc: 0.8297
    Epoch 60/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4918 - tp: 4969.0000 - fp: 1438.0000 - tn: 4888.0000 - fn: 1505.0000 - accuracy: 0.7701 - precision: 0.7756 - recall: 0.7675 - auc: 0.8427 - val_loss: 0.5321 - val_tp: 532.0000 - val_fp: 169.0000 - val_tn: 674.0000 - val_fn: 225.0000 - val_accuracy: 0.7538 - val_precision: 0.7589 - val_recall: 0.7028 - val_auc: 0.8291
    Epoch 61/150
    400/400 [==============================] - 2s 4ms/step - loss: 0.4939 - tp: 4973.0000 - fp: 1450.0000 - tn: 4876.0000 - fn: 1501.0000 - accuracy: 0.7695 - precision: 0.7742 - recall: 0.7681 - auc: 0.8414 - val_loss: 0.5317 - val_tp: 551.0000 - val_fp: 180.0000 - val_tn: 663.0000 - val_fn: 206.0000 - val_accuracy: 0.7588 - val_precision: 0.7538 - val_recall: 0.7279 - val_auc: 0.8283
    Epoch 62/150
    400/400 [==============================] - 2s 5ms/step - loss: 0.4923 - tp: 5012.0000 - fp: 1444.0000 - tn: 4882.0000 - fn: 1462.0000 - accuracy: 0.7730 - precision: 0.7763 - recall: 0.7742 - auc: 0.8422 - val_loss: 0.5260 - val_tp: 559.0000 - val_fp: 183.0000 - val_tn: 660.0000 - val_fn: 198.0000 - val_accuracy: 0.7619 - val_precision: 0.7534 - val_recall: 0.7384 - val_auc: 0.8290
    Epoch 63/150
    400/400 [==============================] - 1s 4ms/step - loss: 0.4929 - tp: 4988.0000 - fp: 1471.0000 - tn: 4855.0000 - fn: 1486.0000 - accuracy: 0.7690 - precision: 0.7723 - recall: 0.7705 - auc: 0.8417 - val_loss: 0.5287 - val_tp: 570.0000 - val_fp: 190.0000 - val_tn: 653.0000 - val_fn: 187.0000 - val_accuracy: 0.7644 - val_precision: 0.7500 - val_recall: 0.7530 - val_auc: 0.8308
    Epoch 64/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4896 - tp: 5005.0000 - fp: 1439.0000 - tn: 4887.0000 - fn: 1469.0000 - accuracy: 0.7728 - precision: 0.7767 - recall: 0.7731 - auc: 0.8446 - val_loss: 0.5284 - val_tp: 548.0000 - val_fp: 178.0000 - val_tn: 665.0000 - val_fn: 209.0000 - val_accuracy: 0.7581 - val_precision: 0.7548 - val_recall: 0.7239 - val_auc: 0.8291
    Epoch 65/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4923 - tp: 4991.0000 - fp: 1432.0000 - tn: 4894.0000 - fn: 1483.0000 - accuracy: 0.7723 - precision: 0.7771 - recall: 0.7709 - auc: 0.8421 - val_loss: 0.5215 - val_tp: 559.0000 - val_fp: 179.0000 - val_tn: 664.0000 - val_fn: 198.0000 - val_accuracy: 0.7644 - val_precision: 0.7575 - val_recall: 0.7384 - val_auc: 0.8333
    Epoch 66/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4900 - tp: 5001.0000 - fp: 1431.0000 - tn: 4895.0000 - fn: 1473.0000 - accuracy: 0.7731 - precision: 0.7775 - recall: 0.7725 - auc: 0.8435 - val_loss: 0.5234 - val_tp: 555.0000 - val_fp: 171.0000 - val_tn: 672.0000 - val_fn: 202.0000 - val_accuracy: 0.7669 - val_precision: 0.7645 - val_recall: 0.7332 - val_auc: 0.8288
    Epoch 67/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4886 - tp: 5001.0000 - fp: 1439.0000 - tn: 4887.0000 - fn: 1473.0000 - accuracy: 0.7725 - precision: 0.7766 - recall: 0.7725 - auc: 0.8450 - val_loss: 0.5439 - val_tp: 605.0000 - val_fp: 269.0000 - val_tn: 574.0000 - val_fn: 152.0000 - val_accuracy: 0.7369 - val_precision: 0.6922 - val_recall: 0.7992 - val_auc: 0.8280
    Epoch 68/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4888 - tp: 5010.0000 - fp: 1445.0000 - tn: 4881.0000 - fn: 1464.0000 - accuracy: 0.7727 - precision: 0.7761 - recall: 0.7739 - auc: 0.8446 - val_loss: 0.5189 - val_tp: 558.0000 - val_fp: 177.0000 - val_tn: 666.0000 - val_fn: 199.0000 - val_accuracy: 0.7650 - val_precision: 0.7592 - val_recall: 0.7371 - val_auc: 0.8324
    Epoch 69/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4864 - tp: 5009.0000 - fp: 1420.0000 - tn: 4906.0000 - fn: 1465.0000 - accuracy: 0.7746 - precision: 0.7791 - recall: 0.7737 - auc: 0.8463 - val_loss: 0.5127 - val_tp: 560.0000 - val_fp: 180.0000 - val_tn: 663.0000 - val_fn: 197.0000 - val_accuracy: 0.7644 - val_precision: 0.7568 - val_recall: 0.7398 - val_auc: 0.8344
    Epoch 70/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4855 - tp: 5041.0000 - fp: 1442.0000 - tn: 4884.0000 - fn: 1433.0000 - accuracy: 0.7754 - precision: 0.7776 - recall: 0.7787 - auc: 0.8472 - val_loss: 0.5155 - val_tp: 573.0000 - val_fp: 195.0000 - val_tn: 648.0000 - val_fn: 184.0000 - val_accuracy: 0.7631 - val_precision: 0.7461 - val_recall: 0.7569 - val_auc: 0.8356
    Epoch 71/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4867 - tp: 5035.0000 - fp: 1433.0000 - tn: 4893.0000 - fn: 1439.0000 - accuracy: 0.7756 - precision: 0.7784 - recall: 0.7777 - auc: 0.8463 - val_loss: 0.5130 - val_tp: 571.0000 - val_fp: 187.0000 - val_tn: 656.0000 - val_fn: 186.0000 - val_accuracy: 0.7669 - val_precision: 0.7533 - val_recall: 0.7543 - val_auc: 0.8354
    Epoch 72/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4845 - tp: 5017.0000 - fp: 1403.0000 - tn: 4923.0000 - fn: 1457.0000 - accuracy: 0.7766 - precision: 0.7815 - recall: 0.7749 - auc: 0.8476 - val_loss: 0.5118 - val_tp: 567.0000 - val_fp: 189.0000 - val_tn: 654.0000 - val_fn: 190.0000 - val_accuracy: 0.7631 - val_precision: 0.7500 - val_recall: 0.7490 - val_auc: 0.8328
    Epoch 73/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4839 - tp: 4991.0000 - fp: 1416.0000 - tn: 4910.0000 - fn: 1483.0000 - accuracy: 0.7735 - precision: 0.7790 - recall: 0.7709 - auc: 0.8483 - val_loss: 0.5238 - val_tp: 575.0000 - val_fp: 203.0000 - val_tn: 640.0000 - val_fn: 182.0000 - val_accuracy: 0.7594 - val_precision: 0.7391 - val_recall: 0.7596 - val_auc: 0.8330
    Epoch 74/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4825 - tp: 5019.0000 - fp: 1408.0000 - tn: 4918.0000 - fn: 1455.0000 - accuracy: 0.7763 - precision: 0.7809 - recall: 0.7753 - auc: 0.8493 - val_loss: 0.5123 - val_tp: 579.0000 - val_fp: 228.0000 - val_tn: 615.0000 - val_fn: 178.0000 - val_accuracy: 0.7462 - val_precision: 0.7175 - val_recall: 0.7649 - val_auc: 0.8358
    Epoch 75/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4833 - tp: 5049.0000 - fp: 1432.0000 - tn: 4894.0000 - fn: 1425.0000 - accuracy: 0.7768 - precision: 0.7790 - recall: 0.7799 - auc: 0.8487 - val_loss: 0.5176 - val_tp: 605.0000 - val_fp: 250.0000 - val_tn: 593.0000 - val_fn: 152.0000 - val_accuracy: 0.7487 - val_precision: 0.7076 - val_recall: 0.7992 - val_auc: 0.8363
    Epoch 76/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4827 - tp: 5030.0000 - fp: 1389.0000 - tn: 4937.0000 - fn: 1444.0000 - accuracy: 0.7787 - precision: 0.7836 - recall: 0.7770 - auc: 0.8488 - val_loss: 0.5211 - val_tp: 551.0000 - val_fp: 195.0000 - val_tn: 648.0000 - val_fn: 206.0000 - val_accuracy: 0.7494 - val_precision: 0.7386 - val_recall: 0.7279 - val_auc: 0.8279
    Epoch 77/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4808 - tp: 5023.0000 - fp: 1396.0000 - tn: 4930.0000 - fn: 1451.0000 - accuracy: 0.7776 - precision: 0.7825 - recall: 0.7759 - auc: 0.8505 - val_loss: 0.5017 - val_tp: 582.0000 - val_fp: 175.0000 - val_tn: 668.0000 - val_fn: 175.0000 - val_accuracy: 0.7812 - val_precision: 0.7688 - val_recall: 0.7688 - val_auc: 0.8404
    Epoch 78/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4818 - tp: 5026.0000 - fp: 1383.0000 - tn: 4943.0000 - fn: 1448.0000 - accuracy: 0.7788 - precision: 0.7842 - recall: 0.7763 - auc: 0.8494 - val_loss: 0.5110 - val_tp: 544.0000 - val_fp: 161.0000 - val_tn: 682.0000 - val_fn: 213.0000 - val_accuracy: 0.7663 - val_precision: 0.7716 - val_recall: 0.7186 - val_auc: 0.8370
    Epoch 79/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4807 - tp: 5014.0000 - fp: 1387.0000 - tn: 4939.0000 - fn: 1460.0000 - accuracy: 0.7776 - precision: 0.7833 - recall: 0.7745 - auc: 0.8504 - val_loss: 0.5112 - val_tp: 601.0000 - val_fp: 222.0000 - val_tn: 621.0000 - val_fn: 156.0000 - val_accuracy: 0.7638 - val_precision: 0.7303 - val_recall: 0.7939 - val_auc: 0.8384
    Epoch 80/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4798 - tp: 5042.0000 - fp: 1399.0000 - tn: 4927.0000 - fn: 1432.0000 - accuracy: 0.7788 - precision: 0.7828 - recall: 0.7788 - auc: 0.8510 - val_loss: 0.4980 - val_tp: 578.0000 - val_fp: 171.0000 - val_tn: 672.0000 - val_fn: 179.0000 - val_accuracy: 0.7812 - val_precision: 0.7717 - val_recall: 0.7635 - val_auc: 0.8428
    Epoch 81/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4782 - tp: 5033.0000 - fp: 1389.0000 - tn: 4937.0000 - fn: 1441.0000 - accuracy: 0.7789 - precision: 0.7837 - recall: 0.7774 - auc: 0.8523 - val_loss: 0.4988 - val_tp: 558.0000 - val_fp: 170.0000 - val_tn: 673.0000 - val_fn: 199.0000 - val_accuracy: 0.7694 - val_precision: 0.7665 - val_recall: 0.7371 - val_auc: 0.8427
    Epoch 82/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4769 - tp: 5028.0000 - fp: 1378.0000 - tn: 4948.0000 - fn: 1446.0000 - accuracy: 0.7794 - precision: 0.7849 - recall: 0.7766 - auc: 0.8532 - val_loss: 0.5071 - val_tp: 577.0000 - val_fp: 179.0000 - val_tn: 664.0000 - val_fn: 180.0000 - val_accuracy: 0.7756 - val_precision: 0.7632 - val_recall: 0.7622 - val_auc: 0.8380
    Epoch 83/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4765 - tp: 5055.0000 - fp: 1376.0000 - tn: 4950.0000 - fn: 1419.0000 - accuracy: 0.7816 - precision: 0.7860 - recall: 0.7808 - auc: 0.8533 - val_loss: 0.4970 - val_tp: 565.0000 - val_fp: 167.0000 - val_tn: 676.0000 - val_fn: 192.0000 - val_accuracy: 0.7756 - val_precision: 0.7719 - val_recall: 0.7464 - val_auc: 0.8430
    Epoch 84/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4783 - tp: 5028.0000 - fp: 1410.0000 - tn: 4916.0000 - fn: 1446.0000 - accuracy: 0.7769 - precision: 0.7810 - recall: 0.7766 - auc: 0.8518 - val_loss: 0.5123 - val_tp: 575.0000 - val_fp: 197.0000 - val_tn: 646.0000 - val_fn: 182.0000 - val_accuracy: 0.7631 - val_precision: 0.7448 - val_recall: 0.7596 - val_auc: 0.8357
    Epoch 85/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4785 - tp: 5011.0000 - fp: 1380.0000 - tn: 4946.0000 - fn: 1463.0000 - accuracy: 0.7779 - precision: 0.7841 - recall: 0.7740 - auc: 0.8518 - val_loss: 0.4967 - val_tp: 595.0000 - val_fp: 191.0000 - val_tn: 652.0000 - val_fn: 162.0000 - val_accuracy: 0.7794 - val_precision: 0.7570 - val_recall: 0.7860 - val_auc: 0.8476
    Epoch 86/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4768 - tp: 5067.0000 - fp: 1388.0000 - tn: 4938.0000 - fn: 1407.0000 - accuracy: 0.7816 - precision: 0.7850 - recall: 0.7827 - auc: 0.8534 - val_loss: 0.5100 - val_tp: 584.0000 - val_fp: 210.0000 - val_tn: 633.0000 - val_fn: 173.0000 - val_accuracy: 0.7606 - val_precision: 0.7355 - val_recall: 0.7715 - val_auc: 0.8380
    Epoch 87/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4776 - tp: 5032.0000 - fp: 1386.0000 - tn: 4940.0000 - fn: 1442.0000 - accuracy: 0.7791 - precision: 0.7840 - recall: 0.7773 - auc: 0.8527 - val_loss: 0.4979 - val_tp: 537.0000 - val_fp: 150.0000 - val_tn: 693.0000 - val_fn: 220.0000 - val_accuracy: 0.7688 - val_precision: 0.7817 - val_recall: 0.7094 - val_auc: 0.8450
    Epoch 88/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4750 - tp: 5043.0000 - fp: 1380.0000 - tn: 4946.0000 - fn: 1431.0000 - accuracy: 0.7804 - precision: 0.7851 - recall: 0.7790 - auc: 0.8544 - val_loss: 0.4970 - val_tp: 561.0000 - val_fp: 170.0000 - val_tn: 673.0000 - val_fn: 196.0000 - val_accuracy: 0.7713 - val_precision: 0.7674 - val_recall: 0.7411 - val_auc: 0.8439
    Epoch 89/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4729 - tp: 5054.0000 - fp: 1358.0000 - tn: 4968.0000 - fn: 1420.0000 - accuracy: 0.7830 - precision: 0.7882 - recall: 0.7807 - auc: 0.8560 - val_loss: 0.4956 - val_tp: 536.0000 - val_fp: 135.0000 - val_tn: 708.0000 - val_fn: 221.0000 - val_accuracy: 0.7775 - val_precision: 0.7988 - val_recall: 0.7081 - val_auc: 0.8489
    Epoch 90/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4728 - tp: 5044.0000 - fp: 1345.0000 - tn: 4981.0000 - fn: 1430.0000 - accuracy: 0.7832 - precision: 0.7895 - recall: 0.7791 - auc: 0.8562 - val_loss: 0.4958 - val_tp: 564.0000 - val_fp: 171.0000 - val_tn: 672.0000 - val_fn: 193.0000 - val_accuracy: 0.7725 - val_precision: 0.7673 - val_recall: 0.7450 - val_auc: 0.8447
    Epoch 91/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4714 - tp: 5068.0000 - fp: 1357.0000 - tn: 4969.0000 - fn: 1406.0000 - accuracy: 0.7841 - precision: 0.7888 - recall: 0.7828 - auc: 0.8570 - val_loss: 0.4993 - val_tp: 547.0000 - val_fp: 145.0000 - val_tn: 698.0000 - val_fn: 210.0000 - val_accuracy: 0.7781 - val_precision: 0.7905 - val_recall: 0.7226 - val_auc: 0.8453
    Epoch 92/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4731 - tp: 5042.0000 - fp: 1361.0000 - tn: 4965.0000 - fn: 1432.0000 - accuracy: 0.7818 - precision: 0.7874 - recall: 0.7788 - auc: 0.8566 - val_loss: 0.5002 - val_tp: 580.0000 - val_fp: 181.0000 - val_tn: 662.0000 - val_fn: 177.0000 - val_accuracy: 0.7763 - val_precision: 0.7622 - val_recall: 0.7662 - val_auc: 0.8425
    Epoch 93/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4715 - tp: 5049.0000 - fp: 1358.0000 - tn: 4968.0000 - fn: 1425.0000 - accuracy: 0.7826 - precision: 0.7880 - recall: 0.7799 - auc: 0.8567 - val_loss: 0.4986 - val_tp: 592.0000 - val_fp: 203.0000 - val_tn: 640.0000 - val_fn: 165.0000 - val_accuracy: 0.7700 - val_precision: 0.7447 - val_recall: 0.7820 - val_auc: 0.8447
    Epoch 94/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4719 - tp: 5070.0000 - fp: 1361.0000 - tn: 4965.0000 - fn: 1404.0000 - accuracy: 0.7840 - precision: 0.7884 - recall: 0.7831 - auc: 0.8567 - val_loss: 0.4957 - val_tp: 590.0000 - val_fp: 192.0000 - val_tn: 651.0000 - val_fn: 167.0000 - val_accuracy: 0.7756 - val_precision: 0.7545 - val_recall: 0.7794 - val_auc: 0.8483
    Epoch 95/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4724 - tp: 5095.0000 - fp: 1359.0000 - tn: 4967.0000 - fn: 1379.0000 - accuracy: 0.7861 - precision: 0.7894 - recall: 0.7870 - auc: 0.8560 - val_loss: 0.4985 - val_tp: 530.0000 - val_fp: 137.0000 - val_tn: 706.0000 - val_fn: 227.0000 - val_accuracy: 0.7725 - val_precision: 0.7946 - val_recall: 0.7001 - val_auc: 0.8478
    Epoch 96/150
    400/400 [==============================] - 1s 3ms/step - loss: 0.4726 - tp: 5058.0000 - fp: 1356.0000 - tn: 4970.0000 - fn: 1416.0000 - accuracy: 0.7834 - precision: 0.7886 - recall: 0.7813 - auc: 0.8557 - val_loss: 0.4953 - val_tp: 560.0000 - val_fp: 168.0000 - val_tn: 675.0000 - val_fn: 197.0000 - val_accuracy: 0.7719 - val_precision: 0.7692 - val_recall: 0.7398 - val_auc: 0.8449
    Epoch 97/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4697 - tp: 5095.0000 - fp: 1365.0000 - tn: 4961.0000 - fn: 1379.0000 - accuracy: 0.7856 - precision: 0.7887 - recall: 0.7870 - auc: 0.8579 - val_loss: 0.5015 - val_tp: 565.0000 - val_fp: 172.0000 - val_tn: 671.0000 - val_fn: 192.0000 - val_accuracy: 0.7725 - val_precision: 0.7666 - val_recall: 0.7464 - val_auc: 0.8426
    Epoch 98/150
    400/400 [==============================] - 1s 2ms/step - loss: 0.4711 - tp: 5055.0000 - fp: 1357.0000 - tn: 4969.0000 - fn: 1419.0000 - accuracy: 0.7831 - precision: 0.7884 - recall: 0.7808 - auc: 0.8571 - val_loss: 0.4959 - val_tp: 583.0000 - val_fp: 200.0000 - val_tn: 643.0000 - val_fn: 174.0000 - val_accuracy: 0.7663 - val_precision: 0.7446 - val_recall: 0.7701 - val_auc: 0.8468
    Epoch 99/150
    370/400 [==========================>...] - ETA: 0s - loss: 0.4691 - tp: 4718.0000 - fp: 1269.0000 - tn: 4598.0000 - fn: 1255.0000 - accuracy: 0.7868 - precision: 0.7880 - recall: 0.7899 - auc: 0.8586Restoring model weights from the end of the best epoch.
    400/400 [==============================] - 1s 2ms/step - loss: 0.4685 - tp: 5101.0000 - fp: 1359.0000 - tn: 4967.0000 - fn: 1373.0000 - accuracy: 0.7866 - precision: 0.7896 - recall: 0.7879 - auc: 0.8589 - val_loss: 0.5110 - val_tp: 618.0000 - val_fp: 243.0000 - val_tn: 600.0000 - val_fn: 139.0000 - val_accuracy: 0.7613 - val_precision: 0.7178 - val_recall: 0.8164 - val_auc: 0.8468
    Epoch 00099: early stopping

- Check training history

```python
def plot_metrics(history):
  metrics =  ['loss', 'auc', 'precision', 'recall']
  for n, metric in enumerate(metrics):
    name = metric.replace("_"," ").capitalize()
    plt.subplot(2,2,n+1)
    plt.plot(history.epoch,  history.history[metric], color=colors[0], label='Train')
    plt.plot(history.epoch, history.history['val_'+metric],
             color=colors[0], linestyle="--", label='Val')
    plt.xlabel('Epoch')
    plt.ylabel(name)
    if metric == 'loss':
      plt.ylim([0, plt.ylim()[1]])
    elif metric == 'auc':
      plt.ylim([0.8,1])
    else:
      plt.ylim([0,1])

    plt.legend()

```

```python
plot_metrics(baseline_history)
```

- Evaluating the **down-sampled 50/50** model using the **down-sampled 50/50** test dataset.

```python
test_loss_ds_5050, test_tp_5050, test_fp_5050, test_tn_5050, test_fn_5050, test_accuracy_ds_5050, test_precision_5050, test_recall_5050, test_auc_5050 = model.evaluate(test_inputs_ds_5050, test_targets_ds_5050)
```

    51/51 [==============================] - 0s 4ms/step - loss: 0.5475 - tp: 556.0000 - fp: 156.0000 - tn: 675.0000 - fn: 214.0000 - accuracy: 0.7689 - precision: 0.7809 - recall: 0.7221 - auc: 0.8465A: 0s - loss: 0.5475 - tp: 556.0000 - fp: 156.0000 - tn: 675.0000 - fn: 214.0000 - accuracy: 0.7689 - precision: 0.7809 - recall: 0.7221 - auc: 0.846

- Predicting probabilities of **down-sampled 50/50** model using the **down-sampled 50/50** test dataset.

```python
y_prob = model.predict(test_inputs_ds_5050)
```

- Predicting classes of **down-sampled 50/50** model using the **down-sampled 50/50** test dataset.

```python
y_pred = model.predict_classes(test_inputs_ds_5050)
```

    WARNING:tensorflow:From <ipython-input-18-135a9bae13dc>:1: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.
    Instructions for updating:
    Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype("int32")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).

- Uniting all results of **down-sampled 50/50** model using **down-sampled 50/50** test dataset into one dataframe.

```python
# Targets of down-sampled 50/50 dataset
targets = pd.DataFrame(test_targets_ds_5050, columns=['target'])

# Predicted classes of down-sampled 50/50 dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each class of down-sampled 50/50 dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_ds_5050_1 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Evaluating the **down-sampled 50/50** model using the **test for all dataset**.

```python
test_loss_ds_5050_tfa, test_tp_5050_tfa, test_fp_5050_tfa, test_tn_5050_tfa, test_fn_5050_tfa, test_accuracy_ds_5050_tfa, test_precision_5050_tfa, test_recall_5050_tfa, test_auc_5050_tfa = model.evaluate(test_inputs_tfa, test_targets_tfa)
```

    3277/3277 [==============================] - 6s 2ms/step - loss: 1.3325 - tp: 870.0000 - fp: 86548.0000 - tn: 17411.0000 - fn: 29.0000 - accuracy: 0.1743 - precision: 0.0100 - recall: 0.9677 - auc: 0.7214

- Predicting classes of **down-sampled 50/50** model using the **test for all** dataset.

```python
y_pred = model.predict_classes(test_inputs_tfa)
```

- Predicting probabilities of **down-sampled 50/50** model using the **test for all** dataset.

```python
y_prob = model.predict(test_inputs_tfa)
```

- Uniting all results of **down-sampled 50/50** model using **test for all** dataset into one dataframe.

```python
# Targets of test for all dataset
targets = pd.DataFrame(test_targets_tfa, columns=['target'])

# Predicted classes of test for all dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each class of test for all dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_ds_5050_2 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Validating results of **down-sampled 50/50** **tensorflow** model with **sklearn** metrics.

```python
# Importing metrics library
from sklearn.metrics import accuracy_score
```

```python
# Single test dataset
accuracy_score(results_ds_5050_1['y_pred'],results_ds_5050_1['target'])
```

    0.768894440974391

```python
# Test for all dataset
accuracy_score(results_ds_5050_2['y_pred'],results_ds_5050_2['target'])
```

    0.17434053672585784

- Training the model using the **down-sampled 05/95** dataset.

- But first, let's set initial bias, this way the model doesn't need to spend the first few epochs just learning that positive examples are unlikely

```python
model.predict(train_inputs_ds_0595[:10])
```

    array([[0.43811303],
           [0.794631  ],
           [0.65202916],
           [0.5235002 ],
           [0.6282989 ],
           [0.19807902],
           [0.8856959 ],
           [0.7882875 ],
           [0.8466856 ],
           [0.8768617 ]], dtype=float32)

```python
results = model.evaluate(train_inputs_ds_0595, train_targets_ds_0595, batch_size=BATCH_SIZE, verbose=0)
print("Loss: {:0.4f}".format(results[0]))
```

    Loss: 1.3115

```python
neg = 152019
pos = 8001
```

```python
initial_bias = np.log([pos/neg])
initial_bias
```

    array([-2.94443898])

```python
model = make_model(output_bias = initial_bias)
model.predict(train_inputs_ds_0595[:10])
```

    array([[0.04244319],
           [0.05525997],
           [0.03877482],
           [0.05157155],
           [0.03389394],
           [0.0330207 ],
           [0.03402829],
           [0.05522433],
           [0.04068433],
           [0.04095746]], dtype=float32)

```python
results = model.evaluate(train_inputs_ds_0595, train_targets_ds_0595, batch_size=BATCH_SIZE, verbose=0)
print("Loss: {:0.4f}".format(results[0]))
```

    Loss: 0.2103

```python
initial_weights = os.path.join(tempfile.mkdtemp(),'initial_weights')
model.save_weights(initial_weights)
```

- Now we train the model

```python
model = make_model()
model.load_weights(initial_weights)
baseline_history = model.fit(
    train_inputs_ds_0595,
    train_targets_ds_0595,
    # batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    callbacks = [early_stopping],
    validation_data=(validation_inputs_ds_0595, validation_targets_ds_0595))
```

    Epoch 1/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1840 - tp: 6275.0000 - fp: 101831.0000 - tn: 262825.0000 - fn: 13117.0000 - accuracy: 0.7007 - precision: 0.0580 - recall: 0.3236 - auc: 0.5925 - val_loss: 0.1655 - val_tp: 2.0000 - val_fp: 6.0000 - val_tn: 15276.0000 - val_fn: 718.0000 - val_accuracy: 0.9548 - val_precision: 0.2500 - val_recall: 0.0028 - val_auc: 0.7513
    Epoch 2/150
    4001/4001 [==============================] - 10s 2ms/step - loss: 0.1785 - tp: 21.0000 - fp: 42.0000 - tn: 121510.0000 - fn: 6443.0000 - accuracy: 0.9493 - precision: 0.3333 - recall: 0.0032 - auc: 0.7606 - val_loss: 0.1631 - val_tp: 2.0000 - val_fp: 3.0000 - val_tn: 15279.0000 - val_fn: 718.0000 - val_accuracy: 0.9549 - val_precision: 0.4000 - val_recall: 0.0028 - val_auc: 0.7672
    Epoch 3/150
    4001/4001 [==============================] - 10s 2ms/step - loss: 0.1747 - tp: 75.0000 - fp: 76.0000 - tn: 121476.0000 - fn: 6389.0000 - accuracy: 0.9495 - precision: 0.4967 - recall: 0.0116 - auc: 0.7733 - val_loss: 0.1626 - val_tp: 14.0000 - val_fp: 35.0000 - val_tn: 15247.0000 - val_fn: 706.0000 - val_accuracy: 0.9537 - val_precision: 0.2857 - val_recall: 0.0194 - val_auc: 0.7778
    Epoch 4/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1727 - tp: 147.0000 - fp: 142.0000 - tn: 121410.0000 - fn: 6317.0000 - accuracy: 0.9495 - precision: 0.5087 - recall: 0.0227 - auc: 0.7826 - val_loss: 0.1607 - val_tp: 13.0000 - val_fp: 25.0000 - val_tn: 15257.0000 - val_fn: 707.0000 - val_accuracy: 0.9543 - val_precision: 0.3421 - val_recall: 0.0181 - val_auc: 0.7828
    Epoch 5/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1711 - tp: 164.0000 - fp: 177.0000 - tn: 121375.0000 - fn: 6300.0000 - accuracy: 0.9494 - precision: 0.4809 - recall: 0.0254 - auc: 0.7894 - val_loss: 0.1592 - val_tp: 12.0000 - val_fp: 26.0000 - val_tn: 15256.0000 - val_fn: 708.0000 - val_accuracy: 0.9541 - val_precision: 0.3158 - val_recall: 0.0167 - val_auc: 0.7837
    Epoch 6/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1698 - tp: 194.0000 - fp: 205.0000 - tn: 121347.0000 - fn: 6270.0000 - accuracy: 0.9494 - precision: 0.4862 - recall: 0.0300 - auc: 0.7945 - val_loss: 0.1589 - val_tp: 17.0000 - val_fp: 35.0000 - val_tn: 15247.0000 - val_fn: 703.0000 - val_accuracy: 0.9539 - val_precision: 0.3269 - val_recall: 0.0236 - val_auc: 0.7856
    Epoch 7/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1689 - tp: 223.0000 - fp: 223.0000 - tn: 121329.0000 - fn: 6241.0000 - accuracy: 0.9495 - precision: 0.5000 - recall: 0.0345 - auc: 0.7977 - val_loss: 0.1595 - val_tp: 22.0000 - val_fp: 40.0000 - val_tn: 15242.0000 - val_fn: 698.0000 - val_accuracy: 0.9539 - val_precision: 0.3548 - val_recall: 0.0306 - val_auc: 0.7953
    Epoch 8/150
    4001/4001 [==============================] - 10s 2ms/step - loss: 0.1678 - tp: 228.0000 - fp: 218.0000 - tn: 121334.0000 - fn: 6236.0000 - accuracy: 0.9496 - precision: 0.5112 - recall: 0.0353 - auc: 0.8021 - val_loss: 0.1568 - val_tp: 22.0000 - val_fp: 42.0000 - val_tn: 15240.0000 - val_fn: 698.0000 - val_accuracy: 0.9538 - val_precision: 0.3438 - val_recall: 0.0306 - val_auc: 0.7980
    Epoch 9/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1671 - tp: 275.0000 - fp: 237.0000 - tn: 121315.0000 - fn: 6189.0000 - accuracy: 0.9498 - precision: 0.5371 - recall: 0.0425 - auc: 0.8043 - val_loss: 0.1564 - val_tp: 18.0000 - val_fp: 34.0000 - val_tn: 15248.0000 - val_fn: 702.0000 - val_accuracy: 0.9540 - val_precision: 0.3462 - val_recall: 0.0250 - val_auc: 0.7965
    Epoch 10/150
    4001/4001 [==============================] - 10s 3ms/step - loss: 0.1663 - tp: 273.0000 - fp: 260.0000 - tn: 121292.0000 - fn: 6191.0000 - accuracy: 0.9496 - precision: 0.5122 - recall: 0.0422 - auc: 0.8074 - val_loss: 0.1547 - val_tp: 26.0000 - val_fp: 42.0000 - val_tn: 15240.0000 - val_fn: 694.0000 - val_accuracy: 0.9540 - val_precision: 0.3824 - val_recall: 0.0361 - val_auc: 0.8031
    Epoch 11/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1656 - tp: 278.0000 - fp: 239.0000 - tn: 121313.0000 - fn: 6186.0000 - accuracy: 0.9498 - precision: 0.5377 - recall: 0.0430 - auc: 0.8092 - val_loss: 0.1539 - val_tp: 34.0000 - val_fp: 45.0000 - val_tn: 15237.0000 - val_fn: 686.0000 - val_accuracy: 0.9543 - val_precision: 0.4304 - val_recall: 0.0472 - val_auc: 0.8085
    Epoch 12/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1649 - tp: 330.0000 - fp: 270.0000 - tn: 121282.0000 - fn: 6134.0000 - accuracy: 0.9500 - precision: 0.5500 - recall: 0.0511 - auc: 0.8121 - val_loss: 0.1567 - val_tp: 31.0000 - val_fp: 47.0000 - val_tn: 15235.0000 - val_fn: 689.0000 - val_accuracy: 0.9540 - val_precision: 0.3974 - val_recall: 0.0431 - val_auc: 0.7998
    Epoch 13/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1641 - tp: 315.0000 - fp: 243.0000 - tn: 121309.0000 - fn: 6149.0000 - accuracy: 0.9501 - precision: 0.5645 - recall: 0.0487 - auc: 0.8144 - val_loss: 0.1530 - val_tp: 33.0000 - val_fp: 45.0000 - val_tn: 15237.0000 - val_fn: 687.0000 - val_accuracy: 0.9543 - val_precision: 0.4231 - val_recall: 0.0458 - val_auc: 0.8103
    Epoch 14/150
    4001/4001 [==============================] - 10s 2ms/step - loss: 0.1634 - tp: 316.0000 - fp: 261.0000 - tn: 121291.0000 - fn: 6148.0000 - accuracy: 0.9499 - precision: 0.5477 - recall: 0.0489 - auc: 0.8175 - val_loss: 0.1528 - val_tp: 23.0000 - val_fp: 20.0000 - val_tn: 15262.0000 - val_fn: 697.0000 - val_accuracy: 0.9552 - val_precision: 0.5349 - val_recall: 0.0319 - val_auc: 0.8101
    Epoch 15/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1633 - tp: 349.0000 - fp: 271.0000 - tn: 121281.0000 - fn: 6115.0000 - accuracy: 0.9501 - precision: 0.5629 - recall: 0.0540 - auc: 0.8175 - val_loss: 0.1523 - val_tp: 28.0000 - val_fp: 35.0000 - val_tn: 15247.0000 - val_fn: 692.0000 - val_accuracy: 0.9546 - val_precision: 0.4444 - val_recall: 0.0389 - val_auc: 0.8111
    Epoch 16/150
    4001/4001 [==============================] - 18s 5ms/step - loss: 0.1626 - tp: 357.0000 - fp: 265.0000 - tn: 121287.0000 - fn: 6107.0000 - accuracy: 0.9502 - precision: 0.5740 - recall: 0.0552 - auc: 0.8205 - val_loss: 0.1577 - val_tp: 52.0000 - val_fp: 74.0000 - val_tn: 15208.0000 - val_fn: 668.0000 - val_accuracy: 0.9536 - val_precision: 0.4127 - val_recall: 0.0722 - val_auc: 0.8172
    Epoch 17/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1618 - tp: 381.0000 - fp: 268.0000 - tn: 121284.0000 - fn: 6083.0000 - accuracy: 0.9504 - precision: 0.5871 - recall: 0.0589 - auc: 0.8221 - val_loss: 0.1514 - val_tp: 48.0000 - val_fp: 51.0000 - val_tn: 15231.0000 - val_fn: 672.0000 - val_accuracy: 0.9548 - val_precision: 0.4848 - val_recall: 0.0667 - val_auc: 0.8162
    Epoch 18/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1613 - tp: 428.0000 - fp: 308.0000 - tn: 121244.0000 - fn: 6036.0000 - accuracy: 0.9504 - precision: 0.5815 - recall: 0.0662 - auc: 0.8237 - val_loss: 0.1526 - val_tp: 36.0000 - val_fp: 42.0000 - val_tn: 15240.0000 - val_fn: 684.0000 - val_accuracy: 0.9546 - val_precision: 0.4615 - val_recall: 0.0500 - val_auc: 0.8156
    Epoch 19/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1611 - tp: 429.0000 - fp: 293.0000 - tn: 121259.0000 - fn: 6035.0000 - accuracy: 0.9506 - precision: 0.5942 - recall: 0.0664 - auc: 0.8245 - val_loss: 0.1497 - val_tp: 32.0000 - val_fp: 43.0000 - val_tn: 15239.0000 - val_fn: 688.0000 - val_accuracy: 0.9543 - val_precision: 0.4267 - val_recall: 0.0444 - val_auc: 0.8263
    Epoch 20/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1603 - tp: 430.0000 - fp: 292.0000 - tn: 121260.0000 - fn: 6034.0000 - accuracy: 0.9506 - precision: 0.5956 - recall: 0.0665 - auc: 0.8270 - val_loss: 0.1506 - val_tp: 52.0000 - val_fp: 56.0000 - val_tn: 15226.0000 - val_fn: 668.0000 - val_accuracy: 0.9548 - val_precision: 0.4815 - val_recall: 0.0722 - val_auc: 0.820400 - tn: 119980.0000 - fn: 5960.0000 - accuracy: 0.9507 - precision: 0.5950 - recall: 0.0667 - auc: 0.8
    Epoch 21/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1599 - tp: 456.0000 - fp: 321.0000 - tn: 121231.0000 - fn: 6008.0000 - accuracy: 0.9506 - precision: 0.5869 - recall: 0.0705 - auc: 0.8280 - val_loss: 0.1507 - val_tp: 34.0000 - val_fp: 37.0000 - val_tn: 15245.0000 - val_fn: 686.0000 - val_accuracy: 0.9548 - val_precision: 0.4789 - val_recall: 0.0472 - val_auc: 0.8244
    Epoch 22/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1595 - tp: 452.0000 - fp: 298.0000 - tn: 121254.0000 - fn: 6012.0000 - accuracy: 0.9507 - precision: 0.6027 - recall: 0.0699 - auc: 0.8296 - val_loss: 0.1517 - val_tp: 31.0000 - val_fp: 26.0000 - val_tn: 15256.0000 - val_fn: 689.0000 - val_accuracy: 0.9553 - val_precision: 0.5439 - val_recall: 0.0431 - val_auc: 0.8220
    Epoch 23/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1590 - tp: 463.0000 - fp: 322.0000 - tn: 121230.0000 - fn: 6001.0000 - accuracy: 0.9506 - precision: 0.5898 - recall: 0.0716 - auc: 0.8312 - val_loss: 0.1510 - val_tp: 36.0000 - val_fp: 43.0000 - val_tn: 15239.0000 - val_fn: 684.0000 - val_accuracy: 0.9546 - val_precision: 0.4557 - val_recall: 0.0500 - val_auc: 0.8256
    Epoch 24/150
    4001/4001 [==============================] - 14s 4ms/step - loss: 0.1590 - tp: 432.0000 - fp: 301.0000 - tn: 121251.0000 - fn: 6032.0000 - accuracy: 0.9505 - precision: 0.5894 - recall: 0.0668 - auc: 0.8319 - val_loss: 0.1493 - val_tp: 51.0000 - val_fp: 59.0000 - val_tn: 15223.0000 - val_fn: 669.0000 - val_accuracy: 0.9545 - val_precision: 0.4636 - val_recall: 0.0708 - val_auc: 0.8264
    Epoch 25/150
    4001/4001 [==============================] - 13s 3ms/step - loss: 0.1583 - tp: 477.0000 - fp: 306.0000 - tn: 121246.0000 - fn: 5987.0000 - accuracy: 0.9508 - precision: 0.6092 - recall: 0.0738 - auc: 0.8330 - val_loss: 0.1553 - val_tp: 75.0000 - val_fp: 95.0000 - val_tn: 15187.0000 - val_fn: 645.0000 - val_accuracy: 0.9538 - val_precision: 0.4412 - val_recall: 0.1042 - val_auc: 0.8170
    Epoch 26/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1581 - tp: 483.0000 - fp: 329.0000 - tn: 121223.0000 - fn: 5981.0000 - accuracy: 0.9507 - precision: 0.5948 - recall: 0.0747 - auc: 0.8348 - val_loss: 0.1491 - val_tp: 55.0000 - val_fp: 59.0000 - val_tn: 15223.0000 - val_fn: 665.0000 - val_accuracy: 0.9548 - val_precision: 0.4825 - val_recall: 0.0764 - val_auc: 0.8234
    Epoch 27/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1579 - tp: 495.0000 - fp: 322.0000 - tn: 121230.0000 - fn: 5969.0000 - accuracy: 0.9509 - precision: 0.6059 - recall: 0.0766 - auc: 0.8349 - val_loss: 0.1548 - val_tp: 59.0000 - val_fp: 68.0000 - val_tn: 15214.0000 - val_fn: 661.0000 - val_accuracy: 0.9544 - val_precision: 0.4646 - val_recall: 0.0819 - val_auc: 0.8197
    Epoch 28/150
    4001/4001 [==============================] - 14s 4ms/step - loss: 0.1578 - tp: 459.0000 - fp: 296.0000 - tn: 121256.0000 - fn: 6005.0000 - accuracy: 0.9508 - precision: 0.6079 - recall: 0.0710 - auc: 0.8357 - val_loss: 0.1472 - val_tp: 50.0000 - val_fp: 51.0000 - val_tn: 15231.0000 - val_fn: 670.0000 - val_accuracy: 0.9549 - val_precision: 0.4950 - val_recall: 0.0694 - val_auc: 0.8332
    Epoch 29/150
    4001/4001 [==============================] - 15s 4ms/step - loss: 0.1573 - tp: 481.0000 - fp: 321.0000 - tn: 121231.0000 - fn: 5983.0000 - accuracy: 0.9508 - precision: 0.5998 - recall: 0.0744 - auc: 0.8369 - val_loss: 0.1491 - val_tp: 39.0000 - val_fp: 44.0000 - val_tn: 15238.0000 - val_fn: 681.0000 - val_accuracy: 0.9547 - val_precision: 0.4699 - val_recall: 0.0542 - val_auc: 0.8281
    Epoch 30/150
    4001/4001 [==============================] - 14s 3ms/step - loss: 0.1571 - tp: 476.0000 - fp: 305.0000 - tn: 121247.0000 - fn: 5988.0000 - accuracy: 0.9508 - precision: 0.6095 - recall: 0.0736 - auc: 0.8372 - val_loss: 0.1518 - val_tp: 66.0000 - val_fp: 76.0000 - val_tn: 15206.0000 - val_fn: 654.0000 - val_accuracy: 0.9544 - val_precision: 0.4648 - val_recall: 0.0917 - val_auc: 0.8282
    Epoch 31/150
    4001/4001 [==============================] - 13s 3ms/step - loss: 0.1568 - tp: 494.0000 - fp: 316.0000 - tn: 121236.0000 - fn: 5970.0000 - accuracy: 0.9509 - precision: 0.6099 - recall: 0.0764 - auc: 0.8370 - val_loss: 0.1527 - val_tp: 69.0000 - val_fp: 81.0000 - val_tn: 15201.0000 - val_fn: 651.0000 - val_accuracy: 0.9543 - val_precision: 0.4600 - val_recall: 0.0958 - val_auc: 0.8291
    Epoch 32/150
    4001/4001 [==============================] - 14s 4ms/step - loss: 0.1566 - tp: 515.0000 - fp: 330.0000 - tn: 121222.0000 - fn: 5949.0000 - accuracy: 0.9510 - precision: 0.6095 - recall: 0.0797 - auc: 0.8388 - val_loss: 0.1541 - val_tp: 52.0000 - val_fp: 58.0000 - val_tn: 15224.0000 - val_fn: 668.0000 - val_accuracy: 0.9546 - val_precision: 0.4727 - val_recall: 0.0722 - val_auc: 0.8231
    Epoch 33/150
    4001/4001 [==============================] - 12s 3ms/step - loss: 0.1563 - tp: 491.0000 - fp: 323.0000 - tn: 121229.0000 - fn: 5973.0000 - accuracy: 0.9508 - precision: 0.6032 - recall: 0.0760 - auc: 0.8400 - val_loss: 0.1528 - val_tp: 71.0000 - val_fp: 99.0000 - val_tn: 15183.0000 - val_fn: 649.0000 - val_accuracy: 0.9533 - val_precision: 0.4176 - val_recall: 0.0986 - val_auc: 0.8322
    Epoch 34/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1560 - tp: 511.0000 - fp: 315.0000 - tn: 121237.0000 - fn: 5953.0000 - accuracy: 0.9510 - precision: 0.6186 - recall: 0.0791 - auc: 0.8404 - val_loss: 0.1513 - val_tp: 76.0000 - val_fp: 93.0000 - val_tn: 15189.0000 - val_fn: 644.0000 - val_accuracy: 0.9539 - val_precision: 0.4497 - val_recall: 0.1056 - val_auc: 0.8350
    Epoch 35/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1560 - tp: 517.0000 - fp: 320.0000 - tn: 121232.0000 - fn: 5947.0000 - accuracy: 0.9510 - precision: 0.6177 - recall: 0.0800 - auc: 0.8402 - val_loss: 0.1457 - val_tp: 53.0000 - val_fp: 62.0000 - val_tn: 15220.0000 - val_fn: 667.0000 - val_accuracy: 0.9544 - val_precision: 0.4609 - val_recall: 0.0736 - val_auc: 0.8424
    Epoch 36/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1556 - tp: 529.0000 - fp: 352.0000 - tn: 121200.0000 - fn: 5935.0000 - accuracy: 0.9509 - precision: 0.6005 - recall: 0.0818 - auc: 0.8421 - val_loss: 0.1494 - val_tp: 54.0000 - val_fp: 64.0000 - val_tn: 15218.0000 - val_fn: 666.0000 - val_accuracy: 0.9544 - val_precision: 0.4576 - val_recall: 0.0750 - val_auc: 0.8351
    Epoch 37/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1552 - tp: 523.0000 - fp: 327.0000 - tn: 121225.0000 - fn: 5941.0000 - accuracy: 0.9510 - precision: 0.6153 - recall: 0.0809 - auc: 0.8429 - val_loss: 0.1469 - val_tp: 54.0000 - val_fp: 52.0000 - val_tn: 15230.0000 - val_fn: 666.0000 - val_accuracy: 0.9551 - val_precision: 0.5094 - val_recall: 0.0750 - val_auc: 0.8352
    Epoch 38/150
    4001/4001 [==============================] - 11s 3ms/step - loss: 0.1551 - tp: 543.0000 - fp: 346.0000 - tn: 121206.0000 - fn: 5921.0000 - accuracy: 0.9510 - precision: 0.6108 - recall: 0.0840 - auc: 0.8439 - val_loss: 0.1555 - val_tp: 69.0000 - val_fp: 105.0000 - val_tn: 15177.0000 - val_fn: 651.0000 - val_accuracy: 0.9528 - val_precision: 0.3966 - val_recall: 0.0958 - val_auc: 0.8245
    Epoch 39/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1551 - tp: 532.0000 - fp: 314.0000 - tn: 121238.0000 - fn: 5932.0000 - accuracy: 0.9512 - precision: 0.6288 - recall: 0.0823 - auc: 0.8431 - val_loss: 0.1458 - val_tp: 58.0000 - val_fp: 63.0000 - val_tn: 15219.0000 - val_fn: 662.0000 - val_accuracy: 0.9547 - val_precision: 0.4793 - val_recall: 0.0806 - val_auc: 0.8396
    Epoch 40/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1547 - tp: 557.0000 - fp: 361.0000 - tn: 121191.0000 - fn: 5907.0000 - accuracy: 0.9510 - precision: 0.6068 - recall: 0.0862 - auc: 0.8449 - val_loss: 0.1502 - val_tp: 61.0000 - val_fp: 72.0000 - val_tn: 15210.0000 - val_fn: 659.0000 - val_accuracy: 0.9543 - val_precision: 0.4586 - val_recall: 0.0847 - val_auc: 0.8303
    Epoch 41/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1550 - tp: 525.0000 - fp: 338.0000 - tn: 121214.0000 - fn: 5939.0000 - accuracy: 0.9510 - precision: 0.6083 - recall: 0.0812 - auc: 0.8437 - val_loss: 0.1465 - val_tp: 45.0000 - val_fp: 53.0000 - val_tn: 15229.0000 - val_fn: 675.0000 - val_accuracy: 0.9545 - val_precision: 0.4592 - val_recall: 0.0625 - val_auc: 0.8385
    Epoch 42/150
    4001/4001 [==============================] - 8s 2ms/step - loss: 0.1545 - tp: 540.0000 - fp: 363.0000 - tn: 121189.0000 - fn: 5924.0000 - accuracy: 0.9509 - precision: 0.5980 - recall: 0.0835 - auc: 0.8459 - val_loss: 0.1466 - val_tp: 59.0000 - val_fp: 61.0000 - val_tn: 15221.0000 - val_fn: 661.0000 - val_accuracy: 0.9549 - val_precision: 0.4917 - val_recall: 0.0819 - val_auc: 0.8375
    Epoch 43/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1544 - tp: 534.0000 - fp: 361.0000 - tn: 121191.0000 - fn: 5930.0000 - accuracy: 0.9509 - precision: 0.5966 - recall: 0.0826 - auc: 0.8457 - val_loss: 0.1464 - val_tp: 55.0000 - val_fp: 59.0000 - val_tn: 15223.0000 - val_fn: 665.0000 - val_accuracy: 0.9548 - val_precision: 0.4825 - val_recall: 0.0764 - val_auc: 0.8372
    Epoch 44/150
    4001/4001 [==============================] - 9s 2ms/step - loss: 0.1546 - tp: 518.0000 - fp: 342.0000 - tn: 121210.0000 - fn: 5946.0000 - accuracy: 0.9509 - precision: 0.6023 - recall: 0.0801 - auc: 0.8448 - val_loss: 0.1457 - val_tp: 54.0000 - val_fp: 55.0000 - val_tn: 15227.0000 - val_fn: 666.0000 - val_accuracy: 0.9549 - val_precision: 0.4954 - val_recall: 0.0750 - val_auc: 0.8372
    Epoch 45/150
    3986/4001 [============================>.] - ETA: 0s - loss: 0.1544 - tp: 547.0000 - fp: 332.0000 - tn: 120777.0000 - fn: 5896.0000 - accuracy: 0.9512 - precision: 0.6223 - recall: 0.0849 - auc: 0.8455Restoring model weights from the end of the best epoch.
    4001/4001 [==============================] - 10s 2ms/step - loss: 0.1544 - tp: 547.0000 - fp: 333.0000 - tn: 121219.0000 - fn: 5917.0000 - accuracy: 0.9512 - precision: 0.6216 - recall: 0.0846 - auc: 0.8455 - val_loss: 0.1463 - val_tp: 40.0000 - val_fp: 34.0000 - val_tn: 15248.0000 - val_fn: 680.0000 - val_accuracy: 0.9554 - val_precision: 0.5405 - val_recall: 0.0556 - val_auc: 0.8371
    Epoch 00045: early stopping

- Checking training history

```python
plot_metrics(baseline_history)
```

- Evaluating the **down-sampled 05/95** model using the **down-sampled 05/95** test dataset.

```python
test_loss_ds_0595, test_tp_0595, test_fp_0595, test_tn_0595, test_fn_0595, test_accuracy_ds_0595, test_precision_0595, test_recall_0595, test_auc_0595 = model.evaluate(test_inputs_ds_0595, test_targets_ds_0595)
```

    501/501 [==============================] - 1s 1ms/step - loss: 0.1581 - tp: 66.0000 - fp: 50.0000 - tn: 15135.0000 - fn: 751.0000 - accuracy: 0.9499 - precision: 0.5690 - recall: 0.0808 - auc: 0.8410

- Predicting classes of **down-sampled 05/95** model using the **down-sampled 05/95** test dataset.

```python
y_pred = model.predict_classes(test_inputs_ds_0595)
```

- Predicting probabilities of **down-sampled 05/95** model using the **down-sampled 05/95** test dataset.

```python
y_prob = model.predict(test_inputs_ds_0595)
```

- Uniting all results of **down-sampled 05/95** model using **down-sampled 05/95** test dataset into one dataframe.

```python
# Targets of down-sampled 05/95 dataset
targets = pd.DataFrame(test_targets_ds_0595, columns=['target'])

# Predicted classes of down-sampled 05/95 dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each classe of down-sampled 05/95 dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_ds_0595_1 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Evaluating the **down-sampled 05/95** model using the **test for all** dataset.

```python
test_loss_ds_0595_tfa, test_tp_0595_tfa, test_fp_0595_tfa, test_tn_0595_tfa, test_fn_0595_tfa, test_accuracy_ds_0595_tfa, test_precision_0595_tfa, test_recall_0595_tfa, test_auc_0595_tfa = model.evaluate(test_inputs_tfa, test_targets_tfa)
```

    3277/3277 [==============================] - 6s 2ms/step - loss: 0.0583 - tp: 63.0000 - fp: 337.0000 - tn: 103622.0000 - fn: 836.0000 - accuracy: 0.9888 - precision: 0.1575 - recall: 0.0701 - auc: 0.8409

- Predicting classes of **down-sampled 05/95** model using the **test for all** dataset.

```python
y_pred = model.predict_classes(test_inputs_tfa)
```

- Predicting probabilities of **down-sampled 05/95** model using the **test for all** dataset.

```python
y_prob = model.predict(test_inputs_tfa)
```

- Uniting all results of **down-sampled 05/95** model using **test for all** dataset into one dataframe.

```python
# Targets of test for all dataset
targets = pd.DataFrame(test_targets_tfa, columns=['target'])

# Predicted classes of test for all dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each class of test for all dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_ds_0595_2 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Validating results of **down-sampled 05/95** **tensorflow** model with **sklearn** metrics.

```python
# Importing metrics library
from sklearn.metrics import accuracy_score
```

```python
# Down-sampled 05/95 test dataset
accuracy_score(results_ds_0595_1['y_pred'],results_ds_0595_1['target'])
```

    0.9499437570303711

```python
# Test for all dataset
accuracy_score(results_ds_0595_2['y_pred'],results_ds_0595_2['target'])
```

    0.9888134429418833

- Training the model using the **up-sampled** dataset.

```python
model = make_model()
baseline_history = model.fit(
    train_inputs_us,  # train inputs
    train_targets_us, # train targets
    # batch_size=BATCH_SIZE, # batch size
    epochs=EPOCHS,         # epochs that we will train for (assuming early stopping doesn't kick in)
    callbacks = [early_stopping],  # early stopping
    validation_data=(validation_inputs_us, validation_targets_us) # validation data
)
```

    Epoch 1/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.6140 - tp: 21570.0000 - fp: 10795.0000 - tn: 125263.0000 - fn: 11230.0000 - accuracy: 0.8696 - precision: 0.6665 - recall: 0.6576 - auc: 0.9261 - val_loss: 0.5827 - val_tp: 2757.0000 - val_fp: 1133.0000 - val_tn: 2817.0000 - val_fn: 1293.0000 - val_accuracy: 0.6967 - val_precision: 0.7087 - val_recall: 0.6807 - val_auc: 0.7647
    Epoch 2/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.5772 - tp: 21871.0000 - fp: 8826.0000 - tn: 23273.0000 - fn: 10030.0000 - accuracy: 0.7054 - precision: 0.7125 - recall: 0.6856 - auc: 0.7701 - val_loss: 0.5686 - val_tp: 2970.0000 - val_fp: 1198.0000 - val_tn: 2752.0000 - val_fn: 1080.0000 - val_accuracy: 0.7153 - val_precision: 0.7126 - val_recall: 0.7333 - val_auc: 0.7833
    Epoch 3/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5618 - tp: 22102.0000 - fp: 8143.0000 - tn: 23956.0000 - fn: 9799.0000 - accuracy: 0.7197 - precision: 0.7308 - recall: 0.6928 - auc: 0.7849 - val_loss: 0.5576 - val_tp: 2737.0000 - val_fp: 947.0000 - val_tn: 3003.0000 - val_fn: 1313.0000 - val_accuracy: 0.7175 - val_precision: 0.7429 - val_recall: 0.6758 - val_auc: 0.7897
    Epoch 4/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5518 - tp: 22450.0000 - fp: 7982.0000 - tn: 24117.0000 - fn: 9451.0000 - accuracy: 0.7276 - precision: 0.7377 - recall: 0.7037 - auc: 0.7942 - val_loss: 0.5476 - val_tp: 2831.0000 - val_fp: 948.0000 - val_tn: 3002.0000 - val_fn: 1219.0000 - val_accuracy: 0.7291 - val_precision: 0.7491 - val_recall: 0.6990 - val_auc: 0.7994
    Epoch 5/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5434 - tp: 22867.0000 - fp: 7949.0000 - tn: 24150.0000 - fn: 9034.0000 - accuracy: 0.7346 - precision: 0.7420 - recall: 0.7168 - auc: 0.8023 - val_loss: 0.5411 - val_tp: 2990.0000 - val_fp: 1075.0000 - val_tn: 2875.0000 - val_fn: 1060.0000 - val_accuracy: 0.7331 - val_precision: 0.7355 - val_recall: 0.7383 - val_auc: 0.8053
    Epoch 6/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.5367 - tp: 23101.0000 - fp: 7827.0000 - tn: 24272.0000 - fn: 8800.0000 - accuracy: 0.7402 - precision: 0.7469 - recall: 0.7241 - auc: 0.8085 - val_loss: 0.5344 - val_tp: 2945.0000 - val_fp: 959.0000 - val_tn: 2991.0000 - val_fn: 1105.0000 - val_accuracy: 0.7420 - val_precision: 0.7544 - val_recall: 0.7272 - val_auc: 0.8115
    Epoch 7/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5309 - tp: 23419.0000 - fp: 7766.0000 - tn: 24333.0000 - fn: 8482.0000 - accuracy: 0.7461 - precision: 0.7510 - recall: 0.7341 - auc: 0.8139 - val_loss: 0.5304 - val_tp: 3039.0000 - val_fp: 1017.0000 - val_tn: 2933.0000 - val_fn: 1011.0000 - val_accuracy: 0.7465 - val_precision: 0.7493 - val_recall: 0.7504 - val_auc: 0.8161
    Epoch 8/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5266 - tp: 23607.0000 - fp: 7687.0000 - tn: 24412.0000 - fn: 8294.0000 - accuracy: 0.7503 - precision: 0.7544 - recall: 0.7400 - auc: 0.8177 - val_loss: 0.5265 - val_tp: 3083.0000 - val_fp: 1026.0000 - val_tn: 2924.0000 - val_fn: 967.0000 - val_accuracy: 0.7509 - val_precision: 0.7503 - val_recall: 0.7612 - val_auc: 0.8175
    Epoch 9/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5230 - tp: 23724.0000 - fp: 7648.0000 - tn: 24451.0000 - fn: 8177.0000 - accuracy: 0.7527 - precision: 0.7562 - recall: 0.7437 - auc: 0.8210 - val_loss: 0.5250 - val_tp: 3099.0000 - val_fp: 1035.0000 - val_tn: 2915.0000 - val_fn: 951.0000 - val_accuracy: 0.7517 - val_precision: 0.7496 - val_recall: 0.7652 - val_auc: 0.8205
    Epoch 10/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5207 - tp: 23862.0000 - fp: 7594.0000 - tn: 24505.0000 - fn: 8039.0000 - accuracy: 0.7557 - precision: 0.7586 - recall: 0.7480 - auc: 0.8229 - val_loss: 0.5229 - val_tp: 3111.0000 - val_fp: 1053.0000 - val_tn: 2897.0000 - val_fn: 939.0000 - val_accuracy: 0.7510 - val_precision: 0.7471 - val_recall: 0.7681 - val_auc: 0.8218
    Epoch 11/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5182 - tp: 23960.0000 - fp: 7693.0000 - tn: 24406.0000 - fn: 7941.0000 - accuracy: 0.7557 - precision: 0.7570 - recall: 0.7511 - auc: 0.8248 - val_loss: 0.5211 - val_tp: 2948.0000 - val_fp: 924.0000 - val_tn: 3026.0000 - val_fn: 1102.0000 - val_accuracy: 0.7467 - val_precision: 0.7614 - val_recall: 0.7279 - val_auc: 0.8222
    Epoch 12/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.5149 - tp: 23990.0000 - fp: 7500.0000 - tn: 24599.0000 - fn: 7911.0000 - accuracy: 0.7592 - precision: 0.7618 - recall: 0.7520 - auc: 0.8276 - val_loss: 0.5190 - val_tp: 3045.0000 - val_fp: 990.0000 - val_tn: 2960.0000 - val_fn: 1005.0000 - val_accuracy: 0.7506 - val_precision: 0.7546 - val_recall: 0.7519 - val_auc: 0.8232: 17278.0000 - fp: 5374.0000 - tn: 17975.0000 - fn: 5677.0000 - accuracy: 0.7613 - precision: 0.7628 - re
    Epoch 13/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.5139 - tp: 24077.0000 - fp: 7535.0000 - tn: 24564.0000 - fn: 7824.0000 - accuracy: 0.7600 - precision: 0.7616 - recall: 0.7547 - auc: 0.8281 - val_loss: 0.5348 - val_tp: 2809.0000 - val_fp: 762.0000 - val_tn: 3188.0000 - val_fn: 1241.0000 - val_accuracy: 0.7496 - val_precision: 0.7866 - val_recall: 0.6936 - val_auc: 0.8142
    Epoch 14/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5117 - tp: 24046.0000 - fp: 7407.0000 - tn: 24692.0000 - fn: 7855.0000 - accuracy: 0.7615 - precision: 0.7645 - recall: 0.7538 - auc: 0.8298 - val_loss: 0.5110 - val_tp: 3064.0000 - val_fp: 950.0000 - val_tn: 3000.0000 - val_fn: 986.0000 - val_accuracy: 0.7580 - val_precision: 0.7633 - val_recall: 0.7565 - val_auc: 0.8295
    Epoch 15/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5097 - tp: 24101.0000 - fp: 7378.0000 - tn: 24721.0000 - fn: 7800.0000 - accuracy: 0.7628 - precision: 0.7656 - recall: 0.7555 - auc: 0.8313 - val_loss: 0.5240 - val_tp: 2727.0000 - val_fp: 690.0000 - val_tn: 3260.0000 - val_fn: 1323.0000 - val_accuracy: 0.7484 - val_precision: 0.7981 - val_recall: 0.6733 - val_auc: 0.8299
    Epoch 16/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5076 - tp: 24138.0000 - fp: 7311.0000 - tn: 24788.0000 - fn: 7763.0000 - accuracy: 0.7645 - precision: 0.7675 - recall: 0.7567 - auc: 0.8330 - val_loss: 0.5164 - val_tp: 3217.0000 - val_fp: 1048.0000 - val_tn: 2902.0000 - val_fn: 833.0000 - val_accuracy: 0.7649 - val_precision: 0.7543 - val_recall: 0.7943 - val_auc: 0.8304
    Epoch 17/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.5065 - tp: 24235.0000 - fp: 7391.0000 - tn: 24708.0000 - fn: 7666.0000 - accuracy: 0.7647 - precision: 0.7663 - recall: 0.7597 - auc: 0.8338 - val_loss: 0.5289 - val_tp: 2797.0000 - val_fp: 732.0000 - val_tn: 3218.0000 - val_fn: 1253.0000 - val_accuracy: 0.7519 - val_precision: 0.7926 - val_recall: 0.6906 - val_auc: 0.8256
    Epoch 18/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5048 - tp: 24211.0000 - fp: 7333.0000 - tn: 24766.0000 - fn: 7690.0000 - accuracy: 0.7653 - precision: 0.7675 - recall: 0.7589 - auc: 0.8346 - val_loss: 0.5139 - val_tp: 2911.0000 - val_fp: 765.0000 - val_tn: 3185.0000 - val_fn: 1139.0000 - val_accuracy: 0.7620 - val_precision: 0.7919 - val_recall: 0.7188 - val_auc: 0.8310
    Epoch 19/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5030 - tp: 24312.0000 - fp: 7328.0000 - tn: 24771.0000 - fn: 7589.0000 - accuracy: 0.7669 - precision: 0.7684 - recall: 0.7621 - auc: 0.8362 - val_loss: 0.5080 - val_tp: 2885.0000 - val_fp: 755.0000 - val_tn: 3195.0000 - val_fn: 1165.0000 - val_accuracy: 0.7600 - val_precision: 0.7926 - val_recall: 0.7123 - val_auc: 0.8352
    Epoch 20/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.5014 - tp: 24344.0000 - fp: 7250.0000 - tn: 24849.0000 - fn: 7557.0000 - accuracy: 0.7686 - precision: 0.7705 - recall: 0.7631 - auc: 0.8373 - val_loss: 0.5089 - val_tp: 3133.0000 - val_fp: 1042.0000 - val_tn: 2908.0000 - val_fn: 917.0000 - val_accuracy: 0.7551 - val_precision: 0.7504 - val_recall: 0.7736 - val_auc: 0.8293
    Epoch 21/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.5011 - tp: 24376.0000 - fp: 7374.0000 - tn: 24725.0000 - fn: 7525.0000 - accuracy: 0.7672 - precision: 0.7677 - recall: 0.7641 - auc: 0.8375 - val_loss: 0.5020 - val_tp: 3035.0000 - val_fp: 836.0000 - val_tn: 3114.0000 - val_fn: 1015.0000 - val_accuracy: 0.7686 - val_precision: 0.7840 - val_recall: 0.7494 - val_auc: 0.8358
    Epoch 22/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.5000 - tp: 24370.0000 - fp: 7282.0000 - tn: 24817.0000 - fn: 7531.0000 - accuracy: 0.7685 - precision: 0.7699 - recall: 0.7639 - auc: 0.8380 - val_loss: 0.5036 - val_tp: 3039.0000 - val_fp: 893.0000 - val_tn: 3057.0000 - val_fn: 1011.0000 - val_accuracy: 0.7620 - val_precision: 0.7729 - val_recall: 0.7504 - val_auc: 0.8354
    Epoch 23/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4989 - tp: 24467.0000 - fp: 7341.0000 - tn: 24758.0000 - fn: 7434.0000 - accuracy: 0.7691 - precision: 0.7692 - recall: 0.7670 - auc: 0.8387 - val_loss: 0.5058 - val_tp: 3049.0000 - val_fp: 850.0000 - val_tn: 3100.0000 - val_fn: 1001.0000 - val_accuracy: 0.7686 - val_precision: 0.7820 - val_recall: 0.7528 - val_auc: 0.8338
    Epoch 24/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4967 - tp: 24508.0000 - fp: 7347.0000 - tn: 24752.0000 - fn: 7393.0000 - accuracy: 0.7697 - precision: 0.7694 - recall: 0.7683 - auc: 0.8402 - val_loss: 0.5099 - val_tp: 3338.0000 - val_fp: 1164.0000 - val_tn: 2786.0000 - val_fn: 712.0000 - val_accuracy: 0.7655 - val_precision: 0.7414 - val_recall: 0.8242 - val_auc: 0.8388
    Epoch 25/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4961 - tp: 24571.0000 - fp: 7265.0000 - tn: 24834.0000 - fn: 7330.0000 - accuracy: 0.7720 - precision: 0.7718 - recall: 0.7702 - auc: 0.8408 - val_loss: 0.4987 - val_tp: 3176.0000 - val_fp: 939.0000 - val_tn: 3011.0000 - val_fn: 874.0000 - val_accuracy: 0.7734 - val_precision: 0.7718 - val_recall: 0.7842 - val_auc: 0.8382
    Epoch 26/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4949 - tp: 24564.0000 - fp: 7265.0000 - tn: 24834.0000 - fn: 7337.0000 - accuracy: 0.7718 - precision: 0.7717 - recall: 0.7700 - auc: 0.8414 - val_loss: 0.4975 - val_tp: 3216.0000 - val_fp: 995.0000 - val_tn: 2955.0000 - val_fn: 834.0000 - val_accuracy: 0.7714 - val_precision: 0.7637 - val_recall: 0.7941 - val_auc: 0.8387
    Epoch 27/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4934 - tp: 24639.0000 - fp: 7222.0000 - tn: 24877.0000 - fn: 7262.0000 - accuracy: 0.7737 - precision: 0.7733 - recall: 0.7724 - auc: 0.8422 - val_loss: 0.5080 - val_tp: 3318.0000 - val_fp: 1117.0000 - val_tn: 2833.0000 - val_fn: 732.0000 - val_accuracy: 0.7689 - val_precision: 0.7481 - val_recall: 0.8193 - val_auc: 0.8387
    Epoch 28/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4920 - tp: 24584.0000 - fp: 7162.0000 - tn: 24937.0000 - fn: 7317.0000 - accuracy: 0.7738 - precision: 0.7744 - recall: 0.7706 - auc: 0.8433 - val_loss: 0.4956 - val_tp: 3146.0000 - val_fp: 929.0000 - val_tn: 3021.0000 - val_fn: 904.0000 - val_accuracy: 0.7709 - val_precision: 0.7720 - val_recall: 0.7768 - val_auc: 0.8391
    Epoch 29/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4916 - tp: 24604.0000 - fp: 7227.0000 - tn: 24872.0000 - fn: 7297.0000 - accuracy: 0.7731 - precision: 0.7730 - recall: 0.7713 - auc: 0.8433 - val_loss: 0.4917 - val_tp: 3111.0000 - val_fp: 895.0000 - val_tn: 3055.0000 - val_fn: 939.0000 - val_accuracy: 0.7707 - val_precision: 0.7766 - val_recall: 0.7681 - val_auc: 0.8425
    Epoch 30/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4902 - tp: 24646.0000 - fp: 7196.0000 - tn: 24903.0000 - fn: 7255.0000 - accuracy: 0.7742 - precision: 0.7740 - recall: 0.7726 - auc: 0.8442 - val_loss: 0.4925 - val_tp: 3097.0000 - val_fp: 848.0000 - val_tn: 3102.0000 - val_fn: 953.0000 - val_accuracy: 0.7749 - val_precision: 0.7850 - val_recall: 0.7647 - val_auc: 0.8418
    Epoch 31/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4894 - tp: 24650.0000 - fp: 7147.0000 - tn: 24952.0000 - fn: 7251.0000 - accuracy: 0.7750 - precision: 0.7752 - recall: 0.7727 - auc: 0.8450 - val_loss: 0.4973 - val_tp: 3060.0000 - val_fp: 862.0000 - val_tn: 3088.0000 - val_fn: 990.0000 - val_accuracy: 0.7685 - val_precision: 0.7802 - val_recall: 0.7556 - val_auc: 0.8392
    Epoch 32/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4890 - tp: 24687.0000 - fp: 7141.0000 - tn: 24958.0000 - fn: 7214.0000 - accuracy: 0.7757 - precision: 0.7756 - recall: 0.7739 - auc: 0.8457 - val_loss: 0.4944 - val_tp: 3114.0000 - val_fp: 917.0000 - val_tn: 3033.0000 - val_fn: 936.0000 - val_accuracy: 0.7684 - val_precision: 0.7725 - val_recall: 0.7689 - val_auc: 0.8403
    Epoch 33/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4870 - tp: 24738.0000 - fp: 7111.0000 - tn: 24988.0000 - fn: 7163.0000 - accuracy: 0.7770 - precision: 0.7767 - recall: 0.7755 - auc: 0.8463 - val_loss: 0.4866 - val_tp: 3167.0000 - val_fp: 901.0000 - val_tn: 3049.0000 - val_fn: 883.0000 - val_accuracy: 0.7770 - val_precision: 0.7785 - val_recall: 0.7820 - val_auc: 0.8453
    Epoch 34/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4851 - tp: 24752.0000 - fp: 7098.0000 - tn: 25001.0000 - fn: 7149.0000 - accuracy: 0.7774 - precision: 0.7771 - recall: 0.7759 - auc: 0.8476 - val_loss: 0.4869 - val_tp: 3117.0000 - val_fp: 870.0000 - val_tn: 3080.0000 - val_fn: 933.0000 - val_accuracy: 0.7746 - val_precision: 0.7818 - val_recall: 0.7696 - val_auc: 0.8448
    Epoch 35/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4838 - tp: 24715.0000 - fp: 7110.0000 - tn: 24989.0000 - fn: 7186.0000 - accuracy: 0.7766 - precision: 0.7766 - recall: 0.7747 - auc: 0.8488 - val_loss: 0.4877 - val_tp: 3038.0000 - val_fp: 782.0000 - val_tn: 3168.0000 - val_fn: 1012.0000 - val_accuracy: 0.7757 - val_precision: 0.7953 - val_recall: 0.7501 - val_auc: 0.8467
    Epoch 36/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4822 - tp: 24743.0000 - fp: 7075.0000 - tn: 25024.0000 - fn: 7158.0000 - accuracy: 0.7776 - precision: 0.7776 - recall: 0.7756 - auc: 0.8496 - val_loss: 0.4858 - val_tp: 3073.0000 - val_fp: 809.0000 - val_tn: 3141.0000 - val_fn: 977.0000 - val_accuracy: 0.7768 - val_precision: 0.7916 - val_recall: 0.7588 - val_auc: 0.8476
    Epoch 37/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4816 - tp: 24782.0000 - fp: 7018.0000 - tn: 25081.0000 - fn: 7119.0000 - accuracy: 0.7791 - precision: 0.7793 - recall: 0.7768 - auc: 0.8499 - val_loss: 0.4870 - val_tp: 3234.0000 - val_fp: 963.0000 - val_tn: 2987.0000 - val_fn: 816.0000 - val_accuracy: 0.7776 - val_precision: 0.7706 - val_recall: 0.7985 - val_auc: 0.8475
    Epoch 38/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4806 - tp: 24806.0000 - fp: 6996.0000 - tn: 25103.0000 - fn: 7095.0000 - accuracy: 0.7798 - precision: 0.7800 - recall: 0.7776 - auc: 0.8513 - val_loss: 0.4812 - val_tp: 3128.0000 - val_fp: 828.0000 - val_tn: 3122.0000 - val_fn: 922.0000 - val_accuracy: 0.7812 - val_precision: 0.7907 - val_recall: 0.7723 - val_auc: 0.8506
    Epoch 39/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4802 - tp: 24760.0000 - fp: 6965.0000 - tn: 25134.0000 - fn: 7141.0000 - accuracy: 0.7796 - precision: 0.7805 - recall: 0.7762 - auc: 0.8512 - val_loss: 0.4863 - val_tp: 3210.0000 - val_fp: 938.0000 - val_tn: 3012.0000 - val_fn: 840.0000 - val_accuracy: 0.7778 - val_precision: 0.7739 - val_recall: 0.7926 - val_auc: 0.8467
    Epoch 40/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4778 - tp: 24846.0000 - fp: 6971.0000 - tn: 25128.0000 - fn: 7055.0000 - accuracy: 0.7808 - precision: 0.7809 - recall: 0.7788 - auc: 0.8527 - val_loss: 0.4928 - val_tp: 3022.0000 - val_fp: 809.0000 - val_tn: 3141.0000 - val_fn: 1028.0000 - val_accuracy: 0.7704 - val_precision: 0.7888 - val_recall: 0.7462 - val_auc: 0.8440
    Epoch 41/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4765 - tp: 24853.0000 - fp: 6903.0000 - tn: 25196.0000 - fn: 7048.0000 - accuracy: 0.7820 - precision: 0.7826 - recall: 0.7791 - auc: 0.8536 - val_loss: 0.4821 - val_tp: 3268.0000 - val_fp: 985.0000 - val_tn: 2965.0000 - val_fn: 782.0000 - val_accuracy: 0.7791 - val_precision: 0.7684 - val_recall: 0.8069 - val_auc: 0.8500
    Epoch 42/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4767 - tp: 24887.0000 - fp: 6983.0000 - tn: 25116.0000 - fn: 7014.0000 - accuracy: 0.7813 - precision: 0.7809 - recall: 0.7801 - auc: 0.8536 - val_loss: 0.4826 - val_tp: 2994.0000 - val_fp: 741.0000 - val_tn: 3209.0000 - val_fn: 1056.0000 - val_accuracy: 0.7754 - val_precision: 0.8016 - val_recall: 0.7393 - val_auc: 0.8516
    Epoch 43/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4742 - tp: 24797.0000 - fp: 6848.0000 - tn: 25251.0000 - fn: 7104.0000 - accuracy: 0.7820 - precision: 0.7836 - recall: 0.7773 - auc: 0.8554 - val_loss: 0.4808 - val_tp: 3296.0000 - val_fp: 983.0000 - val_tn: 2967.0000 - val_fn: 754.0000 - val_accuracy: 0.7829 - val_precision: 0.7703 - val_recall: 0.8138 - val_auc: 0.8516
    Epoch 44/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4738 - tp: 24874.0000 - fp: 6841.0000 - tn: 25258.0000 - fn: 7027.0000 - accuracy: 0.7833 - precision: 0.7843 - recall: 0.7797 - auc: 0.8555 - val_loss: 0.4740 - val_tp: 3148.0000 - val_fp: 861.0000 - val_tn: 3089.0000 - val_fn: 902.0000 - val_accuracy: 0.7796 - val_precision: 0.7852 - val_recall: 0.7773 - val_auc: 0.8551
    Epoch 45/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4726 - tp: 24914.0000 - fp: 6864.0000 - tn: 25235.0000 - fn: 6987.0000 - accuracy: 0.7836 - precision: 0.7840 - recall: 0.7810 - auc: 0.8563 - val_loss: 0.4747 - val_tp: 3121.0000 - val_fp: 780.0000 - val_tn: 3170.0000 - val_fn: 929.0000 - val_accuracy: 0.7864 - val_precision: 0.8001 - val_recall: 0.7706 - val_auc: 0.8548
    Epoch 46/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4714 - tp: 24924.0000 - fp: 6820.0000 - tn: 25279.0000 - fn: 6977.0000 - accuracy: 0.7844 - precision: 0.7852 - recall: 0.7813 - auc: 0.8573 - val_loss: 0.4774 - val_tp: 3073.0000 - val_fp: 793.0000 - val_tn: 3157.0000 - val_fn: 977.0000 - val_accuracy: 0.7788 - val_precision: 0.7949 - val_recall: 0.7588 - val_auc: 0.8532
    Epoch 47/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4704 - tp: 24943.0000 - fp: 6792.0000 - tn: 25307.0000 - fn: 6958.0000 - accuracy: 0.7852 - precision: 0.7860 - recall: 0.7819 - auc: 0.8577 - val_loss: 0.4755 - val_tp: 3238.0000 - val_fp: 925.0000 - val_tn: 3025.0000 - val_fn: 812.0000 - val_accuracy: 0.7829 - val_precision: 0.7778 - val_recall: 0.7995 - val_auc: 0.8552
    Epoch 48/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4693 - tp: 25031.0000 - fp: 6809.0000 - tn: 25290.0000 - fn: 6870.0000 - accuracy: 0.7863 - precision: 0.7861 - recall: 0.7846 - auc: 0.8585 - val_loss: 0.4779 - val_tp: 3023.0000 - val_fp: 732.0000 - val_tn: 3218.0000 - val_fn: 1027.0000 - val_accuracy: 0.7801 - val_precision: 0.8051 - val_recall: 0.7464 - val_auc: 0.8541
    Epoch 49/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4680 - tp: 25000.0000 - fp: 6737.0000 - tn: 25362.0000 - fn: 6901.0000 - accuracy: 0.7869 - precision: 0.7877 - recall: 0.7837 - auc: 0.8596 - val_loss: 0.4742 - val_tp: 3036.0000 - val_fp: 760.0000 - val_tn: 3190.0000 - val_fn: 1014.0000 - val_accuracy: 0.7782 - val_precision: 0.7998 - val_recall: 0.7496 - val_auc: 0.8574
    Epoch 50/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4685 - tp: 25034.0000 - fp: 6765.0000 - tn: 25334.0000 - fn: 6867.0000 - accuracy: 0.7870 - precision: 0.7873 - recall: 0.7847 - auc: 0.8600 - val_loss: 0.4704 - val_tp: 3157.0000 - val_fp: 801.0000 - val_tn: 3149.0000 - val_fn: 893.0000 - val_accuracy: 0.7883 - val_precision: 0.7976 - val_recall: 0.7795 - val_auc: 0.8590
    Epoch 51/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4665 - tp: 25051.0000 - fp: 6814.0000 - tn: 25285.0000 - fn: 6850.0000 - accuracy: 0.7865 - precision: 0.7862 - recall: 0.7853 - auc: 0.8605 - val_loss: 0.4756 - val_tp: 3171.0000 - val_fp: 790.0000 - val_tn: 3160.0000 - val_fn: 879.0000 - val_accuracy: 0.7914 - val_precision: 0.8006 - val_recall: 0.7830 - val_auc: 0.8537
    Epoch 52/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4660 - tp: 25062.0000 - fp: 6823.0000 - tn: 25276.0000 - fn: 6839.0000 - accuracy: 0.7865 - precision: 0.7860 - recall: 0.7856 - auc: 0.8609 - val_loss: 0.4768 - val_tp: 3347.0000 - val_fp: 979.0000 - val_tn: 2971.0000 - val_fn: 703.0000 - val_accuracy: 0.7897 - val_precision: 0.7737 - val_recall: 0.8264 - val_auc: 0.8563
    Epoch 53/150
    2000/2000 [==============================] - 7s 3ms/step - loss: 0.4641 - tp: 25096.0000 - fp: 6753.0000 - tn: 25346.0000 - fn: 6805.0000 - accuracy: 0.7882 - precision: 0.7880 - recall: 0.7867 - auc: 0.8621 - val_loss: 0.4705 - val_tp: 3262.0000 - val_fp: 882.0000 - val_tn: 3068.0000 - val_fn: 788.0000 - val_accuracy: 0.7912 - val_precision: 0.7872 - val_recall: 0.8054 - val_auc: 0.8593
    Epoch 54/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4632 - tp: 25071.0000 - fp: 6673.0000 - tn: 25426.0000 - fn: 6830.0000 - accuracy: 0.7890 - precision: 0.7898 - recall: 0.7859 - auc: 0.8629 - val_loss: 0.4732 - val_tp: 3272.0000 - val_fp: 870.0000 - val_tn: 3080.0000 - val_fn: 778.0000 - val_accuracy: 0.7940 - val_precision: 0.7900 - val_recall: 0.8079 - val_auc: 0.8604
    Epoch 55/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4621 - tp: 25148.0000 - fp: 6612.0000 - tn: 25487.0000 - fn: 6753.0000 - accuracy: 0.7912 - precision: 0.7918 - recall: 0.7883 - auc: 0.8638 - val_loss: 0.4661 - val_tp: 3164.0000 - val_fp: 848.0000 - val_tn: 3102.0000 - val_fn: 886.0000 - val_accuracy: 0.7832 - val_precision: 0.7886 - val_recall: 0.7812 - val_auc: 0.8618
    Epoch 56/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4623 - tp: 25129.0000 - fp: 6682.0000 - tn: 25417.0000 - fn: 6772.0000 - accuracy: 0.7898 - precision: 0.7899 - recall: 0.7877 - auc: 0.8633 - val_loss: 0.4646 - val_tp: 3206.0000 - val_fp: 874.0000 - val_tn: 3076.0000 - val_fn: 844.0000 - val_accuracy: 0.7853 - val_precision: 0.7858 - val_recall: 0.7916 - val_auc: 0.8609
    Epoch 57/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4594 - tp: 25223.0000 - fp: 6643.0000 - tn: 25456.0000 - fn: 6678.0000 - accuracy: 0.7919 - precision: 0.7915 - recall: 0.7907 - auc: 0.8652 - val_loss: 0.4796 - val_tp: 3164.0000 - val_fp: 898.0000 - val_tn: 3052.0000 - val_fn: 886.0000 - val_accuracy: 0.7770 - val_precision: 0.7789 - val_recall: 0.7812 - val_auc: 0.8546
    Epoch 58/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4602 - tp: 25220.0000 - fp: 6664.0000 - tn: 25435.0000 - fn: 6681.0000 - accuracy: 0.7915 - precision: 0.7910 - recall: 0.7906 - auc: 0.8647 - val_loss: 0.4633 - val_tp: 3180.0000 - val_fp: 844.0000 - val_tn: 3106.0000 - val_fn: 870.0000 - val_accuracy: 0.7857 - val_precision: 0.7903 - val_recall: 0.7852 - val_auc: 0.8641
    Epoch 59/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4590 - tp: 25264.0000 - fp: 6659.0000 - tn: 25440.0000 - fn: 6637.0000 - accuracy: 0.7922 - precision: 0.7914 - recall: 0.7920 - auc: 0.8659 - val_loss: 0.4589 - val_tp: 3220.0000 - val_fp: 807.0000 - val_tn: 3143.0000 - val_fn: 830.0000 - val_accuracy: 0.7954 - val_precision: 0.7996 - val_recall: 0.7951 - val_auc: 0.8645
    Epoch 60/150
    2000/2000 [==============================] - 6s 3ms/step - loss: 0.4575 - tp: 25325.0000 - fp: 6618.0000 - tn: 25481.0000 - fn: 6576.0000 - accuracy: 0.7938 - precision: 0.7928 - recall: 0.7939 - auc: 0.8664 - val_loss: 0.4712 - val_tp: 3140.0000 - val_fp: 796.0000 - val_tn: 3154.0000 - val_fn: 910.0000 - val_accuracy: 0.7868 - val_precision: 0.7978 - val_recall: 0.7753 - val_auc: 0.8588
    Epoch 61/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4559 - tp: 25314.0000 - fp: 6623.0000 - tn: 25476.0000 - fn: 6587.0000 - accuracy: 0.7936 - precision: 0.7926 - recall: 0.7935 - auc: 0.8674 - val_loss: 0.4557 - val_tp: 3188.0000 - val_fp: 803.0000 - val_tn: 3147.0000 - val_fn: 862.0000 - val_accuracy: 0.7919 - val_precision: 0.7988 - val_recall: 0.7872 - val_auc: 0.8671
    Epoch 62/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4548 - tp: 25351.0000 - fp: 6599.0000 - tn: 25500.0000 - fn: 6550.0000 - accuracy: 0.7945 - precision: 0.7935 - recall: 0.7947 - auc: 0.8683 - val_loss: 0.4691 - val_tp: 3051.0000 - val_fp: 745.0000 - val_tn: 3205.0000 - val_fn: 999.0000 - val_accuracy: 0.7820 - val_precision: 0.8037 - val_recall: 0.7533 - val_auc: 0.8634
    Epoch 63/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4549 - tp: 25364.0000 - fp: 6645.0000 - tn: 25454.0000 - fn: 6537.0000 - accuracy: 0.7940 - precision: 0.7924 - recall: 0.7951 - auc: 0.8682 - val_loss: 0.4628 - val_tp: 3334.0000 - val_fp: 949.0000 - val_tn: 3001.0000 - val_fn: 716.0000 - val_accuracy: 0.7919 - val_precision: 0.7784 - val_recall: 0.8232 - val_auc: 0.8637
    Epoch 64/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4535 - tp: 25326.0000 - fp: 6588.0000 - tn: 25511.0000 - fn: 6575.0000 - accuracy: 0.7943 - precision: 0.7936 - recall: 0.7939 - auc: 0.8691 - val_loss: 0.4596 - val_tp: 3125.0000 - val_fp: 762.0000 - val_tn: 3188.0000 - val_fn: 925.0000 - val_accuracy: 0.7891 - val_precision: 0.8040 - val_recall: 0.7716 - val_auc: 0.8655
    Epoch 65/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4534 - tp: 25408.0000 - fp: 6554.0000 - tn: 25545.0000 - fn: 6493.0000 - accuracy: 0.7961 - precision: 0.7949 - recall: 0.7965 - auc: 0.8692 - val_loss: 0.4633 - val_tp: 3063.0000 - val_fp: 711.0000 - val_tn: 3239.0000 - val_fn: 987.0000 - val_accuracy: 0.7878 - val_precision: 0.8116 - val_recall: 0.7563 - val_auc: 0.8651
    Epoch 66/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4523 - tp: 25544.0000 - fp: 6616.0000 - tn: 25483.0000 - fn: 6357.0000 - accuracy: 0.7973 - precision: 0.7943 - recall: 0.8007 - auc: 0.8696 - val_loss: 0.4704 - val_tp: 3001.0000 - val_fp: 695.0000 - val_tn: 3255.0000 - val_fn: 1049.0000 - val_accuracy: 0.7820 - val_precision: 0.8120 - val_recall: 0.7410 - val_auc: 0.8629
    Epoch 67/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4519 - tp: 25386.0000 - fp: 6517.0000 - tn: 25582.0000 - fn: 6515.0000 - accuracy: 0.7964 - precision: 0.7957 - recall: 0.7958 - auc: 0.8701 - val_loss: 0.4622 - val_tp: 3310.0000 - val_fp: 915.0000 - val_tn: 3035.0000 - val_fn: 740.0000 - val_accuracy: 0.7931 - val_precision: 0.7834 - val_recall: 0.8173 - val_auc: 0.8632
    Epoch 68/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4503 - tp: 25561.0000 - fp: 6636.0000 - tn: 25463.0000 - fn: 6340.0000 - accuracy: 0.7972 - precision: 0.7939 - recall: 0.8013 - auc: 0.8708 - val_loss: 0.4574 - val_tp: 3189.0000 - val_fp: 775.0000 - val_tn: 3175.0000 - val_fn: 861.0000 - val_accuracy: 0.7955 - val_precision: 0.8045 - val_recall: 0.7874 - val_auc: 0.8658
    Epoch 69/150
    2000/2000 [==============================] - 4s 2ms/step - loss: 0.4511 - tp: 25494.0000 - fp: 6622.0000 - tn: 25477.0000 - fn: 6407.0000 - accuracy: 0.7964 - precision: 0.7938 - recall: 0.7992 - auc: 0.8707 - val_loss: 0.4601 - val_tp: 3195.0000 - val_fp: 827.0000 - val_tn: 3123.0000 - val_fn: 855.0000 - val_accuracy: 0.7897 - val_precision: 0.7944 - val_recall: 0.7889 - val_auc: 0.8648
    Epoch 70/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4497 - tp: 25582.0000 - fp: 6590.0000 - tn: 25509.0000 - fn: 6319.0000 - accuracy: 0.7983 - precision: 0.7952 - recall: 0.8019 - auc: 0.8714 - val_loss: 0.4507 - val_tp: 3217.0000 - val_fp: 798.0000 - val_tn: 3152.0000 - val_fn: 833.0000 - val_accuracy: 0.7961 - val_precision: 0.8012 - val_recall: 0.7943 - val_auc: 0.8697
    Epoch 71/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4490 - tp: 25612.0000 - fp: 6586.0000 - tn: 25513.0000 - fn: 6289.0000 - accuracy: 0.7988 - precision: 0.7955 - recall: 0.8029 - auc: 0.8717 - val_loss: 0.4569 - val_tp: 3353.0000 - val_fp: 903.0000 - val_tn: 3047.0000 - val_fn: 697.0000 - val_accuracy: 0.8000 - val_precision: 0.7878 - val_recall: 0.8279 - val_auc: 0.8706
    Epoch 72/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4476 - tp: 25607.0000 - fp: 6553.0000 - tn: 25546.0000 - fn: 6294.0000 - accuracy: 0.7993 - precision: 0.7962 - recall: 0.8027 - auc: 0.8726 - val_loss: 0.4596 - val_tp: 3068.0000 - val_fp: 697.0000 - val_tn: 3253.0000 - val_fn: 982.0000 - val_accuracy: 0.7901 - val_precision: 0.8149 - val_recall: 0.7575 - val_auc: 0.8678
    Epoch 73/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4463 - tp: 25655.0000 - fp: 6535.0000 - tn: 25564.0000 - fn: 6246.0000 - accuracy: 0.8003 - precision: 0.7970 - recall: 0.8042 - auc: 0.8734 - val_loss: 0.4586 - val_tp: 3176.0000 - val_fp: 743.0000 - val_tn: 3207.0000 - val_fn: 874.0000 - val_accuracy: 0.7979 - val_precision: 0.8104 - val_recall: 0.7842 - val_auc: 0.8655
    Epoch 74/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4467 - tp: 25580.0000 - fp: 6570.0000 - tn: 25529.0000 - fn: 6321.0000 - accuracy: 0.7986 - precision: 0.7956 - recall: 0.8019 - auc: 0.8732 - val_loss: 0.4521 - val_tp: 3381.0000 - val_fp: 919.0000 - val_tn: 3031.0000 - val_fn: 669.0000 - val_accuracy: 0.8015 - val_precision: 0.7863 - val_recall: 0.8348 - val_auc: 0.8715
    Epoch 75/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4473 - tp: 25593.0000 - fp: 6477.0000 - tn: 25622.0000 - fn: 6308.0000 - accuracy: 0.8002 - precision: 0.7980 - recall: 0.8023 - auc: 0.8738 - val_loss: 0.4490 - val_tp: 3334.0000 - val_fp: 889.0000 - val_tn: 3061.0000 - val_fn: 716.0000 - val_accuracy: 0.7994 - val_precision: 0.7895 - val_recall: 0.8232 - val_auc: 0.8728
    Epoch 76/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4442 - tp: 25687.0000 - fp: 6488.0000 - tn: 25611.0000 - fn: 6214.0000 - accuracy: 0.8015 - precision: 0.7984 - recall: 0.8052 - auc: 0.8748 - val_loss: 0.4477 - val_tp: 3258.0000 - val_fp: 828.0000 - val_tn: 3122.0000 - val_fn: 792.0000 - val_accuracy: 0.7975 - val_precision: 0.7974 - val_recall: 0.8044 - val_auc: 0.8730
    Epoch 77/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4461 - tp: 25659.0000 - fp: 6600.0000 - tn: 25499.0000 - fn: 6242.0000 - accuracy: 0.7993 - precision: 0.7954 - recall: 0.8043 - auc: 0.8737 - val_loss: 0.4502 - val_tp: 3308.0000 - val_fp: 877.0000 - val_tn: 3073.0000 - val_fn: 742.0000 - val_accuracy: 0.7976 - val_precision: 0.7904 - val_recall: 0.8168 - val_auc: 0.8702
    Epoch 78/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4441 - tp: 25706.0000 - fp: 6529.0000 - tn: 25570.0000 - fn: 6195.0000 - accuracy: 0.8012 - precision: 0.7975 - recall: 0.8058 - auc: 0.8747 - val_loss: 0.4637 - val_tp: 3106.0000 - val_fp: 717.0000 - val_tn: 3233.0000 - val_fn: 944.0000 - val_accuracy: 0.7924 - val_precision: 0.8125 - val_recall: 0.7669 - val_auc: 0.8621
    Epoch 79/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4465 - tp: 25684.0000 - fp: 6615.0000 - tn: 25484.0000 - fn: 6217.0000 - accuracy: 0.7995 - precision: 0.7952 - recall: 0.8051 - auc: 0.8741 - val_loss: 0.4641 - val_tp: 3107.0000 - val_fp: 745.0000 - val_tn: 3205.0000 - val_fn: 943.0000 - val_accuracy: 0.7890 - val_precision: 0.8066 - val_recall: 0.7672 - val_auc: 0.8672
    Epoch 80/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4436 - tp: 25735.0000 - fp: 6543.0000 - tn: 25556.0000 - fn: 6166.0000 - accuracy: 0.8014 - precision: 0.7973 - recall: 0.8067 - auc: 0.8752 - val_loss: 0.4442 - val_tp: 3134.0000 - val_fp: 700.0000 - val_tn: 3250.0000 - val_fn: 916.0000 - val_accuracy: 0.7980 - val_precision: 0.8174 - val_recall: 0.7738 - val_auc: 0.8767
    Epoch 81/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4426 - tp: 25768.0000 - fp: 6581.0000 - tn: 25518.0000 - fn: 6133.0000 - accuracy: 0.8013 - precision: 0.7966 - recall: 0.8077 - auc: 0.8758 - val_loss: 0.4595 - val_tp: 3265.0000 - val_fp: 859.0000 - val_tn: 3091.0000 - val_fn: 785.0000 - val_accuracy: 0.7945 - val_precision: 0.7917 - val_recall: 0.8062 - val_auc: 0.8693
    Epoch 82/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4414 - tp: 25813.0000 - fp: 6515.0000 - tn: 25584.0000 - fn: 6088.0000 - accuracy: 0.8031 - precision: 0.7985 - recall: 0.8092 - auc: 0.8763 - val_loss: 0.4461 - val_tp: 3299.0000 - val_fp: 802.0000 - val_tn: 3148.0000 - val_fn: 751.0000 - val_accuracy: 0.8059 - val_precision: 0.8044 - val_recall: 0.8146 - val_auc: 0.8746
    Epoch 83/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4424 - tp: 25857.0000 - fp: 6539.0000 - tn: 25560.0000 - fn: 6044.0000 - accuracy: 0.8034 - precision: 0.7982 - recall: 0.8105 - auc: 0.8757 - val_loss: 0.4684 - val_tp: 3289.0000 - val_fp: 957.0000 - val_tn: 2993.0000 - val_fn: 761.0000 - val_accuracy: 0.7853 - val_precision: 0.7746 - val_recall: 0.8121 - val_auc: 0.8619
    Epoch 84/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4425 - tp: 25781.0000 - fp: 6538.0000 - tn: 25561.0000 - fn: 6120.0000 - accuracy: 0.8022 - precision: 0.7977 - recall: 0.8082 - auc: 0.8758 - val_loss: 0.4533 - val_tp: 3078.0000 - val_fp: 684.0000 - val_tn: 3266.0000 - val_fn: 972.0000 - val_accuracy: 0.7930 - val_precision: 0.8182 - val_recall: 0.7600 - val_auc: 0.8745
    Epoch 85/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4419 - tp: 25845.0000 - fp: 6558.0000 - tn: 25541.0000 - fn: 6056.0000 - accuracy: 0.8029 - precision: 0.7976 - recall: 0.8102 - auc: 0.8760 - val_loss: 0.4526 - val_tp: 3435.0000 - val_fp: 968.0000 - val_tn: 2982.0000 - val_fn: 615.0000 - val_accuracy: 0.8021 - val_precision: 0.7801 - val_recall: 0.8481 - val_auc: 0.8730
    Epoch 86/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4415 - tp: 25826.0000 - fp: 6587.0000 - tn: 25512.0000 - fn: 6075.0000 - accuracy: 0.8022 - precision: 0.7968 - recall: 0.8096 - auc: 0.8764 - val_loss: 0.4438 - val_tp: 3342.0000 - val_fp: 866.0000 - val_tn: 3084.0000 - val_fn: 708.0000 - val_accuracy: 0.8033 - val_precision: 0.7942 - val_recall: 0.8252 - val_auc: 0.8753
    Epoch 87/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4407 - tp: 25849.0000 - fp: 6535.0000 - tn: 25564.0000 - fn: 6052.0000 - accuracy: 0.8033 - precision: 0.7982 - recall: 0.8103 - auc: 0.8769 - val_loss: 0.4436 - val_tp: 3226.0000 - val_fp: 766.0000 - val_tn: 3184.0000 - val_fn: 824.0000 - val_accuracy: 0.8012 - val_precision: 0.8081 - val_recall: 0.7965 - val_auc: 0.8756
    Epoch 88/150
    2000/2000 [==============================] - 5s 2ms/step - loss: 0.4409 - tp: 25919.0000 - fp: 6529.0000 - tn: 25570.0000 - fn: 5982.0000 - accuracy: 0.8045 - precision: 0.7988 - recall: 0.8125 - auc: 0.8766 - val_loss: 0.4467 - val_tp: 3286.0000 - val_fp: 866.0000 - val_tn: 3084.0000 - val_fn: 764.0000 - val_accuracy: 0.7962 - val_precision: 0.7914 - val_recall: 0.8114 - val_auc: 0.8723
    Epoch 89/150
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4398 - tp: 25910.0000 - fp: 6573.0000 - tn: 25526.0000 - fn: 5991.0000 - accuracy: 0.8037 - precision: 0.7976 - recall: 0.8122 - auc: 0.8775 - val_loss: 0.4543 - val_tp: 3014.0000 - val_fp: 582.0000 - val_tn: 3368.0000 - val_fn: 1036.0000 - val_accuracy: 0.7977 - val_precision: 0.8382 - val_recall: 0.7442 - val_auc: 0.8741
    Epoch 90/150
    1987/2000 [============================>.] - ETA: 0s - loss: 0.4412 - tp: 25736.0000 - fp: 6496.0000 - tn: 25405.0000 - fn: 5947.0000 - accuracy: 0.8043 - precision: 0.7985 - recall: 0.8123 - auc: 0.8765Restoring model weights from the end of the best epoch.
    2000/2000 [==============================] - 5s 3ms/step - loss: 0.4414 - tp: 25921.0000 - fp: 6546.0000 - tn: 25553.0000 - fn: 5980.0000 - accuracy: 0.8043 - precision: 0.7984 - recall: 0.8125 - auc: 0.8764 - val_loss: 0.4571 - val_tp: 3243.0000 - val_fp: 880.0000 - val_tn: 3070.0000 - val_fn: 807.0000 - val_accuracy: 0.7891 - val_precision: 0.7866 - val_recall: 0.8007 - val_auc: 0.8660
    Epoch 00090: early stopping

- Checking training history

```python
plot_metrics(baseline_history)
```

- Evaluating the **up-sampled** model using the **up-sampled** test dataset.

```python
test_loss_us, test_tp_us, test_fp_us, test_tn_us, test_fn_us, test_accuracy_us, test_precision_us, test_recall_us, test_auc_us = model.evaluate(test_inputs_us, test_targets_us)
```

    250/250 [==============================] - 1s 2ms/step - loss: 0.4488 - tp: 3133.0000 - fp: 714.0000 - tn: 3237.0000 - fn: 916.0000 - accuracy: 0.7962 - precision: 0.8144 - recall: 0.7738 - auc: 0.8747

- Predicting classes of **up-sampled** model using the **up-sampled** test dataset.

```python
y_pred = model.predict_classes(test_inputs_us)
```

- Predicting probabilities of **up-sampled** model using the **up-sampled** test dataset.

```python
y_prob = model.predict(test_inputs_us)
```

- Uniting all results of **up-sampled** model using **up-sampled** test dataset into one dataframe.

```python
# Targets of up-sampled dataset
targets = pd.DataFrame(test_targets_us, columns=['target'])

# Predicted classes of up-sampled dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each classe of up-sampled dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_us_1 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Evaluating the **up-sampled** model using **test for all** dataset.

```python
test_loss_us_tfa, test_tp_us_tfa, test_fp_us_tfa, test_tn_us_tfa, test_fn_us_tfa, test_accuracy_us_tfa, test_precision_us_tfa, test_recall_us_tfa, test_auc_us_tfa = model.evaluate(test_inputs_tfa, test_targets_tfa)
```

    3277/3277 [==============================] - 7s 2ms/step - loss: 1.8532 - tp: 883.0000 - fp: 91710.0000 - tn: 12249.0000 - fn: 16.0000 - accuracy: 0.1252 - precision: 0.0095 - recall: 0.9822 - auc: 0.7153

- Predicting classes with **up-sampled** model using the **test for all** dataset.

```python
y_pred = model.predict_classes(test_inputs_tfa)
```

- Predicting probabilities of **up-sampled** model using the **test for all** dataset.

```python
y_prob = model.predict(test_inputs_tfa)
```

- Combining all results of **up-sampled** model using **test for all** dataset into one dataframe.

```python
# Targets of test for all dataset
targets = pd.DataFrame(test_targets_tfa, columns=['target'])

# Predicted classes of test for all dataset
y_pred = pd.DataFrame(y_pred, columns=['y_pred'])

# Predicted probabilities for each class of test for all dataset
y_prob = pd.DataFrame(y_prob, columns=['y_prob'])

# Uniting targets, predicted values and probabilities into a single data frame
results_us_2 = pd.concat([targets,y_pred,y_prob], axis=1)
```

- Validating results of **up-sampled** **tensorflow** model with **sklearn** metrics.

```python
# Importing metrics library
from sklearn.metrics import accuracy_score
```

```python
# Up-sampled test dataset
accuracy_score(results_us_1['y_pred'],results_us_1['target'])
```

    0.79625

```python
# Test for all dataset
accuracy_score(results_us_2['y_pred'],results_us_2['target'])
```

    0.12523603349291423

## 5. Results

### 5.1. Accuracy and Loss metrics

- Creating a single dataframe with **accuracy** and **loss** results for all trained models (**Down-sampled 50/50**, **Down-sampled 05/95** and **Up-Sampled**) using its **single test** dataset.

```python
# Defining Dataframe
df_loss_accuracy_1 = pd.DataFrame()

# Including trained model names
df_loss_accuracy_1['Model'] = pd.Series(['Down-sampled 50/50','Down-sampled 05/95','Up-Sampled'])

# Adding accuracy results for each model using its single test dataset
df_loss_accuracy_1['Accuracy'] = pd.Series([test_accuracy_ds_5050,test_accuracy_ds_0595,test_accuracy_us])

# Adding loss results for each model using its single test dataset
df_loss_accuracy_1['Loss'] = pd.Series([test_loss_ds_5050,test_loss_ds_0595,test_loss_us])
```

- Creating a single dataframe with **accuracy** an **loss** results for all trained models (**Down-sampled 50/50**, **Down-sampled 20/80** and **Up-Sampled**) using the **test for all** dataset.

```python
# Defining Dataframe
df_loss_accuracy_2 = pd.DataFrame()

# Including trained model names
df_loss_accuracy_2['Model'] = pd.Series(['Down-sampled 50/50','Down-sampled 05/95','Up-Sampled'])

# Adding accuracy results for each model using test for all dataset
df_loss_accuracy_2['Accuracy'] = pd.Series([test_accuracy_ds_5050_tfa,test_accuracy_ds_0595_tfa,test_accuracy_us_tfa])

# Adding loss results for each model using test for all dataset
df_loss_accuracy_2['Loss'] = pd.Series([test_loss_ds_5050_tfa,test_loss_ds_0595_tfa,test_loss_us_tfa])
```

- Plotting Accuracy and Loss results using **single test** and **test for all** datasets

```python
# Defining the figure of the plot and axes
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2,figsize=(10,5))

# Plotting
df_loss_accuracy_1.plot(y=['Accuracy','Loss'], x='Model', kind ='barh', ax=ax1)
df_loss_accuracy_2.plot(y=['Accuracy','Loss'], x='Model', kind ='barh', ax=ax2)

# Defining plot parameters
ax1.set_title('Single Test Dataset', fontsize=12)
ax2.set_title('Test For All Dataset', fontsize=12)
ax2.yaxis.set_visible(False)

# Saving the plot
fig.savefig(path_figures + 'result_accuracy_loss.png', bbox_inches='tight')

plt.show()
```

### 5.2. ROC Curve (AUC)

- Importing the libraries to calculate the **ROC** and **AUC** metrics.

```python
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
```

### 5.2.1 Single Test Datasets

- Defining **X** and **Y** axies values to plot **ROC** and **AUC** metrics using the **single test** dataset of each dataset (**Down-sampled 50/50**, **Down-sampled 20/80** and **Up-Sampled**).

```python
# Defining value parameters

# Defining No-Skill line
ns_probs_1 = [0 for _ in range(len(results_ds_5050_1['target']))] # no skill
ns_probs_2 = [0 for _ in range(len(results_ds_0595_1['target']))] # no skill
ns_probs_3 = [0 for _ in range(len(results_us_1['target']))] # no skill

# Probabilities values using single test dataset
probs_1 = results_ds_5050_1['y_prob'] # Down-sampled 50/50 single test dataset
probs_2 = results_ds_0595_1['y_prob'] # Down-sampled 05/95 single test dataset
probs_3 = results_us_1['y_prob'] # Up-sampled single test dataset

# Calculating AUC Score for No-Skill
ns_auc_1 = roc_auc_score(results_ds_5050_1['target'], ns_probs_1) # Down-sampled 50/50 single test dataset
ns_auc_2 = roc_auc_score(results_ds_0595_1['target'], ns_probs_2) # Down-sampled 05/95 single test dataset
ns_auc_3 = roc_auc_score(results_us_1['target'], ns_probs_3) # Up-sampled single test dataset

# Calculating AUC Score for each test dataset
auc_1 = roc_auc_score(results_ds_5050_1['target'], probs_1) # Down-sampled 50/50 single test dataset
auc_2 = roc_auc_score(results_ds_0595_1['target'], probs_2) # Down-sampled 05/95 single test dataset
auc_3 = roc_auc_score(results_us_1['target'], probs_3) # Up-sampled single test dataset

# Defining No Skill line for each plot
ns_tfp_1, ns_tvp_1, _ = roc_curve(results_ds_5050_1['target'], ns_probs_1) # Down-sampled 50/50 single test dataset
ns_tfp_2, ns_tvp_1, _ = roc_curve(results_ds_0595_1['target'], ns_probs_2) # Down-sampled 05/95 single test dataset
ns_tfp_3, ns_tvp_1, _ = roc_curve(results_us_1['target'], ns_probs_3) # Up-sampled single test dataset

# Defining single test dataset lines
tfp_1, tvp_1, _ = roc_curve(results_ds_5050_1['target'], probs_1) # Down-sampled 50/50 single test dataset
tfp_2, tvp_2, _ = roc_curve(results_ds_0595_1['target'], probs_2) # Down-sampled 05/95 single test dataset
tfp_3, tvp_3, _ = roc_curve(results_us_1['target'], probs_3) # Up-sampled single test dataset

```

- Plotting the **ROC** and **AUC** metrics using the **single test** dataset of each dataset (**Down-sampled 50/50**, **Down-sampled 20/80** and **Up-Sampled**).

```python
# Defining size of figure and quantity of plots
fig, (ax3, ax4, ax5) = plt.subplots(nrows=1, ncols=3, figsize=(15,5))

# Figure parameters
fig.suptitle('ROC Curve - Single Test Dataset', fontsize=16)

# Down-sampled 50/50 Model Plot
# Axis parameters
ax3.set_title('Down-sampled 50/50 Model')
ax3.set_ylabel('True Positive Rate')
ax3.set_xlabel('False Positive Rate')
# Plotting
ax3.plot(ns_tfp_1, ns_tvp_1, linestyle='--', label='No Skill')
ax3.plot(tfp_1, tvp_1, marker='.', label='50/50 Model')
# AUC Result
ax3.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_1), size=12, ha="center",
         transform=ax3.transAxes)
ax3.legend(loc = 'lower right')

# Down-sampled 05/95 Model Plot
# Axis parameters
ax4.set_title('Down-sampled 05/95 Model')
ax4.set_ylabel('True Positive Rate')
ax4.set_xlabel('False Positive Rate')
# Plotting
ax4.plot(ns_tfp_2, ns_tvp_1, linestyle='--', label='No Skill')
ax4.plot(tfp_2, tvp_2, marker='.', label='05/95 Model')
# AUC Result
ax4.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_2), size=12, ha="center",
         transform=ax4.transAxes)
ax4.legend(loc = 'lower right')

# Up-sampled Model Plot
# Axis Parameters
ax5.set_title('Up-sampled Model')
ax5.set_ylabel('True Positive Rate')
ax5.set_xlabel('False Positive Rate')
# Plotting
ax5.plot(ns_tfp_3, ns_tvp_1, linestyle='--', label='No Skill')
ax5.plot(tfp_3, tvp_3, marker='.', label='Up-Sampled Model')
# AUC Result
ax5.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_3), size=12, ha="center",
         transform=ax5.transAxes)
ax5.legend(loc = 'lower right')

# Saving the plot
plt.savefig(path_figures + 'roc_curve_single_test.png', bbox_inches='tight')

plt.show()
```

### 5.2.2 Test For All Dataset

- Defining **X** and **Y** axies values to plot **ROC** and **AUC** metrics using the **test for all** dataset.

```python
# Defining value parameters

# Defining No-Skill line
ns_probs_1 = [0 for _ in range(len(results_ds_5050_2['target']))] # no skill
ns_probs_2 = [0 for _ in range(len(results_ds_0595_2['target']))] # no skill
ns_probs_3 = [0 for _ in range(len(results_us_2['target']))] # no skill

# Probabilities values using single test dataset
probs_1 = results_ds_5050_2['y_prob'] # Test for all dataset using Down-sampled 50/50 trained model
probs_2 = results_ds_0595_2['y_prob'] # Test for all dataset using Down-sampled 05/95 trained model
probs_3 = results_us_2['y_prob'] # Test for all dataset using Up-sampled trained model

# Calculating AUC Score for No-Skill
ns_auc_1 = roc_auc_score(results_ds_5050_2['target'], ns_probs_1) # Test for all dataset using Down-sampled 50/50 trained model
ns_auc_2 = roc_auc_score(results_ds_0595_2['target'], ns_probs_2) # Test for all dataset using Down-sampled 05/95 trained model
ns_auc_3 = roc_auc_score(results_us_2['target'], ns_probs_3) # Test for all dataset using Up-sampled trained model

# Calculating AUC Score for each test dataset
auc_1 = roc_auc_score(results_ds_5050_2['target'], probs_1) # Test for all dataset using Down-sampled 50/50 trained model
auc_2 = roc_auc_score(results_ds_0595_2['target'], probs_2) # Test for all dataset using Down-sampled 05/95 trained model
auc_3 = roc_auc_score(results_us_2['target'], probs_3) # Test for all dataset using Up-sampled trained model

# Defining No Skill line for each plot
ns_tfp_1, ns_tvp_1, _ = roc_curve(results_ds_5050_2['target'], ns_probs_1) # Test for all dataset using Down-sampled 50/50 trained model
ns_tfp_2, ns_tvp_1, _ = roc_curve(results_ds_0595_2['target'], ns_probs_2) # Test for all dataset using Down-sampled 05/95 trained model
ns_tfp_3, ns_tvp_1, _ = roc_curve(results_us_2['target'], ns_probs_3) # Test for all dataset using Up-sampled trained model

# Defining single test dataset lines
tfp_1, tvp_1, _ = roc_curve(results_ds_5050_2['target'], probs_1) # Test for all dataset using Down-sampled 50/50 trained model
tfp_2, tvp_2, _ = roc_curve(results_ds_0595_2['target'], probs_2) # Test for all dataset using Down-sampled 05/95 trained model
tfp_3, tvp_3, _ = roc_curve(results_us_2['target'], probs_3) # Test for all dataset using Up-sampled trained model

```

- Plotting the **ROC** and **AUC** metrics using the **test for all** dataset.

```python
# Defining size of figure and quantity of plots
fig, (ax6, ax7, ax8) = plt.subplots(nrows=1, ncols=3, figsize=(15,5)) #constrained_layout=True)

# Figure parameters
fig.suptitle('ROC Curve - Test For All Dataset', fontsize=16)

# Down-sampled 50/50 Model Plot
# Axis parameters
ax6.set_title('Down-sampled 50/50 Model')
ax6.set_ylabel('True Positive Rate')
ax6.set_xlabel('False Positive Rate')
# Plotting
ax6.plot(ns_tfp_1, ns_tvp_1, linestyle='--', label='No Skill')
ax6.plot(tfp_1, tvp_1, marker='.', label='50/50 Model')
# AUC Result
ax6.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_1), size=12, ha="center",
         transform=ax6.transAxes)
ax6.legend(loc = 'lower right')

# Down-sampled 20/80 Model Plot
# Axis parameters
ax7.set_title('Down-sampled 05/95 Model')
ax7.set_ylabel('True Positive Rate')
ax7.set_xlabel('False Positive Rate')
# Plotting
ax7.plot(ns_tfp_2, ns_tvp_1, linestyle='--', label='No Skill')
ax7.plot(tfp_2, tvp_2, marker='.', label='05/95 Model')
# AUC Result
ax7.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_2), size=12, ha="center",
         transform=ax7.transAxes)
ax7.legend(loc = 'lower right')

# Up-sampled Model Plot
# Axis parameters
ax8.set_title('Up-sampled Model')
ax8.set_ylabel('True Positive Rate')
ax8.set_xlabel('False Positive Rate')
# Plotting
ax8.plot(ns_tfp_3, ns_tvp_1, linestyle='--', label='No Skill')
ax8.plot(tfp_3, tvp_3, marker='.', label='Up-Sampled Model')
# AUC Result
ax8.text(0.5,-0.2,'ROC AUC=%.3f' %(auc_3), size=12, ha="center",
         transform=ax8.transAxes)
ax8.legend(loc = 'lower right')

# Saving the plot
plt.savefig(path_figures + 'roc_curve_all_test.png', bbox_inches='tight')

plt.show()
```

### 6.3. Precision, Recall, Accuracy and F1 Scores

- Importing the necessary libraries to calculate **Precision**, **Recall**, **Accuracy** and **F1 Scores**.

```python
from sklearn.metrics import (recall_score,
                             accuracy_score,
                             precision_score,
                             f1_score)
```

- Calculating **Precision** score based on the target and predicted values of both test datasets (**Single test** and **Test for all** dataset).

\begin{equation}
Precision = \frac{True Positives}{True Positives + False Positives}
\end{equation}

```python
# Precision

# Single test dataset
p_score_1 = precision_score(results_ds_5050_1['target'],results_ds_5050_1['y_pred']) # Down-sampled 50/50 single test dataset
p_score_2 = precision_score(results_ds_0595_1['target'],results_ds_0595_1['y_pred']) # Down-sampled 05/95 single test dataset
p_score_3 = precision_score(results_us_1['target'],results_us_1['y_pred']) # Up-sampled single test dataset

# Test for all dataset
p_score_4 = precision_score(results_ds_5050_2['target'],results_ds_5050_2['y_pred']) # Test for all dataset using 50/50 model
p_score_5 = precision_score(results_ds_0595_2['target'],results_ds_0595_2['y_pred']) # Test for all dataset using 05/95 model
p_score_6 = precision_score(results_us_2['target'],results_us_2['y_pred']) # Test for all dataset using Up-sampled model
```

- Calculating **Recall** score based on the target and predicted values of both test datasets (**Single test** and **Test for all** dataset).

\begin{equation}
Recall = \frac{True Positives}{True Positives + False Negatives}
\end{equation}

```python
# Recall

# Single test dataset
r_score_1 = recall_score(results_ds_5050_1['target'],results_ds_5050_1['y_pred']) # Down-sampled 50/50 single test dataset
r_score_2 = recall_score(results_ds_0595_1['target'],results_ds_0595_1['y_pred']) # Down-sampled 05/95 single test dataset
r_score_3 = recall_score(results_us_1['target'],results_us_1['y_pred']) # Up-sampled single test dataset

# Test for all dataset
r_score_4 = recall_score(results_ds_5050_2['target'],results_ds_5050_2['y_pred']) # Test for all dataset using 50/50 model
r_score_5 = recall_score(results_ds_0595_2['target'],results_ds_0595_2['y_pred']) # Test for all dataset using 05/95 model
r_score_6 = recall_score(results_us_2['target'],results_us_2['y_pred']) # Test for all dataset using Up-sampled model
```

- Calculating **Accuracy** score based on the target and predicted values of both test datasets (**Single test** and **Test for all** dataset).

\begin{equation}
Accuracy = \frac{ True Positives + True Negatives}{ True Positives + False Positives + True Negatives + False Negatives}
\end{equation}

```python
# Accuracy

# Single test dataset (Already collected when validating the results)
a_score_1 = test_accuracy_ds_5050 # Down-sampled 50/50 single test dataset
a_score_2 = test_accuracy_ds_0595 # Down-sampled 05/95 single test dataset
a_score_3 = test_accuracy_us # Up-sampled single test dataset

# Test for all dataset (Already collected when validating the results)
a_score_4 = test_accuracy_ds_5050_tfa # Test for all dataset using 50/50 model
a_score_5 = test_accuracy_ds_0595_tfa # Test for all dataset using 05/95 model
a_score_6 = test_accuracy_us_tfa # Test for all dataset using Up-sampled model
```

- Calculating **F1** score based on the target and predicted values of bothe test datasets (**Single Test** and **Test for all** dataset).

\begin{equation}
F1 = 2 \cdot \frac{ Precision\cdot Recall}{Precision + Recall}
\end{equation}

```python
# F1 Score

# Single test dataset
f_score_1 = f1_score(results_ds_5050_1['target'],results_ds_5050_1['y_pred']) # Down-sampled 50/50 single test dataset
f_score_2 = f1_score(results_ds_0595_1['target'],results_ds_0595_1['y_pred']) # Down-sampled 05/95 single test dataset
f_score_3 = f1_score(results_us_1['target'],results_us_1['y_pred']) # Up-sampled single test dataset

# Test for all dataset
f_score_4 = f1_score(results_ds_5050_2['target'],results_ds_5050_2['y_pred']) # Test for all dataset using 50/50 model
f_score_5 = f1_score(results_ds_0595_2['target'],results_ds_0595_2['y_pred']) # Test for all dataset using 05/95 model
f_score_6 = f1_score(results_us_2['target'],results_us_2['y_pred']) # Test for all dataset using Up-sampled model
```

- Uniting all results into one single dataframe.

```python
score_results = {'Experiment':['Single Test Dataset', 'Single Test Dataset', 'Single Test Dataset',
                           'Test For All Dataset', 'Test For All Dataset', 'Test For All Dataset'],
                 'Model':['Down-sampled 50/50','Down-sampled 05/95','Up-Sampled',
                            'Down-sampled 50/50','Down-sampled 05/95','Up-Sampled'],
                     'Precision':[p_score_1, p_score_2, p_score_3,
                                  p_score_4, p_score_5, p_score_6],
                     'Recall': [r_score_1, r_score_2, r_score_3,
                                r_score_4, r_score_5, r_score_6],
                     'Accuracy': [a_score_1, a_score_2, a_score_3,
                                  a_score_4, a_score_5, a_score_6],
                     'F1_score': [f_score_1, f_score_2, f_score_3,
                                  f_score_4, f_score_5, f_score_6]}
```

```python
df_score_results = pd.DataFrame(score_results)
```

```python
df_score_results
```

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Experiment</th>
      <th>Model</th>
      <th>Precision</th>
      <th>Recall</th>
      <th>Accuracy</th>
      <th>F1_score</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Single Test Dataset</td>
      <td>Down-sampled 50/50</td>
      <td>0.780899</td>
      <td>0.722078</td>
      <td>0.768894</td>
      <td>0.750337</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Single Test Dataset</td>
      <td>Down-sampled 05/95</td>
      <td>0.568966</td>
      <td>0.080783</td>
      <td>0.949944</td>
      <td>0.141479</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Single Test Dataset</td>
      <td>Up-Sampled</td>
      <td>0.814401</td>
      <td>0.773771</td>
      <td>0.796250</td>
      <td>0.793566</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Test For All Dataset</td>
      <td>Down-sampled 50/50</td>
      <td>0.009952</td>
      <td>0.967742</td>
      <td>0.174341</td>
      <td>0.019702</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Test For All Dataset</td>
      <td>Down-sampled 05/95</td>
      <td>0.157500</td>
      <td>0.070078</td>
      <td>0.988813</td>
      <td>0.096998</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Test For All Dataset</td>
      <td>Up-Sampled</td>
      <td>0.009536</td>
      <td>0.982202</td>
      <td>0.125236</td>
      <td>0.018889</td>
    </tr>
  </tbody>
</table>
</div>

### 7.4. Confusion Matrix

### 7.4.1 Single Test Datasets

- Importing the necessary library to plot **confusion matrix**.

```python
from sklearn.metrics import confusion_matrix
```

- Plotting the **Confusion Matrix** using the **single test** dataset of each dataset (**Down-sampled 50/50**, **Down-sampled 20/80** and **Up-Sampled**).

```python
# Defining size of figure and quantity of plots
fig, (ax9, ax10, ax11) = plt.subplots(nrows=1, ncols=3, figsize=(15,5))

# Figure parameters
fig.suptitle('Confusion Matrix - Single Test Dataset', fontsize=16)
# fig.set_figheight(5)
# fig.set_figwidth(10)

# Down-sampled 50/50 Model Plot
# Plotting
sns.heatmap(confusion_matrix(results_ds_5050_1['target'],
                             results_ds_5050_1['y_pred']), cmap='Blues', annot=True, fmt='2.0f', ax=ax9)
# Axis parameters
ax9.set_title('Down-sampled 50/50 Model')
ax9.set_ylabel('F O R E S E E N')
# ax9.set_xlabel('R E A L')
# ax9.xaxis.set_visible(False)

# Down-sampled 05/95 Model
# Plotting
sns.heatmap(confusion_matrix(results_ds_0595_1['target'],
                             results_ds_0595_1['y_pred']), cmap='Greens', annot=True, fmt='2.0f', ax=ax10)
# Axis parameters
ax10.set_title('Down-sampled 05/95 Model')
# ax10.set_ylabel('F O R E S E E N')
ax10.set_xlabel('R E A L')
# ax10.yaxis.set_visible(False)

# Up-Sampled Model
# Plotting
sns.heatmap(confusion_matrix(results_us_1['target'],
                             results_us_1['y_pred']), cmap='Oranges', annot=True, fmt='2.0f', ax=ax11)
# Axis parameters
ax11.set_title('Up-Sampled Model')
# ax11.set_ylabel('F O R E S E E N')
# ax11.set_xlabel('R E A L')
# ax11.yaxis.set_visible(False)
# ax11.xaxis.set_visible(False)

# Saving the plot
plt.savefig(path_figures + 'confusion_matrix_single.png', bbox_inches='tight')

plt.show()
```

### 7.4.2 Test For All Dataset

- Plotting the **Confusion Matrix** using the **test for all** dataset.

```python
# Defining size of figure and quantity of plots
fig, (ax12, ax13, ax14) = plt.subplots(nrows=1, ncols=3, figsize=(15,5))

# Figure parameters
fig.suptitle('Confusion Matrix - Test For All Dataset', fontsize=16)

# Down-sampled 50/50 trained model
# Plotting
sns.heatmap(confusion_matrix(results_ds_5050_2['target'],
                             results_ds_5050_2['y_pred']), cmap='Blues', annot=True, fmt='2.0f', ax=ax12)
# Axis parameters
ax12.set_title('Down-sampled 50/50 Model')
ax12.set_ylabel('F O R E S E E N')
# ax12.set_xlabel('R E A L')

# Down-sampled 05/95 trained model
# Plotting
sns.heatmap(confusion_matrix(results_ds_0595_2['target'],
                             results_ds_0595_2['y_pred']), cmap='Greens', annot=True, fmt='2.0f', ax=ax13)
# Axis parameters
ax13.set_title('Down-sampled 05/95 Model')
# ax13.set_ylabel('F O R E S E E N')
ax13.set_xlabel('R E A L')

# Up-sampled trained model
# Plotting
sns.heatmap(confusion_matrix(results_us_2['target'],
                             results_us_2['y_pred']), cmap='Oranges', annot=True, fmt='2.0f', ax=ax14)
# Axis parameters
ax14.set_title('Up-Sampled Model')
# ax14.set_ylabel('F O R E S E E N')
# ax14.set_xlabel('R E A L')

# Saving the plots
plt.savefig(path_figures + 'confusion_matrix_all.png', bbox_inches='tight')

plt.show()
```