# Import packages & dataset, Data preprocessing

```python
### Import Packages will be used
import sklearn
import numpy as np
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from nltk import word_tokenize
from nltk import download
from nltk.corpus import stopwords
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn.metrics import confusion_matrix
import random
import tensorflow as tf
import math
import itertools
import matplotlib.pyplot as plt

### Load dataset, text preprocessing, prepare train data, valid data & test data
#### Fetch 20 newsgroups Dataset (all) & Text preprocessing
def load_data():
    ##### import and filter data
    newsgroups = fetch_20newsgroups(subset = "all", remove = ("headers", "footers", "quotes"))
    ##### Remove headers, footers and quotes
    ##### as they have too much information about the association between documents
    texts_all, y_all = newsgroups.data, newsgroups.target
    return texts_all, y_all

#### Text preprocessing
download("punkt")
download("stopwords")
stop_words = stopwords.words("english")
def text_preprocess(text):
    text = text.lower()
    doc = word_tokenize(text)
    doc = [word for word in doc if word not in stop_words]
    doc = [word for word in doc if word.isalpha()]
    return doc

#### Removing empty or meaningless emails
def filter_docs(corpus, texts, labels, condition_on_doc):
    """
    Filter corpus, texts and labels given the function condition_on_doc which takes a doc.
    The document doc is kept if condition_on_doc(doc) is true.
    """
    number_of_docs = len(corpus)

    if texts is not None:
        texts = [text for (text, doc) in zip(texts, corpus)
                 if condition_on_doc(doc)]

    labels = [i for (i, doc) in zip(labels, corpus) if condition_on_doc(doc)]
    corpus = [doc for doc in corpus if condition_on_doc(doc)]

    print("{} docs removed".format(number_of_docs - len(corpus)))
    return (corpus, texts, labels)

#### Determine the number of features in preprocessed data
def get_features(texts):
    # Bag-of-words representation
    bow_vectorizer = CountVectorizer()
    bow_all = bow_vectorizer.fit_transform(texts)
    bow_shape_all = bow_all.shape
    bow_feature_names = bow_vectorizer.get_feature_names()
    #print("{} all data points.".format(bow_shape_all[0]))
    #print("{} feature dimension.".format(bow_shape_all[1]))
    return bow_all, bow_feature_names

#### One hot encoding
def onehot_encode(labels):
    values = np.array(labels)
    values = values.reshape(len(values), 1)
    onehot_encoder = OneHotEncoder(sparse = False)
    onehot_encoded = onehot_encoder.fit_transform(values)
    return onehot_encoded

#### Contruct binary matrix
def get_binary_matrix(corpus_matrix):
    n, p = corpus_matrix.shape
    binary_matrix = np.zeros((n, p))
    for i in range(n):
        for j in range(p):
            binary_matrix[i, j] = 0 if corpus_matrix[i, j] == 0 else 1
    return binary_matrix

#### Construct P matrix & normalize it
def get_p_matrix(feature_matrix, top):
    sorted_matrix = np.sort(feature_matrix)
    top_list = sorted_matrix[:, -top]
    n = feature_matrix.shape[0]
    p_matrix = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            if feature_matrix[i, j] >= max(top_list[i], 0.10):
                p_matrix[i, j] = feature_matrix[i, j]
            else:
                p_matrix[i, j] = 0
    p_matrix = p_matrix - np.eye(n)
    return(p_matrix)

def normalize(p_matrix):
    norm = np.linalg.norm(p_matrix)
    if norm == 0:
        return p_matrix
    return p_matrix/norm

#### Obtain all datasets and matrix required for model training step
def data_preprocessed():
    texts_all, y_all = load_data()
    corpus_all = [text_preprocess(text) for text in texts_all]
    corpus_all, texts_all, y_all = filter_docs(corpus_all, texts_all, y_all, lambda doc: (len(doc) != 0))
    bow_all, bow_feature_names = get_features(texts_all)
    mask = bow_all.sum(axis = 0) >= 20
    mask = np.array(mask).ravel()
    bow_all_20 = bow_all[:, mask].toarray()
    word_count_20 = np.log(bow_all_20 + 1)
    return y_all, bow_all_20, word_count_20
y_all, bow_all_20, word_count_20 = data_preprocessed()

def data_management():
    y_onehot = onehot_encode(y_all)
    binary_matrix = get_binary_matrix(bow_all_20.T)
    #### Obtain similarity between any two features
    features = cosine_similarity(binary_matrix, binary_matrix)
    p_matrix = get_p_matrix(features, 26)
    normalized_p_matrix = normalize(p_matrix)
    return y_onehot, features, normalized_p_matrix
y_onehot, features, normalized_p_matrix = data_management()

def data_split(numEntries, numList, seed):
    random.seed(seed)  #### To make sure the result is reproducable
    selected = np.copy(random.sample(range(numEntries), numList))
    NG20_X, NG20_Y = word_count_20[selected, :], y_onehot[selected, :]
    trainX, testX, trainY, testY = train_test_split(NG20_X, NG20_Y, test_size = 0.2, random_state = 18)
    return NG20_X, NG20_Y, trainX, testX, trainY, testY

numEntries = len(y_all)
numList = 2500
trainvalidX, trainvalidY, trainX, testX, trainY, testY = data_split(numEntries = numEntries,
                                                                    numList = numList,
                                                                    seed = 2018)
print(trainvalidX.shape)
print(trainvalidY.shape)
print(trainX.shape)
print(trainY.shape)
print(testX.shape)
print(testY.shape)

```

    [nltk_data] Downloading package punkt to /Users/veronica/nltk_data...
    [nltk_data]   Package punkt is already up-to-date!
    [nltk_data] Downloading package stopwords to
    [nltk_data]     /Users/veronica/nltk_data...
    [nltk_data]   Package stopwords is already up-to-date!
    555 docs removed
    (2500, 11961)
    (2500, 20)
    (2000, 11961)
    (2000, 20)
    (500, 11961)
    (500, 20)

# Training model by minimizing the following loss function
\begin{equation} \label{eq:loss}
loss(\mathbf{w}) = \sum_{i = 1}^{n}l(\mathbf{x_{i}, y_{i}}; \mathbf{w})
						   + \alpha\sum_{j = 1}^{p}\big(\mathbf{w_{j}} - \sum_{k}P_{jk}\mathbf{w_k} \big)^2
						   + \beta||\mathbf{w}||_{2}^{2}
\end{equation}

##### In this part, the accuracy was defined, which actually means the proportion of true positives

```python
numFeatures = features.shape[0]
numLabels = 20
def accuracy(predictions, labels):
    return (100.0 * np.sum(np.argmax(predictions, 1) ==
            np.argmax(labels, 1))/predictions.shape[0])

def network_norm2(trainX, trainY, testX, testY, alpha, beta):
    graph = tf.Graph()
    with graph.as_default():
        ### Input data (constants)
        tf_trainX = tf.constant(trainX, dtype = "float32")
        tf_trainY = tf.constant(trainY, dtype = "float32")
        tf_testX = tf.constant(testX, dtype = "float32")

        ### Set variables
        weights = tf.Variable(tf.truncated_normal([numFeatures, numLabels]))
        biases = tf.Variable(tf.zeros([numLabels]))

        ### Training computation
        logits = tf.matmul(tf_trainX, weights) + biases
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = tf_trainY, logits = logits))

        ### Network Penalty
        network = tf.matmul(weights, tf.cast(np.eye(numFeatures) - normalized_p_matrix, tf.float32),
                            transpose_a = True, transpose_b = True)
        regularizer_Network = tf.nn.l2_loss(network)
        ### Loss function using L2 Regularization
        regularizer_L2 = tf.nn.l2_loss(weights)
        loss = tf.reduce_mean(loss + alpha * regularizer_Network + beta * regularizer_L2)

        optimizer = tf.train.AdamOptimizer(0.5).minimize(loss)
        train_prediction = tf.nn.softmax(logits)
        test_prediction = tf.nn.softmax(tf.matmul(tf_testX, weights) + biases)

    with tf.Session(graph = graph) as session:
        tf.global_variables_initializer().run()
        print("Initialized")
        for step in range(1000):
            _, l, predictions = session.run([optimizer, loss, train_prediction])
            if step % 100 == 0:
                print("Loss at step {}: {}".format(step, l))
                print("Training accuracy: {:.1f}".format(accuracy(predictions, trainY)))
                print("Validation/Test accuracy: {:.1f}".format(accuracy(test_prediction.eval(), testY)))
        train_acc = accuracy(predictions, trainY)
        test_acc = accuracy(test_prediction.eval(), testY)
    return train_acc, test_acc
```

# Conduct 5-fold cross validation to determine the hyperparameters

```python
def get_hyper(base, lower, upper, step):
    power = np.arange(lower, upper, step)
    hyper_list = []
    L = upper - lower
    for p in power:
        hyper = math.pow(base, p)
        hyper_list.append(hyper)
    alpha = list(itertools.chain.from_iterable(itertools.repeat(x, L) for x in hyper_list))
    beta = hyper_list * L
    hyper_parameter = np.vstack((alpha, beta))
    return hyper_parameter

#hyper_parameter = get_hyper(2, -10, 11, 1)
hyper_parameter = get_hyper(10, -4, 0, 1)

def do_k_fold(trainvalidX, trainvalidY, model, hyper_parameter):
    average_accuracy_cross_folds = []
    for i in range(np.shape(hyper_parameter)[1]):
        print("k-fold loop index is: {}".format(i))
        kf = KFold(n_splits = 5, shuffle = False)
        print("split number is: {}".format(kf.get_n_splits(trainvalidX)))
        train_acc_list = []
        valid_acc_list = []
        fold_num = 0
        #### Loop over folds
        for train_index, valid_index in kf.split(trainvalidX):
            fold_num += 1
            print("fold number is: {}".format(fold_num))
            trainX, validX = trainvalidX[train_index], trainvalidX[valid_index]
            trainY, validY = trainvalidY[train_index], trainvalidY[valid_index]
            train_acc, valid_acc = network_norm2(trainX, trainY, validX, validY,
                                                 alpha = hyper_parameter[0, i],
                                                 beta = hyper_parameter[1, i])
            train_acc_list.append(train_acc)
            valid_acc_list.append(valid_acc)

        average_accuracy_cross_folds.append(np.mean(valid_acc_list))
    print("average accuracies are: {}".format(average_accuracy_cross_folds))
    opt_index = np.argmax(average_accuracy_cross_folds)
    return hyper_parameter[:, opt_index], np.take(average_accuracy_cross_folds, opt_index)

best_k_index, best_valid_accuracy = do_k_fold(trainvalidX, trainvalidY, network_norm2, hyper_parameter)

```

# By above step, we got following values:
#### best_k_index: array([0.0001, 0.001 ])
#### best_valid_accuracy: 48.92
# Redefine the model to create confusion matrix:

```python
def network_norm2_update(trainX, trainY, testX, testY, alpha, beta):
    graph = tf.Graph()
    with graph.as_default():
        ### Input data (constants)
        tf_trainX = tf.constant(trainX, dtype = "float32")
        tf_trainY = tf.constant(trainY, dtype = "float32")
        tf_testX = tf.constant(testX, dtype = "float32")

        ### Set variables
        weights = tf.Variable(tf.truncated_normal([numFeatures, numLabels]))
        biases = tf.Variable(tf.zeros([numLabels]))

        ### Training computation
        logits = tf.matmul(tf_trainX, weights) + biases
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = tf_trainY, logits = logits))

        ### Network Penalty
        network = tf.matmul(weights, tf.cast(np.eye(numFeatures) - normalized_p_matrix, tf.float32),
                            transpose_a = True, transpose_b = True)
        regularizer_Network = tf.nn.l2_loss(network)
        ### Loss function using L2 Regularization
        regularizer_L2 = tf.nn.l2_loss(weights)
        loss = tf.reduce_mean(loss + alpha * regularizer_Network + beta * regularizer_L2)

        optimizer = tf.train.AdamOptimizer(0.5).minimize(loss)
        train_prediction = tf.nn.softmax(logits)
        test_prediction = tf.nn.softmax(tf.matmul(tf_testX, weights) + biases)

    with tf.Session(graph = graph) as session:
        tf.global_variables_initializer().run()
        print("Initialized")
        for step in range(1000):
            _, l, predictions = session.run([optimizer, loss, train_prediction])
            if step % 100 == 0:
                print("Loss at step {}: {}".format(step, l))
                print("Training accuracy: {:.1f}".format(accuracy(predictions, trainY)))
                print("Validation/Test accuracy: {:.1f}".format(accuracy(test_prediction.eval(), testY)))
        train_acc = accuracy(predictions, trainY)
        test_acc = accuracy(test_prediction.eval(), testY)
        cls_y = testY.argmax(1)
        cls_yhat = test_prediction.eval().argmax(1)
    return train_acc, test_acc, cls_y, cls_yhat

train_acc, test_acc, cls_y, cls_yhat = network_norm2_update(trainX, trainY, testX, testY,
                                                            alpha = 0.0001, beta = 0.001)

#train_acc, test_acc, cls_y, cls_yhat = network_norm2_update(trainX, trainY, testX, testY,
#                                                            alpha = best_k_index[0], beta = best_k_index[1])
```

    WARNING:tensorflow:From <ipython-input-3-822d74dc9760>:15: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
    Instructions for updating:

    Future major versions of TensorFlow will allow gradients to flow
    into the labels input on backprop by default.

    See tf.nn.softmax_cross_entropy_with_logits_v2.

    Initialized
    Loss at step 0: 113.43694305419922
    Training accuracy: 5.5
    Validation/Test accuracy: 15.2
    Loss at step 100: 1.4227286577224731
    Training accuracy: 95.2
    Validation/Test accuracy: 45.6
    Loss at step 200: 2.5290682315826416
    Training accuracy: 90.1
    Validation/Test accuracy: 41.2
    Loss at step 300: 1.155403733253479
    Training accuracy: 97.5
    Validation/Test accuracy: 48.8
    Loss at step 400: 2.9868111610412598
    Training accuracy: 85.9
    Validation/Test accuracy: 43.6
    Loss at step 500: 1.63816499710083
    Training accuracy: 93.1
    Validation/Test accuracy: 49.0
    Loss at step 600: 1.2645134925842285
    Training accuracy: 96.8
    Validation/Test accuracy: 45.8
    Loss at step 700: 1.2323973178863525
    Training accuracy: 96.0
    Validation/Test accuracy: 48.4
    Loss at step 800: 1.365545392036438
    Training accuracy: 96.8
    Validation/Test accuracy: 47.6
    Loss at step 900: 3.2537412643432617
    Training accuracy: 86.3
    Validation/Test accuracy: 48.0

# Following codes were used to generate confusion matrix:

```python
cnf_matrix = confusion_matrix(cls_y, cls_yhat)
def plot_confusion_matrix(cm, classes, normalize = False, title = "Confusion Matrix", cmap = plt.cm.Purples):
    if normalize:
        cm = cm.astype("float")/cm.sum(axis = 1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')
    print(cm)
    plt.imshow(cm, interpolation = "nearest", cmap = cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation = 45)
    plt.yticks(tick_marks, classes)

    fmt = ".2f" if normalize else "d"
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment = "center",
                 color = "white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel("True label")
    plt.xlabel("Predicted label")

class_names = ["C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10",
               "C11", "C12", "C13", "C14", "C15", "C16", "C17", "C18", "C19", "C20"]

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes = class_names, title = "Un-normalized Confusion Matrix")

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes = class_names, normalize = True, title = "Normalized Confusion Matrix")
plt.show()
```

# Calculate average accuracy of each class, in which the accuracy was defined as $accuracy = \frac{TPs + TNs}{Total}$

```python
def get_average_accuracy(cm):
    TPs = []
    TNs = []
    accuracys = []
    sums = np.sum(cm)
    for i in range(cm.shape[0]):
        TP = cm[i, i]
        TN = sums - np.sum(cm[i, :]) - np.sum(cm[:, i]) + cm[i, i]
        accuracy = (TP + TN)/sums
        TPs = np.append(TPs, TP)
        TNs = np.append(TNs, TN)
        accuracys = np.append(accuracys, accuracy)
        avg_acc = np.mean(accuracys)
    return avg_acc

avg_acc = get_average_accuracy(cnf_matrix)
print(avg_acc)
```

    0.9440000000000002

```python

```


```python
import sys
from __future__ import division
```

```python
import numpy as np
from phasor.utilities.ipynb.displays import *
#from YALL.utilities.tabulate import tabulate

import declarative

from declarative.bunch import (
    DeepBunch
)

import phasor.math.dispatched as dmath
#import phasor.math.dispatch_sympy
```

    Populating the interactive namespace from numpy and matplotlib

```python
import phasor.utilities.version as version
print(version.foundations_version())

from phasor.utilities.np import logspaced
from phasor.utilities.ipynb.sympy import *

from phasor import optics
from phasor import base
from phasor import signals
from phasor import system
from phasor import readouts
from phasor import mechanical
```

    b'2017-09-22 09:45:51 -0400 (a76310c5d4476904171a3f1b18117db454719432)'
    Sympy version:  1.0

```python
d = sympy.var('d')
N = sympy.var('N')
l = sympy.var('l')
A = sympy.var('A')
s = sympy.var('s')
k = sympy.var('k')
F = sympy.var('F')
epsilon = sympy.var('epsilon')
epsilon_0 = sympy.var('epsilon_0')

d_l = sympy.var('l_delta')
V = sympy.var('V')
I = sympy.var('I')
C = sympy.var('C')

Z_e = sympy.var('Z_e', real = True)
Z_m = sympy.var('Z_m', real = True)
k_e = sympy.sqrt(1/sympy.re(Z_e))
k_m = sympy.sqrt(1/sympy.im(Z_m))
k_e = sympy.var('k_e', real = True)
k_m = sympy.var('k_m', real = True)
k_e = sympy.sympify(1)
k_m = sympy.sympify(1)

expr1 = d * F  - I/s + C * V
display(expr1)
expr2 = -d_l * k + F + k * d * V
display(expr2)

rel = sympy.Matrix([[C, -1/s, d, 0],[d*k, 0, 1, -k]])
var = sympy.Matrix([V, I, F, d_l])
rel * var
```

```python
trans = sympy.Matrix([
    [k_e/2,  k_e/2 * Z_e            , 0, 0],
    [k_e/2, -k_e/2 * Z_e.conjugate(), 0, 0],
    [0, 0, k_m/2,  k_m/2 * Z_m            ],
    [0, 0, k_m/2, -k_m/2 * Z_m.conjugate()],
])
trans * var
```

```python
rel_ab = rel * trans**-1
rel_a = rel_ab[:,::2]
rel_b = rel_ab[:,1::2]

```

```python
rel_b
```

```python
rel_atob = -rel_b**-1 * rel_a
rel_atob.simplify()
rel_atob
```

```python
from sympy.utilities.lambdify import lambdastr
```

```python
e = rel_atob[1,1]
def pyval(expr):
    sym = list(expr.free_symbols)
    return lambdastr(sym, expr).split(':')[1].strip()
```

```python
pyval(rel_atob[0,0])
```

    '((C*Z_e*Z_m*s + C*Z_e*k*s - Z_e*Z_m*d**2*k*s - Z_m - k)/(-C*Z_e*Z_m*s - C*Z_e*k*s + Z_e*Z_m*d**2*k*s - Z_m - k))'

```python
pyval(rel_atob[1,1])
```

    '((Z_e*Z_m*d**2*k*s - (Z_m - k)*(C*Z_e*s + 1))/(-Z_e*Z_m*d**2*k*s + Z_m*(C*Z_e*s + 1) + k*(C*Z_e*s + 1)))'

```python
pyval(rel_atob[0,1])
```

    '(2*Z_e*d*k*s/(-C*Z_e*Z_m*s - C*Z_e*k*s + Z_e*Z_m*d**2*k*s - Z_m - k))'

```python
pyval(rel_atob[1,0])
```

    '(2*Z_m*d*k/(-C*Z_e*Z_m*s - C*Z_e*k*s + Z_e*Z_m*d**2*k*s - Z_m - k))'

```python

```


# Tutorial 5: Model Selection: Bias-variance trade-off
**Week 1, Day 3: Model Fitting**

**By Neuromatch Academy**

**Content creators**: Pierre-Étienne Fiquet, Anqi Wu, Alex Hyafil with help from Ella Batty

**Content reviewers**: Lina Teichmann, Patrick Mineault, Michael Waskom

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives

*Estimated timing of tutorial: 25 minutes*

This is Tutorial 5 of a series on fitting models to data. We start with simple linear regression, using least squares optimization (Tutorial 1) and Maximum Likelihood Estimation (Tutorial 2). We will use bootstrapping to build confidence intervals around the inferred linear model parameters (Tutorial 3). We'll finish our exploration of regression models by generalizing to multiple linear regression and polynomial regression (Tutorial 4). We end by learning how to choose between these various models. We discuss the bias-variance trade-off (Tutorial 5) and Cross Validation for model selection (Tutorial 6).

In this tutorial, we will learn about the bias-variance tradeoff and see it in action using polynomial regression models.

Tutorial objectives:

* Understand difference between test and train data
* Compare train and test error for models of varying complexity
* Understand how bias-variance tradeoff relates to what model we choose

```python
# @title Tutorial slides

# @markdown These are the slides for the videos in all tutorials today
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/2mkq4/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

```python
# @title Video 1: Bias Variance Tradeoff
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1dg4y1v7wP", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="NcUH_seBcVw", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

---
# Setup

```python
# Imports

import numpy as np
import matplotlib.pyplot as plt
```

```python
#@title Figure Settings
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
# @title Plotting Functions

def plot_MSE_poly_fits(mse_train, mse_test, max_order):
  """
    Plot the MSE values for various orders of polynomial fits on the same bar
    graph

    Args:
      mse_train (ndarray): an array of MSE values for each order of polynomial fit
      over the training data
      mse_test (ndarray): an array of MSE values for each order of polynomial fit
      over the test data
      max_order (scalar): max order of polynomial fit
  """
  fig, ax = plt.subplots()
  width = .35

  ax.bar(np.arange(max_order + 1) - width / 2, mse_train, width, label="train MSE")
  ax.bar(np.arange(max_order + 1) + width / 2, mse_test , width, label="test MSE")

  ax.legend()
  ax.set(xlabel='Polynomial order', ylabel='MSE', title ='Comparing polynomial fits');
```

```python
# @title Helper functions

def ordinary_least_squares(x, y):
  """Ordinary least squares estimator for linear regression.

  Args:
    x (ndarray): design matrix of shape (n_samples, n_regressors)
    y (ndarray): vector of measurements of shape (n_samples)

  Returns:
    ndarray: estimated parameter values of shape (n_regressors)
  """

  return np.linalg.inv(x.T @ x) @ x.T @ y

def make_design_matrix(x, order):
  """Create the design matrix of inputs for use in polynomial regression

  Args:
    x (ndarray): input vector of shape (n_samples)
    order (scalar): polynomial regression order

  Returns:
    ndarray: design matrix for polynomial regression of shape (samples, order+1)
  """

  # Broadcast to shape (n x 1) so dimensions work
  if x.ndim == 1:
    x = x[:, None]

  #if x has more than one feature, we don't want multiple columns of ones so we assign
  # x^0 here
  design_matrix = np.ones((x.shape[0],1))

  # Loop through rest of degrees and stack columns
  for degree in range(1, order+1):
      design_matrix = np.hstack((design_matrix, x**degree))

  return design_matrix

def solve_poly_reg(x, y, max_order):
  """Fit a polynomial regression model for each order 0 through max_order.

  Args:
    x (ndarray): input vector of shape (n_samples)
    y (ndarray): vector of measurements of shape (n_samples)
    max_order (scalar): max order for polynomial fits

  Returns:
    dict: fitted weights for each polynomial model (dict key is order)
  """

  # Create a dictionary with polynomial order as keys, and np array of theta
  # (weights) as the values
  theta_hats = {}

  # Loop over polynomial orders from 0 through max_order
  for order in range(max_order+1):

    X = make_design_matrix(x, order)
    this_theta = ordinary_least_squares(X, y)

    theta_hats[order] = this_theta

  return theta_hats
```

---
# Section 1: Train vs test data

*Estimated timing to here from start of tutorial: 8 min*

 The data used for the fitting procedure for a given model is the **training data**. In tutorial 4, we computed MSE on the training data of our polynomial regression models and compared training MSE across models. An additional important type of data is **test data**. This is held-out data that is not used (in any way) during the fitting procedure. When fitting models, we often want to consider both the train error (the quality of prediction on the training data) and the test error (the quality of prediction on the test data) as we will see in the next section.

We will generate some noisy data for use in this tutorial using a similar process as in Tutorial 4.However, now we will also generate test data. We want to see how our model generalizes beyond the range of values see in the training phase. To accomplish this, we will generate x from a wider range of values ([-3, 3]). We then plot the train and test data together.

```python
# @markdown Execute this cell to simulate both training and test data

### Generate training data
np.random.seed(0)
n_train_samples = 50
x_train = np.random.uniform(-2, 2.5, n_train_samples) # sample from a uniform distribution over [-2, 2.5)
noise = np.random.randn(n_train_samples) # sample from a standard normal distribution
y_train =  x_train**2 - x_train - 2 + noise

### Generate testing data
n_test_samples = 20
x_test = np.random.uniform(-3, 3, n_test_samples) # sample from a uniform distribution over [-2, 2.5)
noise = np.random.randn(n_test_samples) # sample from a standard normal distribution
y_test =  x_test**2 - x_test - 2 + noise

## Plot both train and test data
fig, ax = plt.subplots()
plt.title('Training & Test Data')
plt.plot(x_train, y_train, '.', markersize=15, label='Training')
plt.plot(x_test, y_test, 'g+', markersize=15, label='Test')
plt.legend()
plt.xlabel('x')
plt.ylabel('y');
```

---
# Section 2: Bias-variance tradeoff

*Estimated timing to here from start of tutorial: 10 min*

 Click here for text recap of video

Finding a good model can be difficult. One of the most important concepts to keep in mind when modeling is the **bias-variance tradeoff**.

**Bias** is the difference between the prediction of the model and the corresponding true output variables you are trying to predict. Models with high bias will not fit the training data well since the predictions are quite different from the true data. These high bias models are overly simplified - they do not have enough parameters and complexity to accurately capture the patterns in the data and are thus **underfitting**.

**Variance** refers to the variability of model predictions for a given input. Essentially, do the model predictions change a lot with changes in the exact training data used? Models with high variance are highly dependent on the exact training data used - they will not generalize well to test data. These high variance models are **overfitting** to the data.

In essence:

* High bias, low variance models have high train and test error.
* Low bias, high variance models have low train error, high test error
* Low bias, low variance models have low train and test error

As we can see from this list, we ideally want low bias and low variance models! These goals can be in conflict though - models with enough complexity to have low bias also tend to overfit and depend on the training data more. We need to decide on the correct tradeoff.

In this section, we will see the bias-variance tradeoff in action with polynomial regression models of different orders.

Graphical illustration of bias and variance.
(Source: http://scott.fortmann-roe.com/docs/BiasVariance.html)

We will first fit polynomial regression models of orders 0-5 on our simulated training data just as we did in Tutorial 4.

```python
# @markdown Execute this cell to estimate theta_hats
max_order = 5
theta_hats = solve_poly_reg(x_train, y_train, max_order)
```

## Coding Exercise 2: Compute and compare train vs test error

We will use MSE as our error metric again. Compute MSE on training data ($x_{train},y_{train}$) and test data ($x_{test}, y_{test}$) for each polynomial regression model (orders 0-5). Since you already developed code in T4 Exercise 4 for making design matrices and evaluating fit polynomials, we have ported that here into the functions `make_design_matrix` and `evaluate_poly_reg` for your use.

*Please think about after completing exercise before reading the following text! Do you think the order 0 model has high or low bias? High or low variance? How about the order 5 model?*

```python
# @markdown Execute this cell for function `evalute_poly_reg`

def evaluate_poly_reg(x, y, theta_hats, max_order):
    """ Evaluates MSE of polynomial regression models on data

    Args:
      x (ndarray): input vector of shape (n_samples)
      y (ndarray): vector of measurements of shape (n_samples)
      theta_hats (dict):  fitted weights for each polynomial model (dict key is order)
      max_order (scalar): max order of polynomial fit

    Returns
      (ndarray): mean squared error for each order, shape (max_order)
    """

    mse = np.zeros((max_order + 1))
    for order in range(0, max_order + 1):
      X_design = make_design_matrix(x, order)
      y_hat = np.dot(X_design, theta_hats[order])
      residuals = y - y_hat
      mse[order] = np.mean(residuals ** 2)

    return mse
```

```python
def compute_mse(x_train, x_test, y_train, y_test, theta_hats, max_order):
  """Compute MSE on training data and test data.

  Args:
    x_train(ndarray): training data input vector of shape (n_samples)
    x_test(ndarray): test data input vector of shape (n_samples)
    y_train(ndarray): training vector of measurements of shape (n_samples)
    y_test(ndarray): test vector of measurements of shape (n_samples)
    theta_hats(dict): fitted weights for each polynomial model (dict key is order)
    max_order (scalar): max order of polynomial fit

  Returns:
    ndarray, ndarray: MSE error on training data and test data for each order
  """

  #######################################################
  ## TODO for students: calculate mse error for both sets
  ## Hint: look back at tutorial 5 where we calculated MSE
  # Fill out function and remove
  # raise NotImplementedError("Student excercise: calculate mse for train and test set")
  #######################################################

  mse_train = evaluate_poly_reg(x_train, y_train, theta_hats, max_order)
  mse_test =  evaluate_poly_reg(x_test, y_test, theta_hats, max_order)

  return mse_train, mse_test

# Compute train and test MSE
mse_train, mse_test = compute_mse(x_train, x_test, y_train, y_test, theta_hats, max_order)

# Visualize
plot_MSE_poly_fits(mse_train, mse_test, max_order)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D3_ModelFitting/solutions/W1D3_Tutorial5_Solution_bb5f169f.py)

*Example output:*

As we can see from the plot above, more complex models (higher order polynomials) have lower MSE for training data. The overly simplified models (orders 0 and 1) have high MSE on the training data. As we add complexity to the model, we go from high bias to low bias.

The MSE on test data follows a different pattern. The best test MSE is for an order 2 model - this makes sense as the data was generated with an order 2 model. Both simpler models and more complex models have higher test MSE.

So to recap:

Order 0 model: High bias, low variance

Order 5 model: Low bias, high variance

Order 2 model: Just right, low bias, low variance

---
# Summary

*Estimated timing of tutorial: 25 minutes*

- Training data is the data used for fitting, test data is held-out data.
- We need to strike the right balance between bias and variance. Ideally we want to find a model with optimal model complexity that has both low bias and low variance
 - Too complex models have low bias and high variance.
 - Too simple models have high bias and low variance.

**Note**
 - Bias and variance are very important concepts in modern machine learning, but it has recently been observed that they do not necessarily trade off (see for example the phenomenon and theory of "double descent")

**Further readings:**
- [The elements of statistical learning](https://web.stanford.edu/~hastie/ElemStatLearn/) by Hastie, Tibshirani and Friedman

---
# Bonus

## Bonus Exercise

Prove the bias-variance decomposition for MSE

\begin{align}
\mathbb{E}_{x}\left[\left(y-\hat{y}(x ; \theta)\right)^{2}\right]=\left(\operatorname{Bias}_{x}[\hat{y}(x ; \theta)]\right)^{2}+\operatorname{Var}_{x}[\hat{y}(x ; \theta)]+\sigma^{2}
\end{align}
where

\begin{align}
\operatorname{Bias}_{x}[\hat{y}(x ; \theta)]=\mathbb{E}_{x}[\hat{y}(x ; \theta)]-y
\end{align}

and
\begin{align}
\operatorname{Var}_{x}[\hat{y}(x ; \theta)]=\mathbb{E}_{x}\left[\hat{y}(x ; \theta)^{2}\right]-\mathrm{E}_{x}[\hat{y}(x ; \theta)]^{2}
\end{align}

Hint: use
\begin{align}
\operatorname{Var}[X]=\mathbb{E}\left[X^{2}\right]-(\mathrm{E}[X])^{2}
\end{align}


```python
import snap
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import networkx as nx
from datetime import datetime
from matplotlib.pyplot import loglog

%matplotlib inline

```

```python
def ProcessEdgeRow(row, Graph):
    src = row['src']
    dst = row['dst']
    timestamp = row['timestamp']

    if not Graph.IsNode(src):
        Graph.AddNode(src)

    if not Graph.IsNode(dst):
        Graph.AddNode(dst)

    if not Graph.IsEdge(src, dst):
        EId = Graph.AddEdge(src, dst)
    else:
        EId = Graph.GetEI(src, dst)

    Graph.AddIntAttrDatE(EId, timestamp, 'timestamp')

    return

```

```python
def GenerateDirectedGraph(df):
    '''
    Returns a TNEANet Graph object
    '''
    Graph = snap.TNEANet.New()
    df.apply(ProcessEdgeRow, axis=1, Graph=Graph)

    return Graph
```

```python
def GenerateGraph(filename='/home/merchantsameer2014/project/dnc-temporalGraph/out.dnc-temporalGraph'):
    ''' read a list of edges
        and return a snap multigraph
    '''
    df = pd.read_csv(filename, sep='\t', header=None)
    df.columns = ['src', 'dst', 'weight', 'timestamp']
    G = GenerateDirectedGraph(df)
    #print "Nodes: ", G.GetNodes()
    #print "Edges: ", G.GetEdges()
    return (df, G)
```

```python
def plotDegreeDistribution(G):
    #
    # Get Degree Distribution
    #
    OutDegToCntV = snap.TIntPrV()
    snap.GetOutDegCnt(G, OutDegToCntV)
    count = 0
    nodeList = []
    degreeList = []
    for item in OutDegToCntV:
        (n, d) = (item.GetVal2(), item.GetVal1())
        nodeList.append(n)
        degreeList.append(d)
    x = np.array( [ np.log10(item.GetVal1()) for itemm in OutDegToCntV if item.GetVal1() > 0 ] )
    y = np.array( [ np.log10(item.GetVal2()) for item in OutDegToCntV if item.GetVal2() > 0 ] )
    #
    # Plot Degree Distribution
    #
    plt.figure(figsize=(15,15))
    loglog(degreeList, nodeList, 'bo')
    #plt.plot(x_plot, 10**b*x_plot**a, 'r-')
    plt.title("LogLog plot of out-degree distribution")
    plt.show()
    return
```

### 1. Compute Degree Centrality

```python
def computeDegreeCentrality(G, NodeAttributes):
    #
    # 1. Degree Centrality
    #    Get In Degree and Out Degree for each node
    #
    InDegV = snap.TIntPrV()
    OutDegV = snap.TIntPrV()

    snap.GetNodeOutDegV(G, OutDegV)
    snap.GetNodeInDegV(G, InDegV)

    InDegreeList = [ (item.GetVal1(), item.GetVal2()) for item in InDegV ]
    OutDegreeList = [ (item.GetVal1(), item.GetVal2()) for item in OutDegV ]

    InDegreeList.sort(key=lambda x: x[1], reverse=True)
    OutDegreeList.sort(key=lambda x: x[1], reverse=True)

    minOutDegree = min(OutDegreeList, key = lambda x: x[1])[1]
    maxOutDegree = max(OutDegreeList, key = lambda x: x[1])[1]
    minInDegree = min(InDegreeList, key = lambda x: x[1])[1]
    maxInDegree = max(InDegreeList, key = lambda x: x[1])[1]

    #
    # Sanity Check
    #print maxOutDegree, minOutDegree, maxInDegree, minInDegree
    #print InDegreeList[0], InDegreeList[-1]

    for (nodeId, Degree) in InDegreeList:
        if not NodeAttributes.get(nodeId, None):
            NodeAttributes[nodeId] = dict()
        NodeAttributes[nodeId]['InDegree'] = Degree
        normalizedDegree = (float(Degree) - float(minInDegree))/(float(maxInDegree - float(minInDegree)))
        NodeAttributes[nodeId]['NormInDegree'] = normalizedDegree

    for (nodeId, Degree) in OutDegreeList:
        NodeAttributes[nodeId]['OutDegree'] = Degree
        normalizedDegree = (float(Degree) - float(minOutDegree))/(float(maxOutDegree - float(minOutDegree)))
        NodeAttributes[nodeId]['NormOutDegree'] = normalizedDegree

    #
    # Sanity Check
    #
    #print NodeAttributes[1874]
    #print NodeAttributes[893]

    return NodeAttributes
```

### 2. Compute Clustering Coefficient

```python
def computeClusteringCoeff(G, NodeAttributes):
    NIdCCfH = snap.TIntFltH()
    snap.GetNodeClustCf(G, NIdCCfH)

    ClusterCoeffList = list()
    for nodeId in NIdCCfH:
        NodeAttributes[nodeId]['ClusterCoeff'] = NIdCCfH[nodeId]
        ClusterCoeffList.append((nodeId, NIdCCfH[nodeId]))

    ClusterCoeffList.sort(key=lambda x: x[1], reverse=True)
    minClusterCoeff = min(ClusterCoeffList, key=lambda x: x[1])[1]
    maxClusterCoeff = max(ClusterCoeffList, key=lambda x: x[1])[1]

    #
    # Sanity Check
    #
    print ClusterCoeffList[1], maxClusterCoeff, ClusterCoeffList[-1], minClusterCoeff

    NIdCCfH = snap.TIntFltH()
    snap.GetNodeClustCf(G, NIdCCfH)
    ClusterCoeffList = list()
    for nodeId in NIdCCfH:
        clusterCoeff = NIdCCfH[nodeId]
        normClusterCoeff = (clusterCoeff - minClusterCoeff)/(maxClusterCoeff - minClusterCoeff)
        NodeAttributes[nodeId]['NormClusterCoeff'] = normClusterCoeff

    #print NodeAttributes[2012]
    return NodeAttributes
```

### 3. Compute Avergate shortest path length

```python
def computeAvgShortestPath(G, NodeAttributes):
    nodeCount = float(G.GetNodes() - 1)
    avgShortPathLenList = list()
    for src in G.Nodes():
        srcId = src.GetId()
        totalShortPathLength = 0

        for dst in G.Nodes():
            dstId = dst.GetId()
            #
            # Skip Self Edges
            #
            if srcId == dstId:
                continue

            #
            # Compute Shortest Path
            #
            l = snap.GetShortPath(G, srcId, dstId, True)

            #
            # Skip nodes that cannot be reached
            #
            if l < 0:
                continue

            totalShortPathLength += float(l)
        NodeAttributes[srcId]['AvgShortPathLen'] = totalShortPathLength/nodeCount
        avgShortPathLenList.append((srcId, totalShortPathLength/nodeCount))

    minAvgShortPathLength = min(avgShortPathLenList, key=lambda x: x[1])[1]
    maxAvgShortPathLength = max(avgShortPathLenList, key=lambda x: x[1])[1]

    for (node, spLen) in avgShortPathLenList:
        normAvgShortPath = (spLen - minAvgShortPathLength)/(maxAvgShortPathLength - minAvgShortPathLength)
        NodeAttributes[node]['normAvgShortPathLen'] = normAvgShortPath

    #print NodeAttributes[480]

    return NodeAttributes
```

### 4. Compute Betweeness Centrality

```python
def computeBetweenessCentrality(G, NodeAttributes):
    Nodes = snap.TIntFltH()
    Edges = snap.TIntPrFltH()
    BetweenessNodeList = list()
    BetweenessEdgeList = list()

    snap.GetBetweennessCentr(G, Nodes, Edges, 1.0)
    for node in Nodes:
        NodeAttributes[node]['Betweeness'] = Nodes[node]
        BetweenessNodeList.append((node, Nodes[node]))

    for edge in Edges:
        #print "edge: (%d, %d) centrality: %f" % (edge.GetVal1(), edge.GetVal2(), Edges[edge])
        BetweenessEdgeList.append((edge.GetVal1(), edge.GetVal2(), Edges[edge]))

    BetweenessNodeList.sort(key=lambda x: x[1], reverse=True)
    BetweenessEdgeList.sort(key=lambda x: x[2], reverse=True)

    #print BetweenessNodeList[0], BetweenessNodeList[-1]

    minBetweeness = BetweenessNodeList[-1][1]
    maxBetweeness = BetweenessNodeList[0][1]
    for (node, betweeness) in BetweenessNodeList:
        normBetweeness = (betweeness - minBetweeness)/(maxBetweeness - minBetweeness)
        NodeAttributes[node]['normBetweeness'] = normBetweeness

    #print NodeAttributes[1669]
    #print NodeAttributes[884]

    return NodeAttributes
```

### 5.  Compute Auth and Hub Score

```python
def computeAuthHubScore(G, NodeAttributes):
    NIdHubH = snap.TIntFltH()
    NIdAuthH = snap.TIntFltH()
    snap.GetHits(G, NIdHubH, NIdAuthH)
    HubNodes = []
    for nodeId in  NIdHubH:
        HubNodes.append((nodeId,  NIdHubH[nodeId]))
        NodeAttributes[nodeId]['HubScore'] = NIdHubH[nodeId]

    HubNodes.sort(key = lambda x: x[1], reverse=True)

    AuthNodes = []
    for nodeId in  NIdAuthH:
        AuthNodes.append((nodeId,  NIdAuthH[nodeId]))
        NodeAttributes[nodeId]['AuthScore'] = NIdAuthH[nodeId]

    AuthNodes.sort(key = lambda x: x[1], reverse=True)

    #print AuthNodes[0], AuthNodes[-1]
    #print HubNodes[0], HubNodes[-1]

    minAuthNodes = AuthNodes[-1][1]
    maxAuthNodes = AuthNodes[0][1]
    minHubNodes = HubNodes[-1][1]
    maxHubNodes = HubNodes[0][1]

    for (node, hubScore) in HubNodes:
        normHubScore = (hubScore - minHubNodes)/(maxHubNodes - minHubNodes)
        NodeAttributes[node]['normHubScore'] = normHubScore

    for (node, authScore) in AuthNodes:
        normAuthScore = (authScore - minAuthNodes)/(maxAuthNodes - minAuthNodes)
        NodeAttributes[node]['normAuthScore'] = normAuthScore

    #print NodeAttributes[1874]
    #print NodeAttributes[893]
    return NodeAttributes
```

## Get Raw Statistics

    1. Degree Centrality
    2. Clustering Coefficient
    3. Mean shortest path from each node
    4. Betweeness Centralitily
    5. Hub and Authority Score

```python
def GetRawStatistics(G, NodeAttributes):

    #
    # Step 1: Compute Degree Centrality
    #
    print "Computing Degree Centrality... "
    computeDegreeCentrality(G, NodeAttributes)

    #
    # Step 2: Compute Clustering Coefficients
    #
    print "Computing Clustering coefficient..."
    computeClusteringCoeff(G, NodeAttributes)

    #
    # Step 3: Avg. Shortest path length
    #
    print "Computing Average Shortest path length ... "
    computeAvgShortestPath(G, NodeAttributes)

    #
    # Step 4: Betweeness Centrality
    #
    print "Computing Betweeness centrality... "
    computeBetweenessCentrality(G, NodeAttributes)

    #
    # Step 5: Hub and Auth Score
    #
    print "Computing Hub and Auth Score ... "
    computeAuthHubScore(G, NodeAttributes)

    return
```

## Automatic Social Hierarchy Detection From Email Network

### Steps

    1. Compute node's importance on response time for email
    2. Get all Cliques (Algorithm 457)
    3. Number of clique each node is part of
    4. Raw Clique Score computed using

 $$R*2^{n-1}$$

    5. Weighted Clique Score (Based on importance of a person)

 $$W = t*2^{n-1}$$

### Use networkx library for Clique analysis

    1. Build a multigraph with edges for each (src, dst, timestamp) entry in the email data set
    2. Build undirected graph with edges between nodes only when email count exceeds N=4 between two those nodes

```python
def ProcessNxEdgeRow(row, Graph):
    src = row['src']
    dst = row['dst']
    timestamp = row['timestamp']

    Graph.add_node(src)
    Graph.add_node(dst)
    Graph.add_edge(src, dst, timestamp=timestamp)

    return

```

```python
def GenerateDirectedNxGraph(df):
    '''
    Returns a TNEANet Graph object
    '''
    Graph = nx.MultiDiGraph(name="DNC Email Network")
    df.apply(ProcessNxEdgeRow, axis=1, Graph=Graph)

    #print "Networkx Nodes: ", Graph.number_of_nodes()
    #print "Networkx Edges: ", Graph.number_of_edges()

    return Graph
```

### Generate undirected graph with edges between nodes with email count > N

#### Prune all edges with N <=4 emails exchanged
As per the paper - consider edges between nodes only if the nodes have exchanged > N messages
N is a tunable parameter

DNC email has a median of message exchanged = 2

```python
def GeneratePrunedDirectedGraph(GNx,  N = 4):

    edgeCount = dict()
    for edge in GNx.edges():
        if not edgeCount.get(edge, None):
            edgeCount[edge] = 0
        edgeCount[edge] += 1

    emailDist= [ v for k, v in edgeCount.iteritems() ]
    emailDist.sort(reverse=True)
    #print emailDist[0], emailDist[-1], np.median(emailDist), len(emailDist)

    pruneList = [ k for k, v in edgeCount.iteritems() if v <= N ]
    prunedEdgeList = [ k for k, v in edgeCount.iteritems() if v > N ]

    uGNx = nx.Graph()

    for edge in prunedEdgeList:
        (src, dst) = edge
        uGNx.add_edge(int(src), int(dst))

    #print "Networkx Nodes: ", uGNx.number_of_nodes()
    #print "Networkx Edges: ", uGNx.number_of_edges()

    return uGNx
```

### Step 1: Compute Nodes importance on email response time

### Approach 1: Based on Rowe et. al. paper

Use response time to measure importance of a node

    1. For each node get all the out bound email timestamps
    2. For each email sent - check the response time from the receiver
    3. Consider responses within a day for computing time score.
    4. Consider response time for nodes that have exchanged at least 100 emails
       (fewer emails with high response time can falsely promote node based on time score)

```python
def getEmailSentByNode(GNx):
    temporalMap = dict()
    for n, nbrs in GNx.adjacency():
        temporalMap[n] = dict()
        for nbr, edict in nbrs.items():
            t1 = [ d['timestamp'] for d in edict.values() ]
            t1.sort()
            temporalMap[n][nbr] = t1

    return temporalMap
```

```python
def getAvgResponseTime(temporalMap):
    avgNodeResponse = list()

    for src, destinations in temporalMap.iteritems():
        totalRequestResp = 0
        totalResponseTime = 0.0

        for dst, reqTimestamps in destinations.iteritems():
            responseTime = None
            #
            # Walk through ALl requests sent to a destination
            #
            for req in reqTimestamps:
                reqTime = datetime.fromtimestamp(req)

                #
                # Look for response time from the destinaton to ths source
                #
                for resp in temporalMap[dst].get(src, list()):
                    respTime = datetime.fromtimestamp(resp)

                    if resp < req:
                        #
                        # Look for first response after the request time
                        #
                        continue

                    deltaTime = respTime - reqTime

                    if deltaTime.total_seconds() > 86400:
                        #
                        # If response time exceeds a day don't
                        # consider it as a response
                        break

                    #print "Found Response time: src: %d, dst: %d req: %s, resp: %s" % (src, dst, reqTime, respTime)

                    totalRequestResp += 1
                    totalResponseTime += deltaTime.total_seconds()
                    break
        #
        # Compute average across all dst response times
        #
        if totalRequestResp > 0 and totalResponseTime > 0:
            avgResponse = totalResponseTime/totalRequestResp
        else:
            #
            # Set default response to 7 day
            #
            avgResponse = float(7*86400)
        #
        # Lower response Higher the timeScore
        #
        timeScore = 1/avgResponse
        avgNodeResponse.append((src, timeScore, totalResponseTime, totalRequestResp ))

    avgNodeTimeScore = list()

    #
    # Ignore response time for email exchanges fewer than 10
    #
    for x in avgNodeResponse:
        (src, timeScore, totalResponseTime, totalRequestResp) = x

        if totalRequestResp <= 200:
            timeScore = 1.0/float(7*86400)

        avgNodeTimeScore.append((src, timeScore, totalResponseTime, totalRequestResp))

    avgNodeTimeScore.sort(key=lambda x: x[1], reverse=True)

    #print "\nLast Node avg response time: ", avgNodeTimeScore[-1]
    #print "\nTop top two nodes avg response time:", avgNodeTimeScore[:2]

    normTimeScore = dict()
    minAvgTimeScore = avgNodeTimeScore[-1][1]
    maxAvgTimeScore = avgNodeTimeScore[0][1]

    for (node, timeScore, _, _) in avgNodeTimeScore:
        normTimeScore[node] = (timeScore - minAvgTimeScore)/(maxAvgTimeScore - minAvgTimeScore)

    #print "Node 1625 normTimeScore: ", normTimeScore[1625]

    return normTimeScore

```

```python
def GetSocialAttributes(df, NodeAttributes, threshold=0):

    # Generate NetworkX graph
    #
    GNx = GenerateDirectedNxGraph(df)

    #
    # Prune graph based on number of emails exchanged
    # Keep edges with edge weight > 4 (exchanged > 4 emails)
    #

    uGNx = GeneratePrunedDirectedGraph(GNx, N=threshold)

    temporalMap = getEmailSentByNode(GNx)
    normTimeScore = getAvgResponseTime(temporalMap)

    nodeCliqueCount = dict()
    rawCliqueScore = dict()
    weightedCliqueScore = dict()

    #
    # Find all maximal cliques
    #
    for clique in nx.find_cliques(uGNx):
        for node in clique:

            if not nodeCliqueCount.get(node, None):
                nodeCliqueCount[node] = 0

            if not rawCliqueScore.get(node, None):
                rawCliqueScore[node] = 0

            if not weightedCliqueScore.get(node, None):
                weightedCliqueScore[node] = 0

            nodeCliqueCount[node] += 1

            n = len(clique)

            rawCliqueScore[node] += 2**n

            weightedCliqueScore[node] += (2**n)*normTimeScore[node]
    #
    # Get a sorted list of nodes based on their clique count
    #
    nodeCliqueList = [ (k, v) for k, v in nodeCliqueCount.iteritems() ]
    nodeCliqueList.sort(key=lambda x: x[1], reverse=True )

    rawCliqueList = [ (k, v) for k, v in rawCliqueScore.iteritems() ]
    rawCliqueList.sort(key=lambda x: x[1], reverse=True )

    weightedCliqueList = [ (k, v) for k, v in weightedCliqueScore.iteritems() ]
    weightedCliqueList.sort(key=lambda x: x[1], reverse=True )

    #print "\n Top 10 nodes based on Cliques count: " , nodeCliqueList[:10]
    #print "\n Top 10 nodes based on Raw Clique size: ",  rawCliqueList[:10]
    #print "\n Top 10 nodes based on weighted Clique size: " , weightedCliqueList[:10]

    minNodeCliqueCount = nodeCliqueList[-1][1]
    maxNodeCliqueCount = nodeCliqueList[0][1]

    minRawCliqueScore = rawCliqueList[-1][1]
    maxRawCliqueScore = rawCliqueList[0][1]

    minWeightedCliqueScore = weightedCliqueList[-1][1]
    maxWeightedCliqueScore = weightedCliqueList[0][1]

    for node, score in nodeCliqueList:
        NodeAttributes[node]['nodeClique'] = score
        NodeAttributes[node]['normNodeClique'] = float(score - minNodeCliqueCount)/float(maxNodeCliqueCount - minNodeCliqueCount)

    for node, score in rawCliqueList:
        NodeAttributes[node]['rawClique'] = score
        NodeAttributes[node]['normRawClique'] = float(score - minRawCliqueScore)/float(maxRawCliqueScore - minRawCliqueScore)

    for node, score in weightedCliqueList:
        NodeAttributes[node]['weightedClique'] = score
        NodeAttributes[node]['normWeightedClique'] = float(score - minWeightedCliqueScore)/float(maxWeightedCliqueScore - minWeightedCliqueScore)

    #print "\nNode 1874 features: ", NodeAttributes[1874]

    return
```

## Modified version of Social Score

Social hierarchy can be determined by how often a person receives a response to their mail within first 10 mails sent by the receipient. Instead of using time as a measure which can be skewed due to different time zones or working hours, a priority of mails can be a better indicator of a persons importance in the organization

```python
def getNodeResponseList(GNx):
    nodeResponseTuple = dict()
    nodeAdj = dict()

    for n, nbrs in GNx.adjacency():
        nodeAdj[n] = dict()
        nodeResponseTuple[n] = list()
        for nbr, edict in nbrs.items():
            t1 = [ d['timestamp'] for d in edict.values() ]
            t2 = [nbr]*len(t1)
            response = zip(t1,t2)
            response.sort(key=lambda x: x[0])
            nodeResponseTuple[n].extend(response)

            if nbr not in nodeAdj[n]:
                nodeAdj[n][nbr] = list()
            nodeAdj[n][nbr].extend(t1)

    return nodeAdj, nodeResponseTuple
```

```python
def getAvgResponseScore(nodeAdj, nodeResponseTuple):
    avgNodeResponse = list()

    for src, destinations in nodeAdj.iteritems():
        totalRequests = 0.0
        totalPriorityResponse = 0.0
        #print "Processing %r" % src
        for dst, reqTimestamps in destinations.iteritems():
            #
            # Walk through ALl requests sent to a destination
            #
            for req in reqTimestamps:
                totalRequests += 1
                #
                # check if destination responded to src within next 5 mails
                #
                count = 0
                found_response = False
                for resp, node in nodeResponseTuple[dst]:
                    if resp < req:
                        #
                        # Look for first response after the request time
                        #
                        continue

                    count += 1
                    found_response = True
                    if node == src:
                        #print "Found a response in top 5.."
                        break

                    if count > 5:
                        break

                if found_response and (count <= 5):
                    totalPriorityResponse += 1
                    break
        #
        # Compute average across all dst response times
        #
        #
        #print "src: %r, total mails sent %r, priority: %r" % (src, totalRequests, totalPriorityResponse)
        if totalRequests > 0 and totalPriorityResponse > 0:
            avgResponse = totalPriorityResponse/totalRequests
        else:
            #
            # Set default priority respone to zero
            #
            avgResponse = 0

        avgNodeResponse.append((src, avgResponse, totalRequests, totalPriorityResponse ))

    avgNodeResponseScore = list()

    #
    # Ignore response time for email exchanges fewer than 200
    #
    for x in avgNodeResponse:
        (src, avgResponse, totalRequests, totalPriorityResponse) = x

        if totalRequests <= 200:
            avgResponse = 0

        avgNodeResponseScore.append((src, avgResponse, totalRequests, totalPriorityResponse))

    avgNodeResponseScore.sort(key=lambda x: x[1], reverse=True)

    #print "\nLast Node avg response score: ", avgNodeResponseScore[-1]
    print "\nTop top nodes avg response score:", avgNodeResponseScore[:10]

    normResponseScore = dict()
    minAvgResponseScore = avgNodeResponseScore[-1][1]
    maxAvgResponseScore = avgNodeResponseScore[0][1]

    for (node, responseScore, _, _) in avgNodeResponseScore:
        normResponseScore[node] = (responseScore - minAvgResponseScore)/(maxAvgResponseScore - minAvgResponseScore)

    #print "\nNode 1625 normTimeScore: ", normResponseScore[1625]

    return normResponseScore

```

```python
def GetModifiedSocialAttributes(df, NodeAttributes, threshold=0):

    # Generate NetworkX graph
    #
    GNx = GenerateDirectedNxGraph(df)

    #
    # Prune graph based on number of emails exchanged
    # Keep edges with edge weight > 4 (exchanged > 4 emails)
    #

    uGNx = GeneratePrunedDirectedGraph(GNx, N=threshold)

    nodeAdj, nodeResponseTuple = getNodeResponseList(GNx)
    normResponseScore = getAvgResponseScore(nodeAdj, nodeResponseTuple)

    nodeCliqueCount = dict()
    rawCliqueScore = dict()
    weightedCliqueScore = dict()

    #
    # Find all maximal cliques
    #
    for clique in nx.find_cliques(uGNx):
        for node in clique:

            if not nodeCliqueCount.get(node, None):
                nodeCliqueCount[node] = 0

            if not rawCliqueScore.get(node, None):
                rawCliqueScore[node] = 0

            if not weightedCliqueScore.get(node, None):
                weightedCliqueScore[node] = 0

            nodeCliqueCount[node] += 1

            n = len(clique)

            rawCliqueScore[node] += 2**n

            weightedCliqueScore[node] += (2**n)*normResponseScore[node]
    #
    # Get a sorted list of nodes based on their clique count
    #
    nodeCliqueList = [ (k, v) for k, v in nodeCliqueCount.iteritems() ]
    nodeCliqueList.sort(key=lambda x: x[1], reverse=True )

    rawCliqueList = [ (k, v) for k, v in rawCliqueScore.iteritems() ]
    rawCliqueList.sort(key=lambda x: x[1], reverse=True )

    weightedCliqueList = [ (k, v) for k, v in weightedCliqueScore.iteritems() ]
    weightedCliqueList.sort(key=lambda x: x[1], reverse=True )

    #print "\n Top 10 nodes based on Cliques count: " , nodeCliqueList[:10]
    #print "\n Top 10 nodes based on Raw Clique size: ",  rawCliqueList[:10]
    print "\n Top 10 nodes based on weighted Clique size: " , weightedCliqueList[:10]

    minNodeCliqueCount = nodeCliqueList[-1][1]
    maxNodeCliqueCount = nodeCliqueList[0][1]

    minRawCliqueScore = rawCliqueList[-1][1]
    maxRawCliqueScore = rawCliqueList[0][1]

    minWeightedCliqueScore = weightedCliqueList[-1][1]
    maxWeightedCliqueScore = weightedCliqueList[0][1]

    for node, score in nodeCliqueList:
        if not NodeAttributes.get(node, None):
            NodeAttributes[node] = dict()
        NodeAttributes[node]['nodeClique'] = score
        NodeAttributes[node]['normNodeClique'] = float(score - minNodeCliqueCount)/float(maxNodeCliqueCount - minNodeCliqueCount)

    for node, score in rawCliqueList:
        NodeAttributes[node]['rawClique'] = score
        NodeAttributes[node]['normRawClique'] = float(score - minRawCliqueScore)/float(maxRawCliqueScore - minRawCliqueScore)

    for node, score in weightedCliqueList:
        NodeAttributes[node]['weightedClique'] = score
        NodeAttributes[node]['normWeightedClique'] = float(score - minWeightedCliqueScore)/float(maxWeightedCliqueScore - minWeightedCliqueScore)

    #print "\nNode 1874 features: ", NodeAttributes[1874]

    return
```

## Compute Weighted Social Score

### Social Score is computed as follows

\begin{align}
w_x * C_x & = w_x * 100 * \left[ \frac{x_i - infx}{sup\ x - inf\ x} \right] \\
\\
score & = \frac{\Sigma_{all\ x} w * C_x}{\Sigma_{all\ x} w}
\end{align}

```python
def ComputeSocialScore(NodeAttributes):

    socialScore = list()

    #
    # Weights:
    #
    # W = [ Weighted Clique, RawClique, NumClique, OutDegree, InDegree, Cluster Coeff, Betweeness, Avg ShortPath, Auth, Hub]
    #
    w_weightedClique = 0.9
    w_rawClique = 0.8
    w_numClique = 0.7
    w_outDegree = 0.6
    w_inDegree = 0.6
    w_clusterCoeff = 0.3
    w_betweeness = 0.3
    w_shortpath = 0.5
    w_auth = 0.2
    w_hub = 0.1

    w = np.array([w_weightedClique,\
                 w_rawClique, \
                 w_numClique, \
                 w_outDegree, \
                 w_inDegree, \
                 w_clusterCoeff, \
                 w_betweeness, \
                 w_shortpath, \
                 w_auth, \
                 w_hub])

    w_sum = np.sum(w)

    for node, attributes in NodeAttributes.iteritems():
        weightedClique = attributes.get('normWeightedClique', 0.0)
        rawClique = attributes.get('normRawClique', 0.0)
        numClique = attributes.get('normNodeClique', 0.0)
        outDeg =  attributes.get('NormOutDegree', 0.0)
        inDeg =  attributes.get('NormInDegree', 0.0)
        clusterCoeff =  attributes.get('NormClusterCoeff', 0.0)
        betweeness = attributes.get('normBetweeness', 0.0)
        shortestPath = attributes.get('normAvgShortPathLen', 0.0)
        authScore = attributes.get('normAuthScore', 0.0)
        hubSccore = attributes.get('normHubScore', 0.0)

        C_x = np.array([weightedClique, \
                       rawClique, \
                       numClique, \
                       outDeg, \
                       inDeg, \
                       clusterCoeff, \
                       betweeness, \
                       shortestPath, \
                       authScore, \
                       hubSccore])

        score = 100.0 * np.dot(w, C_x)/w_sum
        socialScore.append((node, score))

    socialScore.sort(key=lambda x: x[1], reverse=True)
    print socialScore[:10]

    return socialScore
```

```python
def ComputeNodeScore(df, G, modified=False, threshold=0):
    #
    # Create a Node Attribute Dictionary
    #
    NodeAttributes = dict()

    #
    # Get Node Statistics and Graph attributes
    #
    GetRawStatistics(G, NodeAttributes)

    #
    # Get Social Attributes for nodes
    #
    if modified:
        print "Get Modified Social Score Attributes... "
        GetModifiedSocialAttributes(df, NodeAttributes, threshold=threshold)
    else:
        print "Get Social Score Attributes... "
        GetSocialAttributes(df, NodeAttributes, threshold=threshold)

    print "Compute Social Score... "
    socialScore = ComputeSocialScore(NodeAttributes)

    return socialScore, NodeAttributes

```

```python
if __name__ == '__main__':

    df, G = GenerateGraph(filename='/home/merchantsameer2014/project/dnc-temporalGraph/out.dnc-temporalGraph')

    start_time = datetime.now()
    socialScore, NodeAttributes = ComputeNodeScore(df, G, modified=False, threshold=0)
    end_time = datetime.now()
    print('Total time to compute Social Score: {}'.format(end_time - start_time))

    with open('socialScoreModifed_w07.txt', 'w') as fd:
        for (node, score) in socialScore:
            fd.write("%r\t%r\n" % (node, score))

    '''
    with open('../results/nodeSocialScoreFeature.txt', 'w') as fd:
        fd.write("node,normNodeClique,normRawClique,normWeightedClique,nodeClique,rawClique,weightedClique\n")
        for (node, attributes) in NodeAttributes.iteritems():
            fd.write("%r,%r,%r,%r,%r,%r,%r\n" % (node,\
                                        attributes.get('normNodeClique', 0.0),\
                                        attributes.get('normRawClique', 0.0),\
                                        attributes.get('normWeightedClique', 0.0),\
                                        attributes.get('nodeClique', 0.0),\
                                        attributes.get('rawClique', 0.0),\
                                        attributes.get('weigthedClique', 0.0),\
                                       ))

    '''
```

    Computing Degree Centrality...
    Computing Clustering coefficient...
    (1753, 1.0) 1.0 (1381, 0.0) 0.0
    Computing Average Shortest path length ...
    Computing Betweeness centrality...
    Computing Hub and Auth Score ...
    Get Social Score Attributes...
    Compute Social Score...
    [(1874, 80.78021754303406), (1258, 55.4659080081707), (999, 51.65134719039465), (1669, 42.936504335383276), (511, 41.144926456992756), (1998, 38.94771394756082), (585, 38.78737308016589), (1440, 34.7177822103923), (852, 27.237611856350544), (1278, 26.892696751894427)]
    Total time to compute Social Score: 0:06:06.993746

```python

```


```python
from Pseudo_Goldstone.Get_Feynman_Rules import Make_Feynman_Rules, VertexValue
from Pseudo_Goldstone import Tuples
import os
import sympy as sp
import numpy as np
import re
from IPython.display import Math

def ApplyTex(expr):
    tmpE=sp.latex(expr)
    for i in SubsTex:
        tmpE=sp.latex(tmpE).replace(i[0],i[1])

    return tmpE

def ApplyTexfields(expr):
    tmpE=sp.latex(map(eval,expr))
    for i in SubsTex:
        tmpE=sp.latex(tmpE).replace(i[0],i[1])

    return tmpE

def MathNice(expr):

    return Math(ApplyTex(expr))
```

```python

```

```python
dimN=2
gauge='feyn'
Dir=r'Frules//SU{}/'.format(dimN)
I=sp.I
sqrt=lambda x: sp.sqrt(x)
conjugate=lambda x: sp.conjugate(x)
Abs=lambda x: sp.Abs(x)

Read_Dict={}
for i in os.listdir(Dir):
    if gauge in i:

        a = re.search(r'.*\.(.*)',i).group(1)
        Frules_in=open(Dir+'/'+str(i),'r')
        if a=='vrt':
            point= re.search(r'.*\_(.)-point.',i).group(1)
            Read_Dict[point+'-point']=Frules_in.read().split('\n')
            Read_Dict[point+'-point'].remove('')
            Read_Dict[point+'-point']=np.array(Read_Dict[point+'-point'])
        else:
            Read_Dict[a]=Frules_in.read().split('\n')
            for i in range(100):
                try:
                    Read_Dict[a].remove('')
                except:
                    break
            Read_Dict[a]=np.array(Read_Dict[a])
        Frules_in.close()

for i in Read_Dict['fields']:
    sp.var(i)
#Read_Dict.pop('fields',None)#I need the fields

param_it=iter(Read_Dict['parameters'])
while True:
    try:
        Sym=param_it.next()
        tmp=param_it.next()

        assume=re.search('\[(.*)\,(.*)\,(.*)\]',tmp)
        sp.var(Sym, complex=assume.group(1),real=assume.group(2),positive=assume.group(3) )
        #print eval(Sym), eval(Sym+'.assumptions0')
        #print '..........................'

    except StopIteration:
        Read_Dict.pop('parameters',None)
        break

SubsTex=Read_Dict['subs'][:]
SubsTex=np.array(list(reversed(SubsTex)))

Read_Dict.pop('subs',None)
def Convert(line):
    tmp=line.replace(' ','').replace('\n','').split('|')
    tmp[0]=tmp[0].replace('(','_._').replace(')','_._').replace(',','_._').split('_._')
    while True:
        try:
            tmp[0].remove('')
        except ValueError:
            break
    p=len(tmp[0])
    if p==2:
        #-1 because I need to convert it in order to get the correct result (from VertexValue) after Make_Feynman_Rules
        return [ map(eval,tmp[0]), eval(tmp[1]),-1 ]
    else:
        #I because I need to convert it in order to get the correct result (from VertexValue) after Make_Feynman_Rules
        return [ map(eval,tmp[0]), eval(tmp[1]),I ]

VConvert=np.vectorize(Convert)

Frules={}
Read_Dict_Tmp=Read_Dict.copy()
for i in Read_Dict:
    if '-point' in i:
        j=eval(i.replace('-point',''))
        Frules[j]=VConvert(np.array(Read_Dict[i]))
        Read_Dict_Tmp.pop(i,None)

Read_Dict=Read_Dict_Tmp
Make_Feynman_Rules(Frules)

tmp=np.array([])
for i in SubsTex:
    i=i.replace(' ','').split('|')
    tmp=np.append(i,tmp)
SubsTex=tmp.reshape(len(tmp)/2,2)[:]
del tmp
```

```python
Mass=[]
for k in [2]:
    for i in Tuples(Read_Dict['fields'],k):
        s='VertexValue('
        it=iter(i)

        while True:
            try:
                j=it.next()
                s+=j+','
            except:

                s+=')'
                break
        if   eval(s) !=0:
            Mass+= ( i, sp.simplify(eval(s)) )

Mass=np.array(Mass).reshape(len(Mass)/2,2)

Frules=[]
for k in [3,4]:
    for i in Tuples(Read_Dict['fields'],k):
        s='VertexValue('
        it=iter(i)

        while True:
            try:
                j=it.next()
                s+=j+','
            except:

                s+=')'
                break
        if   eval(s) !=0:
            Frules+= ( i, sp.simplify(eval(s)) )

Frules=np.array(Frules).reshape(len(Frules)/2,2)
```

```python
a=open('HTML/subs.dat','r')
tmp=a.read().split('\n')
a.close()
Relations=[]
tmp.remove('')
for i in tmp:
    a=i.replace(' ','').split('|')
    sp.var(a[0])
    Relations+=[(a[0],a[1])]

sp.var('MH MS0');
```

```python

```

```python
in_html=open('HTML/misc','r')
file_html=open('HTML/Frules/Frules.html','w')
file_html.write( in_html.read() )
in_html.close()

file_html.write('<body> \n')

file_html.write('<p>')
#FRules
file_html.write('\\begin{array}{|l|l|}')
file_html.write(  ' {\\rm \\bf Vertex}'+ ' \qquad \qquad &' + '{\\rm \\bf Feynman \; Rule}'+'  \\\\ \hline \\\\' )
for i in Frules:

    file_html.write(  ApplyTexfields(i[0])+ ' \qquad \qquad &' + ApplyTex(i[1])+' \\\\ \hline \\\\' )
file_html.write('\\end{array}')
file_html.write('<p>')
#Mass Matrix entries
file_html.write('  \\begin{array}{ll)}')
file_html.write(  ' {\\rm \\bf Particles}'+ '  &' + '{\\rm \\bf Mass \; Matrix \; element}'+'  \\\\ \hline \\\\' )
for i in Mass:
    file_html.write(  ApplyTexfields(i[0])+ ' \qquad \qquad &' + ApplyTex(i[1])+' \\\\ \hline \\\\' )

file_html.write('\\end{array} ')
#relations
file_html.write(  '${\\rm \\bf Relations: }$' )
file_html.write('  \\begin{align}')

for i in Relations:
    file_html.write(  '&' + ApplyTex(eval(i[0]))+ ' = ' + ApplyTex(eval(i[1]))+' \\\\ ' )

file_html.write('\\end{align} ')

file_html.write('\n </body> \n </html>')
file_html.close()
```

```python
in_html=open('HTML/misc','r')
file_html=open('HTML/L.html','w')
file_html.write( in_html.read() )
in_html.close()

file_html.write('<body> \n')

file_html.write('<p>')

file_html.write('  \\begin{align}')
file_html.write(ApplyTex(eval(Read_Dict['LMassInt'][0])))
file_html.write('  \\end{align}')

file_html.write('\n </body> \n </html>')
file_html.close()
```

```python

```

```python

```

```python

```


```python
# import pandas as pd
# import numpy as np
import matplotlib.pyplot as plt
# import math
# from networkx import minimum_cut, DiGraph
from mip import *
# from itertools import product

from src.read_instance import MDOVRP
# rnd = np.random
# rnd.seed(454)
```

```python
# filepath = "Instancias/ES-n78-m2-Q10138.txt"
filepath = "Instancias/Vrp-Set-A/A-n80-m2-Q60.vrp"
```

```python
N, D, V, Q, q, c, coord_x, coord_y = MDOVRP(filepath)
```

```python
# fazendo c ser simétrica
# c = {(i,j) : (c[i,j] + c[j, i])/2  for (i,j) in c}
```

Usando a formulação $MDOVRP_{2i− flv}$ de Lalla-Ruiz e Mes (2019)

    Lalla-Ruiz, Eduardo, and Martijn Mes. "Mathematical formulations and improvements for the multi-depot open vehicle routing problem." Optimization Letters 15 (2021): 271-286.

```python
#cria o modelo
model = Model('PRVMD', solver_name = GUROBI)

# Variaveis de decisao
x = [[model.add_var(var_type=BINARY) if i!=j else model.add_var(lb=0, ub=0) for i in V] for j in V]
u = [[model.add_var(var_type=CONTINUOUS) if i!=j else model.add_var(lb=0, ub=0) for i in V] for j in V]
y = [model.add_var(var_type=BINARY) for i in N]
```

    Set parameter WLSAccessID
    Set parameter WLSSecret
    Set parameter LicenseID
    Academic license - for non-commercial use only - registered to thiago.giachetto@aluno.ufop.edu.br

\begin{equation}
\label{eq:9}
    x_{ij} \in \{0, 1\}, \quad \forall i, j \in V
\end{equation}

\begin{equation}
\label{eq:10}
    u_{ij} \geq 0, \quad \forall i, j \in V
\end{equation}

```python
# Funcao objetivo
M = 10

# model.objective = minimize(xsum(x[i][j] * c[i,j] for i in V for j in V if i != j))
model.objective = minimize(xsum(x[i][j] * c[i,j] for i in V for j in V if i != j) -
                           M*xsum(x[i][j] * q[j] for i in V for j in N if i != j))
```

\begin{equation}
\label{eq:1}
 Minimizar \quad \sum_{i \in V }\sum_{j \in V} c_{ij}x_{ij}
\end{equation}

```python
# Restricao 2
for j in N:
#     model += xsum(x[i][j] for i in V if i != j and c[i][j] <= 180) == 1
#     model += xsum(x[i][j] for i in V if i != j) == 1
    model += xsum(x[i][j] for i in V if i != j) <= 1
```

\begin{equation}
    \sum_{i \in V, i \neq j}x_{ij} = 1, \quad \forall j \in N
    \tag{2}
\end{equation}

```python
# Restricao 18

for j in N:
    model += (xsum(x[i][j] for i in V if i != j) - xsum(x[j][i] for i in N if i != j)) >= 0
```

\begin{equation}
    \sum_{i \in V, i \neq j}x_{ij} - \sum_{i \in N, i \neq j}x_{ji} \geq 0, \quad \forall j \in N
    \tag {18}
\end{equation}

```python
# Restricao 19
for i in V:
    for j in V:
        if i != j:
            model += x[i][j] + x[j][i] <= 1
```

\begin{equation}
    x_{ij} + x_{ji} \leq 1, \quad \forall i, j \in V, i \neq j
    \tag {19}
\end{equation}

```python
# Restricao 20
# Ninguem volta para o depósito
model += xsum(x[j][k] for k in D for j in V) == 0
```

\begin{equation}
    \sum_{k \in D}\sum_{j \in V}x_{jk} = 0
    \tag {20}
\end{equation}

```python
# Restricao 21
for j in N:
#     model += xsum(u[i][j] for i in V if i != j) - xsum(u[j][i] for i in V if i != j) >= q[j]
    model += xsum(u[i][j] for i in V if i != j) - xsum(u[j][i] for i in V if i != j) >= xsum(x[i][j]*q[j] for i in V if i != j)
```

\begin{equation}
    \sum_{i \in V, i \neq j}u_{ij} - \sum_{i \in V, i \neq j}u_{ji} \geq q_j, \quad \forall j \in N
    \tag {21}
\end{equation}

```python
#Restricao 22
for i in N:
    for j in N:
        model += (Q - q[i])*x[i][j] >= u[i][j]
```

\begin{equation}
    (Q - q_i) \cdot x_{ij} \geq u_{ij}, \quad \forall i, j \in N
    \tag {22}
\end{equation}

```python
# Restricao 23
for k in D:
    for j in N:
        model += Q * x[k][j] >= u[k][j]
```

\begin{equation}
    Q \cdot x_{kj} \geq u_{kj}, \quad \forall k \in D, j \in N
    \tag {23}
\end{equation}

```python
# # Restricao 8
# d = [min([c[j, i] for j in D]) for i in N]
# r = [min([c[j, i] for j in N if i != j]) for i in N]
# M = max(r)

# for i in N:
#     model += d[i] + M * y[i] >= r[i]
```

\begin{equation}
    d_{i} + M y_{i} \geq r_{i}, \quad \forall k \in N
    \tag {8}
\end{equation}

```python
# # Restricao 9

# for i in N:
#     k_l = D[np.argmin([c[j, i] for j in D], axis=0)]
#     model += x[k_l][i] >= y[i]
```

\begin{equation}
    x_{k'i} \geq y_{i}, \quad \forall i \in N, k' = argmin(c_{ki})_{k \in D}
    \tag {9}
\end{equation}

```python
# # Restricao 10

# model += xsum(x[k][i] for i in N for k in D) >= np.ceil(sum(q)/Q)
```

\begin{equation}
    \sum_{i \in N}\sum_{k \in D}x_{ki} \geq \lceil \frac{\sum_{i \in N} q_{i}}{Q} \rceil
    \tag {10}
\end{equation}

```python
# # Restricao de distancia maxima entre duas cidades

# for i in V:
#     for j in V:
#         if i != j:
#             model += c[i,j]*x[i][j] <= 180
```

```python
# limite de carros total
NUM_VEICULOS = 8

model += xsum(x[k][i] for k in D for i in V) <= NUM_VEICULOS
```

```python
# Resolve o modelo
# model.optimize(max_seconds=7200)
model.optimize(max_seconds=30)
print("status: {} objective value : {} best possible: {}".format(model.status, model.objective_value, model.objective_bound))

#                   Objective                Residual
# Iter       Primal          Dual         Primal    Dual     Compl     Time
# usando c simétrica
# 421367 260494 2594.76360   95  102 2608.15000 2560.04596  1.84%   117 7200s
# usando c original
# 351473 211907 2581.12423   69  161 2608.75395 2552.97499  2.14%   138 7200s
```

    Set parameter TimeLimit to value 30
    Set parameter NodeLimit to value 1073741824
    Set parameter SolutionLimit to value 1073741824
    Set parameter IntFeasTol to value 1e-06
    Set parameter Method to value 3
    Gurobi Optimizer version 9.5.0 build v9.5.0rc5 (linux64)
    Thread count: 2 physical cores, 4 logical processors, using up to 4 threads
    Academic license - for non-commercial use only - registered to thiago.giachetto@aluno.ufop.edu.br
    Optimize a model with 13444 rows, 13528 columns and 65452 nonzeros
    Model fingerprint: 0xa56a9c07
    Variable types: 6806 continuous, 6722 integer (6722 binary)
    Coefficient statistics:
      Matrix range     [1e+00, 6e+01]
      Objective range  [4e-01, 2e+02]
      Bounds range     [1e+00, 1e+00]
      RHS range        [1e+00, 8e+00]
    Found heuristic solution: objective 0.0000000
    Presolve removed 3723 rows and 728 columns
    Presolve time: 0.34s
    Presolved: 9721 rows, 12800 columns, 50960 nonzeros
    Variable types: 6320 continuous, 6480 integer (6480 binary)
    Found heuristic solution: objective -863.4314575

    Concurrent LP optimizer: dual simplex and barrier
    Showing barrier log only...

    Root barrier log...

    Ordering time: 0.01s

    Barrier statistics:
     AA' NZ     : 1.458e+04
     Factor NZ  : 2.237e+04 (roughly 3 MB of memory)
     Factor Ops : 3.064e+05 (less than 1 second per iteration)
     Threads    : 1

                      Objective                Residual
    Iter       Primal          Dual         Primal    Dual     Compl     Time
       0   7.19299397e+05 -1.04127440e+07  4.66e+02 5.65e+01  1.34e+04     0s
       1   2.96707155e+05 -8.49617638e+06  1.88e+02 1.82e-12  4.96e+03     0s
       2   5.42971703e+04 -1.86573972e+06  3.40e+01 1.09e-11  8.51e+02     0s
       3   1.97409207e+03 -2.01734571e+05  1.40e-14 1.09e-11  2.22e+01     0s
       4   1.80205486e+03 -1.15421549e+04  2.00e-15 2.73e-12  1.46e+00     0s
       5   7.62596601e+02 -9.83742757e+03  1.89e-15 2.30e-12  1.16e+00     0s
       6  -9.80384404e+02 -4.99844550e+03  1.55e-15 7.67e-13  4.38e-01     0s
       7  -2.25733367e+03 -4.11338851e+03  4.11e-15 4.26e-13  2.02e-01     0s
       8  -2.54653785e+03 -3.78858568e+03  3.00e-15 2.27e-13  1.35e-01     1s
       9  -2.87444423e+03 -3.55755123e+03  2.11e-15 1.85e-13  7.45e-02     1s
      10  -3.09899426e+03 -3.46585316e+03  4.11e-15 1.14e-13  4.00e-02     1s
      11  -3.16614719e+03 -3.42711677e+03  3.55e-15 1.14e-13  2.85e-02     1s
      12  -3.32315456e+03 -3.41138623e+03  3.00e-14 1.14e-13  9.62e-03     1s
      13  -3.39590308e+03 -3.40513769e+03  1.32e-14 2.84e-13  1.01e-03     1s
      14  -3.40382404e+03 -3.40404046e+03  2.11e-14 2.27e-13  2.36e-05     1s
      15  -3.40402001e+03 -3.40402022e+03  1.99e-14 1.71e-13  2.36e-08     1s
      16  -3.40402020e+03 -3.40402020e+03  2.16e-14 1.14e-13  3.30e-14     1s

    Barrier solved model in 16 iterations and 0.53 seconds (0.25 work units)
    Optimal objective -3.40402020e+03

    Use crossover to convert LP symmetric solution to basic solution...

    Solved with dual simplex

    Root relaxation: objective -3.404020e+03, 1774 iterations, 0.17 seconds (0.06 work units)

        Nodes    |    Current Node    |     Objective Bounds      |     Work
     Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time

         0     0 -3404.0202    0   73 -863.43146 -3404.0202   294%     -    1s
    H    0     0                    -2714.730881 -3404.0202  25.4%     -    1s
    H    0     0                    -3047.517387 -3404.0202  11.7%     -    4s
         0     0 -3379.1674    0  122 -3047.5174 -3379.1674  10.9%     -    6s
    H    0     0                    -3284.337561 -3379.1674  2.89%     -    7s
         0     0 -3379.1674    0  122 -3284.3376 -3379.1674  2.89%     -    7s
         0     0 -3376.3292    0  115 -3284.3376 -3376.3292  2.80%     -    8s
         0     0 -3367.4933    0   56 -3284.3376 -3367.4933  2.53%     -   10s
         0     0 -3365.5031    0  104 -3284.3376 -3365.5031  2.47%     -   10s
    H    0     0                    -3288.413970 -3365.5031  2.34%     -   11s
         0     0 -3365.2434    0  108 -3288.4140 -3365.2434  2.34%     -   11s
         0     0 -3365.2434    0   88 -3288.4140 -3365.2434  2.34%     -   12s
         0     0 -3363.1824    0  140 -3288.4140 -3363.1824  2.27%     -   12s
         0     0 -3363.0122    0  117 -3288.4140 -3363.0122  2.27%     -   12s
         0     0 -3363.0079    0  117 -3288.4140 -3363.0079  2.27%     -   12s
         0     0 -3363.0078    0  117 -3288.4140 -3363.0078  2.27%     -   12s
         0     0 -3362.1378    0  128 -3288.4140 -3362.1378  2.24%     -   13s
         0     0 -3361.5747    0  121 -3288.4140 -3361.5747  2.22%     -   13s
         0     0 -3361.5279    0  152 -3288.4140 -3361.5279  2.22%     -   13s
         0     0 -3361.4833    0  153 -3288.4140 -3361.4833  2.22%     -   13s
         0     0 -3361.4833    0  153 -3288.4140 -3361.4833  2.22%     -   13s
         0     0 -3360.5351    0  150 -3288.4140 -3360.5351  2.19%     -   14s
         0     0 -3360.5176    0  150 -3288.4140 -3360.5176  2.19%     -   14s
         0     0 -3360.5176    0  150 -3288.4140 -3360.5176  2.19%     -   14s
         0     0 -3360.3409    0  153 -3288.4140 -3360.3409  2.19%     -   14s
         0     0 -3360.3409    0  153 -3288.4140 -3360.3409  2.19%     -   14s
         0     0 -3360.2356    0  153 -3288.4140 -3360.2356  2.18%     -   15s
    H    0     0                    -3291.752475 -3360.2356  2.08%     -   15s
         0     0 -3360.2352    0  153 -3291.7525 -3360.2352  2.08%     -   15s
         0     0 -3360.2260    0  160 -3291.7525 -3360.2260  2.08%     -   15s
         0     0 -3360.2231    0  163 -3291.7525 -3360.2231  2.08%     -   15s
         0     0 -3360.2204    0  162 -3291.7525 -3360.2204  2.08%     -   15s
         0     0 -3360.2204    0  161 -3291.7525 -3360.2204  2.08%     -   17s
         0     0 -3360.2204    0   43 -3291.7525 -3360.2204  2.08%     -   19s
         0     0 -3360.2204    0  120 -3291.7525 -3360.2204  2.08%     -   21s
    H    0     0                    -3309.971019 -3360.2204  1.52%     -   22s
         0     0 -3360.2204    0  145 -3309.9710 -3360.2204  1.52%     -   22s
         0     0 -3360.1379    0  158 -3309.9710 -3360.1379  1.52%     -   22s
         0     0 -3360.1367    0  158 -3309.9710 -3360.1367  1.52%     -   22s
         0     0 -3359.8213    0  146 -3309.9710 -3359.8213  1.51%     -   23s
         0     0 -3359.8213    0   64 -3309.9710 -3359.8213  1.51%     -   23s
         0     0 -3359.8213    0  125 -3309.9710 -3359.8213  1.51%     -   24s
         0     0 -3359.7816    0  151 -3309.9710 -3359.7816  1.50%     -   25s
         0     0 -3359.7033    0  150 -3309.9710 -3359.7033  1.50%     -   25s
         0     0 -3359.6948    0  150 -3309.9710 -3359.6948  1.50%     -   25s
         0     0 -3359.6948    0  151 -3309.9710 -3359.6948  1.50%     -   25s
         0     0 -3359.4602    0  159 -3309.9710 -3359.4602  1.50%     -   25s
         0     0 -3359.4384    0  163 -3309.9710 -3359.4384  1.49%     -   25s
         0     0 -3359.4335    0  164 -3309.9710 -3359.4335  1.49%     -   25s
         0     0 -3359.4335    0  164 -3309.9710 -3359.4335  1.49%     -   25s
         0     0 -3359.0624    0  142 -3309.9710 -3359.0624  1.48%     -   25s
         0     0 -3359.0288    0  143 -3309.9710 -3359.0288  1.48%     -   26s
         0     0 -3359.0253    0  146 -3309.9710 -3359.0253  1.48%     -   26s
         0     0 -3358.9990    0  153 -3309.9710 -3358.9990  1.48%     -   26s
         0     0 -3358.9985    0  153 -3309.9710 -3358.9985  1.48%     -   26s
         0     0 -3358.0933    0  173 -3309.9710 -3358.0933  1.45%     -   26s
         0     0 -3357.9900    0  163 -3309.9710 -3357.9900  1.45%     -   26s
         0     0 -3357.9661    0  172 -3309.9710 -3357.9661  1.45%     -   26s
         0     0 -3357.9648    0  173 -3309.9710 -3357.9648  1.45%     -   26s
         0     0 -3357.9405    0  164 -3309.9710 -3357.9405  1.45%     -   26s
         0     0 -3357.9270    0  168 -3309.9710 -3357.9270  1.45%     -   26s
         0     0 -3357.9253    0  165 -3309.9710 -3357.9253  1.45%     -   26s
         0     0 -3357.9003    0  155 -3309.9710 -3357.9003  1.45%     -   26s
         0     0 -3357.8806    0  154 -3309.9710 -3357.8806  1.45%     -   26s
         0     0 -3357.8772    0  154 -3309.9710 -3357.8772  1.45%     -   26s
         0     0 -3357.8732    0  162 -3309.9710 -3357.8732  1.45%     -   27s
         0     0 -3357.8732    0  149 -3309.9710 -3357.8732  1.45%     -   27s
         0     2 -3357.8732    0  149 -3309.9710 -3357.8732  1.45%     -   28s
        29    32 -3355.0683   12  137 -3309.9710 -3356.8623  1.42%   289   30s

    Cutting planes:
      Lift-and-project: 18
      Implied bound: 33
      MIR: 40
      Flow cover: 44
      GUB cover: 3
      Zero half: 3
      RLT: 6
      Relax-and-lift: 4

    Explored 31 nodes (67215 simplex iterations) in 30.00 seconds (13.90 work units)
    Thread count was 4 (of 4 available processors)

    Solution count 9: -3309.97 -3291.75 -3288.41 ... 0

    Time limit reached
    Best objective -3.309971019063e+03, best bound -3.356862337182e+03, gap 1.4167%
    status: OptimizationStatus.FEASIBLE objective value : -3309.9710190626565 best possible: -3356.862337181724

```python
# --- PLOTA O GRAFICO ---
plt.figure(figsize=(10, 6))
plt.scatter(coord_x[0:], coord_y[0:])
for i in N:
    plt.annotate(f"{i}", (coord_x[i], coord_y[i]))
for i in D:
    plt.plot(coord_x[i], coord_y[i], c = 'r', marker = 's')

for i in V:
    for j in V:
        if x[i][j].x > 0:
            plt.plot([coord_x[i], coord_x[j]], [coord_y[i], coord_y[j]], c='g', zorder=0)

#escala dos eixos
plt.yticks([i for i in range(1,80,20)]);
plt.xticks([i for i in range(1,80,20)]);
```

```python
# para onde cada veículo vai
for k in D:
    print(f"{k} - ", end='')
    print([i for i, el in enumerate(u[k]) if el.x > 0])
```

    80 - [1, 5, 6, 7]
    81 - [40, 41, 43, 46]

```python
# # u matrix
# for i in N:
#     print(f"{i} - ", end='')
#     print([(j, int(el.x)) for j, el in enumerate(u[i]) if el.x > 0])
```

```python
# caminho dos carros olhando arcos
for k in D:
    print(f"Depot: {k}")
    for i, el in enumerate(x[k]):
        if el.x > 0:
            visitados = [i]
            next_list = [j for j, el in enumerate(x[i]) if el.x > 0]
            while len(next_list) > 0:
                if len(next_list) > 1:
                    print(f"ERROR: {next_list}")
                next_el = next_list[0]
                visitados.append(next_el)
                next_list = [j for j, el in enumerate(x[next_el]) if el.x > 0]
            print(visitados)
```

    Depot: 80
    [1, 2, 10, 9, 17, 25, 18, 26]
    [5, 3, 0, 8, 16, 24]
    [6, 14, 22, 30, 38, 73, 65, 57, 49, 50, 58, 66]
    [7, 4, 12, 20, 28, 15, 23, 31]
    Depot: 81
    [40, 48, 56, 64, 72, 37, 29, 21, 13, 11, 19, 27]
    [41, 42, 44, 52, 60, 68, 76]
    [43, 45, 51, 59, 67, 53, 61, 69]
    [46, 47, 54, 62, 70, 55, 63, 71]

```python
# caminho dos carros olhando demanda
for k in D:
    print(f"Depot: {k}")
    for i, el in enumerate(u[k]):
        if el.x > 0:
            visitados = [(i, int(el.x))]
            next_list = [(j, int(el.x)) for j, el in enumerate(u[i]) if el.x > 0]
            while len(next_list) > 0:
                if len(next_list) > 1:
                    print(f"ERROR: {next_list}")
                next_el, value = next_list[0]
                visitados.append((next_el, value))
                next_list = [(j, int(el.x)) for j, el in enumerate(u[next_el]) if el.x > 0]
            print([(*el, q[el[0]]) for el in visitados])
```

    Depot: 80
    [(1, 60, 12), (2, 48, 12), (10, 36, 8), (9, 28, 8), (17, 20, 4), (25, 7, 2), (18, 5, 4), (26, 1, 2)]
    [(5, 60, 12), (3, 37, 12), (0, 25, 12), (8, 13, 8), (16, 5, 4), (24, 1, 2)]
    [(6, 60, 12), (14, 48, 8), (22, 40, 4), (30, 36, 2), (38, 34, 1), (73, 33, 1), (65, 32, 2), (57, 30, 4), (49, 26, 8), (50, 18, 8), (58, 5, 4), (66, 1, 2)]
    [(7, 60, 12), (4, 48, 12), (12, 36, 8), (20, 28, 4), (28, 24, 2), (15, 22, 8), (23, 5, 4), (31, 2, 2)]
    Depot: 81
    [(40, 60, 12), (48, 48, 8), (56, 40, 4), (64, 36, 2), (72, 34, 1), (37, 33, 1), (29, 32, 2), (21, 30, 4), (13, 26, 8), (11, 18, 8), (19, 5, 4), (27, 1, 2)]
    [(41, 60, 12), (42, 48, 12), (44, 36, 12), (52, 24, 8), (60, 16, 4), (68, 12, 2), (76, 10, 1)]
    [(43, 60, 12), (45, 48, 12), (51, 36, 8), (59, 19, 4), (67, 15, 2), (53, 13, 8), (61, 5, 4), (69, 1, 2)]
    [(46, 60, 12), (47, 48, 12), (54, 36, 8), (62, 28, 4), (70, 24, 2), (55, 22, 8), (63, 5, 4), (71, 1, 2)]

```python
# dist percorrida
sum([(1 if x[i][j].x>0.5 else 0) * c[i,j] for i in V for j in V if i != j])
```

    900.0289809373432

```python
for j in N:
    enter = int(sum([u[i][j].x for i in V if i != j]) - sum([u[j][i].x for i in N if i != j]))
    sum([x[i][j].x*q[j] for i in V if i != j])
    if enter < q[j]:
        print(f"{j}-> enter: {enter}  need: {q[j]}  const: {int(sum([x[i][j].x*q[j] for i in V if i != j]))}")
```

    2-> enter: 11  need: 12  const: 12
    3-> enter: 11  need: 12  const: 12
    4-> enter: 11  need: 12  const: 12
    8-> enter: 7  need: 8  const: 8
    9-> enter: 7  need: 8  const: 8
    10-> enter: 7  need: 8  const: 8
    12-> enter: 7  need: 8  const: 8
    13-> enter: 7  need: 8  const: 8
    14-> enter: 7  need: 8  const: 8
    18-> enter: 3  need: 4  const: 4
    20-> enter: 3  need: 4  const: 4
    21-> enter: 3  need: 4  const: 4
    22-> enter: 3  need: 4  const: 4
    23-> enter: 3  need: 4  const: 4
    24-> enter: 1  need: 2  const: 2
    26-> enter: 1  need: 2  const: 2
    27-> enter: 1  need: 2  const: 2
    28-> enter: 1  need: 2  const: 2
    29-> enter: 1  need: 2  const: 2
    30-> enter: 1  need: 2  const: 2
    32-> enter: 0  need: 1  const: 0
    33-> enter: 0  need: 1  const: 0
    34-> enter: 0  need: 1  const: 0
    35-> enter: 0  need: 1  const: 0
    36-> enter: 0  need: 1  const: 0
    37-> enter: 0  need: 1  const: 1
    38-> enter: 0  need: 1  const: 1
    39-> enter: 0  need: 1  const: 0
    42-> enter: 11  need: 12  const: 12
    44-> enter: 11  need: 12  const: 12
    45-> enter: 11  need: 12  const: 12
    47-> enter: 11  need: 12  const: 12
    48-> enter: 7  need: 8  const: 8
    52-> enter: 7  need: 8  const: 8
    54-> enter: 7  need: 8  const: 8
    56-> enter: 3  need: 4  const: 4
    57-> enter: 3  need: 4  const: 4
    58-> enter: 3  need: 4  const: 4
    60-> enter: 3  need: 4  const: 4
    61-> enter: 3  need: 4  const: 4
    62-> enter: 3  need: 4  const: 4
    63-> enter: 3  need: 4  const: 4
    64-> enter: 1  need: 2  const: 2
    65-> enter: 1  need: 2  const: 2
    66-> enter: 1  need: 2  const: 2
    68-> enter: 1  need: 2  const: 2
    69-> enter: 1  need: 2  const: 2
    70-> enter: 1  need: 2  const: 2
    71-> enter: 1  need: 2  const: 2
    72-> enter: 0  need: 1  const: 1
    73-> enter: 0  need: 1  const: 1
    74-> enter: 0  need: 1  const: 0
    75-> enter: 0  need: 1  const: 0
    77-> enter: 0  need: 1  const: 0
    78-> enter: 0  need: 1  const: 0
    79-> enter: 0  need: 1  const: 0

```python
# nao atendidos
# [j for j in N if ((sum([u[i][j].x for i in V if i != j]) - sum([u[j][i].x for i in N if i != j])) < q[j])]
not_served = [j for j in N if (int(sum([u[i][j].x for i in V if i != j]) - sum([u[j][i].x for i in N if i != j])) < q[j])]
print(len(not_served))
not_served
# [q[j] for j in N]
```

    56

    [2,
     3,
     4,
     8,
     9,
     10,
     12,
     13,
     14,
     18,
     20,
     21,
     22,
     23,
     24,
     26,
     27,
     28,
     29,
     30,
     32,
     33,
     34,
     35,
     36,
     37,
     38,
     39,
     42,
     44,
     45,
     47,
     48,
     52,
     54,
     56,
     57,
     58,
     60,
     61,
     62,
     63,
     64,
     65,
     66,
     68,
     69,
     70,
     71,
     72,
     73,
     74,
     75,
     77,
     78,
     79]

```python
# nro de veiculos
sum(x[k][i].x for k in D for i in N)
```

    8.0

```python
# cidades que tem mais de dois arcos ligadas a ela (depositos podem ter mais)
print([j for j in V if len([i for i in V if x[j][i].x > 0.5 or x[i][j].x > 0.5]) > 2])
```

    [80, 81]

```python
# [el.x for el in w]
```