```python
project_name = "reco-tut-ffr"; branch = "main"; account = "sparsh-ai"
```

```python
!cp /content/drive/MyDrive/mykeys.py /content
import mykeys
!rm /content/mykeys.py
path = "/content/" + project_name;
!mkdir "{path}"
%cd "{path}"
import sys; sys.path.append(path)
!git config --global user.email "nb@recohut.com"
!git config --global user.name  "colab-sparsh"
!git init
!git remote add origin https://"{mykeys.git_token}":x-oauth-basic@github.com/"{account}"/"{project_name}".git
!git pull origin "{branch}"
!git checkout main
```

```python
%cd "/content/reco-tut-ffr"
```

    /content/reco-tut-ffr
    /content/reco-tut-ffr

```python
import os
import csv
import pickle
import random
import pandas as pd
import numpy as np
import matplotlib.pylab as plt
from scipy.sparse.linalg import svds, eigs
import networkx as nx
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split
```

```python
data_path_silver = './data/silver'

def read_df(name):
    return pd.read_parquet(os.path.join(data_path_silver,name+'.parquet.gzip'))

X_train_pos = read_df('X_train_pos')
X_train_neg = read_df('X_train_neg')
X_test_pos = read_df('X_test_pos')
X_test_neg = read_df('X_test_neg')

X_train = X_train_pos.append(X_train_neg, ignore_index=True)
X_test = X_test_pos.append(X_test_neg, ignore_index=True)
X_train.to_csv('/content/train_joined.csv', header=False, index=False)
X_test.to_csv('/content/test_joined.csv', header=False, index=False)

read_df('y_train').to_csv('/content/y_train.csv', header=False, index=False)
read_df('y_test').to_csv('/content/y_test.csv', header=False, index=False)
```

```python
filename = "/content/train_joined.csv"
n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)
s = 100000 #desired sample size
skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))
#https://stackoverflow.com/a/22259008/4084039
```

```python
filename = "/content/test_joined.csv"
n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)
s = 50000 #desired sample size
skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))
```

```python
print("Number of rows in the train data file:", n_train)
print("Number of rows we are going to elimiate in train data are",len(skip_train))
print("Number of rows in the test data file:", n_test)
print("Number of rows we are going to elimiate in test data are",len(skip_test))
```

    Number of rows in the train data file: 15100030
    Number of rows we are going to elimiate in train data are 15000030
    Number of rows in the test data file: 3775008
    Number of rows we are going to elimiate in test data are 3725008

```python
df_final_train = pd.read_csv('/content/train_joined.csv', skiprows=skip_train, names=['source_node', 'destination_node'])
df_final_train['indicator_link'] = pd.read_csv('/content/y_train.csv', skiprows=skip_train, names=['indicator_link'])
print("Our train matrix size ",df_final_train.shape)
df_final_train.head(2)
```

    Our train matrix size  (100001, 3)

      source_node
      destination_node
      indicator_link

      0
      273084
      1505602
      1

      1
      536820
      226930
      1

```python
df_final_test = pd.read_csv('/content/test_joined.csv', skiprows=skip_test, names=['source_node', 'destination_node'])
df_final_test['indicator_link'] = pd.read_csv('/content/y_test.csv', skiprows=skip_test, names=['indicator_link'])
print("Our test matrix size ",df_final_test.shape)
df_final_test.head(2)
```

    Our test matrix size  (50001, 3)

      source_node
      destination_node
      indicator_link

      0
      848424
      784690
      1

      1
      1344580
      342611
      1

```python
X_train_pos = read_df('X_train_pos')
train_graph = nx.from_pandas_edgelist(X_train_pos,
                            source='source_node',
                            target='destination_node',
                            create_using=nx.DiGraph())
```

```python
data_path_gold = './data/gold'
if not os.path.exists(data_path_gold):
    os.makedirs(data_path_gold)
```

### Similarity measures

#### Jaccard distance

\begin{equation}
j = \frac{|X\cap Y|}{|X \cup Y|}
\end{equation}

```python
def jaccard_for_followees(a,b):
    try:
        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:
            return 0
        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\
                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))
    except:
        return 0
    return sim
```

```python
def jaccard_for_followers(a,b):
    try:
        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:
            return 0
        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\
                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))
        return sim
    except:
        return 0
```

#### Cosine distance

\begin{equation}
CosineDistance = \frac{|X\cap Y|}{|X|\cdot|Y|}
\end{equation}

```python
def cosine_for_followees(a,b):
    try:
        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:
            return 0
        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\
                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))
        return sim
    except:
        return 0
```

```python
def cosine_for_followers(a,b):
    try:

        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:
            return 0
        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\
                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))
        return sim
    except:
        return 0
```

### Ranking measures

#### Pagerank

```python
pr = nx.pagerank(train_graph, alpha=0.85)
pickle.dump(pr,open(os.path.join(data_path_gold,'page_rank.p'),'wb'))
```

```python
print('min',pr[min(pr, key=pr.get)])
print('max',pr[max(pr, key=pr.get)])
#for imputing to nodes which are not there in Train data
print('mean_pr',float(sum(pr.values())) / len(pr))
```

    min 1.6556497245737814e-07
    max 2.7098251341935827e-05
    mean_pr 5.615699699389075e-07

### Other graph features

#### Shortest path

Getting Shortest path between two nodes, and if any 2 given nodes have a direct path i.e directly connected then we are removing that edge and calculating path.

```python
def compute_shortest_path_length(a,b):
    p=-1
    try:
        if train_graph.has_edge(a,b):
            train_graph.remove_edge(a,b)
            p= nx.shortest_path_length(train_graph,source=a,target=b)
            train_graph.add_edge(a,b)
        else:
            p= nx.shortest_path_length(train_graph,source=a,target=b)
        return p
    except:
        return -1
```

```python
# unit test 1
compute_shortest_path_length(77697, 826021)
```

    10

```python
# unit test 2
compute_shortest_path_length(669354, 1635354)
```

    -1

#### Same community

```python
wcc = list(nx.weakly_connected_components(train_graph))
```

```python
def belongs_to_same_wcc(a,b):
    index = []
    if train_graph.has_edge(b,a):
        return 1
    if train_graph.has_edge(a,b):
            for i in wcc:
                if a in i:
                    index= i
                    break
            if (b in index):
                train_graph.remove_edge(a,b)
                if compute_shortest_path_length(a,b)==-1:
                    train_graph.add_edge(a,b)
                    return 0
                else:
                    train_graph.add_edge(a,b)
                    return 1
            else:
                return 0
    else:
            for i in wcc:
                if a in i:
                    index= i
                    break
            if(b in index):
                return 1
            else:
                return 0
```

#### Admaic/Adar index

Adamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices: $A(x,y)=\sum_{u \in N(x) \cap N(y)}\frac{1}{log(|N(u)|)}$

```python
def calc_adar_in(a,b):
    sum=0
    try:
        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))
        if len(n)!=0:
            for i in n:
                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))
            return sum
        else:
            return 0
    except:
        return 0
```

### Is person following back?

```python
def follows_back(a,b):
    if train_graph.has_edge(b,a):
        return 1
    else:
        return 0
```

#### Katz centrality

Katz centrality computes the centrality for a node based on the centrality of its neighbors. It is a generalization of the eigenvector centrality. The Katz centrality for node i is: $x_i = \alpha \sum_{j} A_{ij} x_j + \beta$

```python
katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)
pickle.dump(katz,open(os.path.join(data_path_gold,'katz.p'),'wb'))
```

```python
print('min',katz[min(katz, key=katz.get)])
print('max',katz[max(katz, key=katz.get)])
print('mean',float(sum(katz.values())) / len(katz))
```

    min 0.0007313532484065916
    max 0.003394554981699122
    mean 0.0007483800935562018

## Adding a set of features
we will create these each of these features for both train and test data points:
- jaccard_followers
- jaccard_followees
- cosine_followers
- cosine_followees
- num_followers_s
- num_followees_s
- num_followers_d
- num_followees_d
- inter_followers
- inter_followees

```python
#mapping jaccrd followers to train and test data
df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:
                                        jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)
df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:
                                        jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)

#mapping jaccrd followees to train and test data
df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:
                                        jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)
df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:
                                        jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)

#mapping jaccrd followers to train and test data
df_final_train['cosine_followers'] = df_final_train.apply(lambda row:
                                        cosine_for_followers(row['source_node'],row['destination_node']),axis=1)
df_final_test['cosine_followers'] = df_final_test.apply(lambda row:
                                        cosine_for_followers(row['source_node'],row['destination_node']),axis=1)

#mapping jaccrd followees to train and test data
df_final_train['cosine_followees'] = df_final_train.apply(lambda row:
                                        cosine_for_followees(row['source_node'],row['destination_node']),axis=1)
df_final_test['cosine_followees'] = df_final_test.apply(lambda row:
                                        cosine_for_followees(row['source_node'],row['destination_node']),axis=1)
```

```python
def compute_features_stage1(df_final):
    #calculating no of followers followees for source and destination
    #calculating intersection of followers and followees for source and destination
    num_followers_s=[]
    num_followees_s=[]
    num_followers_d=[]
    num_followees_d=[]
    inter_followers=[]
    inter_followees=[]
    for i,row in df_final.iterrows():
        try:
            s1=set(train_graph.predecessors(row['source_node']))
            s2=set(train_graph.successors(row['source_node']))
        except:
            s1 = set()
            s2 = set()
        try:
            d1=set(train_graph.predecessors(row['destination_node']))
            d2=set(train_graph.successors(row['destination_node']))
        except:
            d1 = set()
            d2 = set()
        num_followers_s.append(len(s1))
        num_followees_s.append(len(s2))

        num_followers_d.append(len(d1))
        num_followees_d.append(len(d2))

        inter_followers.append(len(s1.intersection(d1)))
        inter_followees.append(len(s2.intersection(d2)))

    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees
```

```python
if not os.path.isfile(os.path.join(data_path_gold, 'storage_sample_stage1.h5')):
    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \
    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \
    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)

    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \
    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \
    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)

    hdf = pd.HDFStore(os.path.join(data_path_gold, 'storage_sample_stage1.h5'))
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage1.h5'), 'train_df',mode='r')
    df_final_test = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage1.h5'), 'test_df',mode='r')
```

## Adding new set of features
we will create these each of these features for both train and test data points:
- adar index
- is following back
- belongs to same weakly connect components
- shortest path between source and destination

```python
if not os.path.isfile(os.path.join(data_path_gold, 'storage_sample_stage2.h5')):
    #mapping adar index on train
    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)
    #mapping adar index on test
    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping followback or not on train
    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)

    #mapping followback or not on test
    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping same component of wcc or not on train
    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)

    ##mapping same component of wcc or not on train
    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping shortest path on train
    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)
    #mapping shortest path on test
    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)

    hdf = pd.HDFStore(os.path.join(data_path_gold, 'storage_sample_stage2.h5'))
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage2.h5'), 'train_df',mode='r')
    df_final_test = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage2.h5'), 'test_df',mode='r')
```

## Adding new set of features
we will create these each of these features for both train and test data points:
- Weight Features
    - weight of incoming edges
    - weight of outgoing edges
    - weight of incoming edges + weight of outgoing edges
    - weight of incoming edges * weight of outgoing edges
    - 2*weight of incoming edges + weight of outgoing edges
    - weight of incoming edges + 2*weight of outgoing edges
- Page Ranking of source
- Page Ranking of dest
- katz of source
- katz of dest
- hubs of source
- hubs of dest
- authorities_s of source
- authorities_s of dest

### Weight Features
In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other. credit - Graph-based Features for Supervised Link Prediction William Cukierski, Benjamin Hamner, Bo Yang

$W = \frac{1}{\sqrt{1+|X|}}$

it is directed graph so calculated Weighted in and Weighted out differently.

```python
#weight for source and destination of each link
Weight_in = {}
Weight_out = {}
for i in  tqdm(train_graph.nodes()):
    s1=set(train_graph.predecessors(i))
    w_in = 1.0/(np.sqrt(1+len(s1)))
    Weight_in[i]=w_in

    s2=set(train_graph.successors(i))
    w_out = 1.0/(np.sqrt(1+len(s2)))
    Weight_out[i]=w_out

#for imputing with mean
mean_weight_in = np.mean(list(Weight_in.values()))
mean_weight_out = np.mean(list(Weight_out.values()))
```

    HBox(children=(FloatProgress(value=0.0, max=1780722.0), HTML(value='')))

```python
#mapping to pandas train
df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))
df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))

#mapping to pandas test
df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))
df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))

#some features engineerings on the in and out weights
df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out
df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out
df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)
df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)

#some features engineerings on the in and out weights
df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out
df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out
df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)
df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)
```

```python
pr = pickle.load(open(os.path.join(data_path_gold,'page_rank.p'),'rb'))
mean_pr = float(sum(pr.values())) / len(pr)

katz = pickle.load(open(os.path.join(data_path_gold,'katz.p'),'rb'))
mean_katz = float(sum(katz.values())) / len(katz)
```

```python
if not os.path.isfile(os.path.join(data_path_gold, 'storage_sample_stage3.h5')):

    #page rank for source and destination in Train and Test
    #if anything not there in train graph then adding mean page rank
    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))
    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))

    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))
    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))
    #================================================================================

    #Katz centrality score for source and destination in Train and test
    #if anything not there in train graph then adding mean katz score
    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))
    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))

    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))
    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))
    #================================================================================

    hdf = pd.HDFStore(os.path.join(data_path_gold, 'storage_sample_stage3.h5'))
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage3.h5'), 'train_df',mode='r')
    df_final_test = pd.read_hdf(os.path.join(data_path_gold, 'storage_sample_stage3.h5'), 'test_df',mode='r')
```

### Adding new feature Preferential Attachement
One well-known concept in social networks is that users with many friends tend to create more connections in the future. This is due to the fact that in some social networks, like in finance, the rich get richer. We estimate how ”rich” our two vertices are by calculating the multiplication between the number of friends (|Γ(x)|) or followers each vertex has.

```python
# Preferential Attachement for followers
#for train dataset
nfs=np.array(df_final_train['num_followers_s'])
nfd=np.array(df_final_train['num_followers_d'])
preferential_followers=[]
for i in range(len(nfs)):
    preferential_followers.append(nfd[i]*nfs[i])
df_final_train['prefer_Attach_followers']= preferential_followers

#for test dataset
nfs=np.array(df_final_test['num_followers_s'])
nfd=np.array(df_final_test['num_followers_d'])
preferential_followers=[]
for i in range(len(nfs)):
    preferential_followers.append(nfd[i]*nfs[i])
df_final_test['prefer_Attach_followers']= preferential_followers

# Preferential Attachement for followers
#for train dataset
nfs=np.array(df_final_train['num_followees_s'])
nfd=np.array(df_final_train['num_followees_d'])
preferential_followees=[]
for i in range(len(nfs)):
    preferential_followees.append(nfd[i]*nfs[i])
df_final_train['prefer_Attach_followees']= preferential_followees

#for test dataset
nfs=np.array(df_final_test['num_followees_s'])
nfd=np.array(df_final_test['num_followees_d'])
preferential_followees=[]
for i in range(len(nfs)):
    preferential_followees.append(nfd[i]*nfs[i])
df_final_test['prefer_Attach_followees']= preferential_followees
```

### SVD features for both source and destination

```python
def svd(x, S):
    try:
        z = sadj_dict[x]
        return S[z]
    except:
        return [0,0,0,0,0,0]
```

```python
#for svd features to get feature vector creating a dict node val and index in svd vector
sadj_col = sorted(train_graph.nodes())
sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}

Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()

U, s, V = svds(Adj, k = 6)
print('Adjacency matrix Shape',Adj.shape)
print('U Shape',U.shape)
print('V Shape',V.shape)
print('s Shape',s.shape)
```

    Adjacency matrix Shape (1780722, 1780722)
    U Shape (1780722, 6)
    V Shape (6, 1780722)
    s Shape (6,)

```python
df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \
df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)

df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \
df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)
#===================================================================================================

df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \
df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)

df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \
df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)
#===================================================================================================

df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \
df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)

df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \
df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)

#===================================================================================================

df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \
df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)

df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \
df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)
```

```python
df_final_train.columns
```

    Index(['source_node', 'destination_node', 'indicator_link',
           'jaccard_followers', 'jaccard_followees', 'cosine_followers',
           'cosine_followees', 'num_followers_s', 'num_followers_d',
           'num_followees_s', 'num_followees_d', 'inter_followers',
           'inter_followees', 'adar_index', 'follows_back', 'same_comp',
           'shortest_path', 'weight_in', 'weight_out', 'weight_f1', 'weight_f2',
           'weight_f3', 'weight_f4', 'page_rank_s', 'page_rank_d', 'katz_s',
           'katz_d', 'prefer_Attach_followers', 'prefer_Attach_followees',
           'svd_u_s_1', 'svd_u_s_2', 'svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5',
           'svd_u_s_6', 'svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4',
           'svd_u_d_5', 'svd_u_d_6', 'svd_v_s_1', 'svd_v_s_2', 'svd_v_s_3',
           'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6', 'svd_v_d_1', 'svd_v_d_2',
           'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5', 'svd_v_d_6'],
          dtype='object')

### Adding feature svd_dot
svd_dot is Dot product between sourse node svd and destination node svd features

```python
#for train datasets
s1,s2,s3,s4,s5,s6=df_final_train['svd_u_s_1'],df_final_train['svd_u_s_2'],df_final_train['svd_u_s_3'],df_final_train['svd_u_s_4'],df_final_train['svd_u_s_5'],df_final_train['svd_u_s_6']
s7,s8,s9,s10,s11,s12=df_final_train['svd_v_s_1'],df_final_train['svd_v_s_2'],df_final_train['svd_v_s_3'],df_final_train['svd_v_s_4'],df_final_train['svd_v_s_5'],df_final_train['svd_v_s_6']

d1,d2,d3,d4,d5,d6=df_final_train['svd_u_d_1'],df_final_train['svd_u_d_2'],df_final_train['svd_u_d_3'],df_final_train['svd_u_d_4'],df_final_train['svd_u_d_5'],df_final_train['svd_u_d_6']
d7,d8,d9,d10,d11,d12=df_final_train['svd_v_d_1'],df_final_train['svd_v_d_2'],df_final_train['svd_v_d_3'],df_final_train['svd_v_d_4'],df_final_train['svd_v_d_5'],df_final_train['svd_v_d_6']
```

```python
svd_dot=[]
for i in range(len(np.array(s1))):
    a=[]
    b=[]
    a.append(np.array(s1[i]))
    a.append(np.array(s2[i]))
    a.append(np.array(s3[i]))
    a.append(np.array(s4[i]))
    a.append(np.array(s5[i]))
    a.append(np.array(s6[i]))
    a.append(np.array(s7[i]))
    a.append(np.array(s8[i]))
    a.append(np.array(s9[i]))
    a.append(np.array(s10[i]))
    a.append(np.array(s11[i]))
    a.append(np.array(s12[i]))
    b.append(np.array(d1[i]))
    b.append(np.array(d2[i]))
    b.append(np.array(d3[i]))
    b.append(np.array(d4[i]))
    b.append(np.array(d5[i]))
    b.append(np.array(d6[i]))
    b.append(np.array(d7[i]))
    b.append(np.array(d8[i]))
    b.append(np.array(d9[i]))
    b.append(np.array(d10[i]))
    b.append(np.array(d11[i]))
    b.append(np.array(d12[i]))
    svd_dot.append(np.dot(a,b))

df_final_train['svd_dot']=svd_dot
```

```python
df_final_train.head()
```

      source_node
      destination_node
      indicator_link
      jaccard_followers
      jaccard_followees
      cosine_followers
      cosine_followees
      num_followers_s
      num_followers_d
      num_followees_s
      num_followees_d
      inter_followers
      inter_followees
      adar_index
      follows_back
      same_comp
      shortest_path
      weight_in
      weight_out
      weight_f1
      weight_f2
      weight_f3
      weight_f4
      page_rank_s
      page_rank_d
      katz_s
      katz_d
      prefer_Attach_followers
      prefer_Attach_followees
      svd_u_s_1
      svd_u_s_2
      svd_u_s_3
      svd_u_s_4
      svd_u_s_5
      svd_u_s_6
      svd_u_d_1
      svd_u_d_2
      svd_u_d_3
      svd_u_d_4
      svd_u_d_5
      svd_u_d_6
      svd_v_s_1
      svd_v_s_2
      svd_v_s_3
      svd_v_s_4
      svd_v_s_5
      svd_v_s_6
      svd_v_d_1
      svd_v_d_2
      svd_v_d_3
      svd_v_d_4
      svd_v_d_5
      svd_v_d_6
      svd_dot

      0
      273084
      1505602
      1
      0
      0.000000
      0
      0
      11
      6
      15
      8
      0
      0
      0.000000
      0
      1
      4
      0.377964
      0.250000
      0.627964
      0.094491
      1.005929
      0.877964
      2.045290e-06
      3.459963e-07
      0.000773
      0.000756
      66
      120
      -1.666376e-13
      4.613670e-13
      1.043022e-05
      6.677503e-13
      2.451075e-13
      3.584762e-12
      -2.038018e-11
      5.397735e-13
      1.068940e-06
      1.192410e-12
      2.279486e-12
      3.581020e-12
      -7.216110e-13
      3.926207e-13
      1.983688e-06
      1.545078e-13
      8.108382e-13
      1.719699e-14
      -1.355354e-12
      4.675181e-13
      1.128577e-06
      6.616539e-14
      9.771045e-13
      4.159814e-14
      1.338803e-11

      1
      536820
      226930
      1
      0
      0.000000
      0
      0
      3
      8
      8
      8
      0
      0
      0.000000
      0
      1
      6
      0.333333
      0.333333
      0.666667
      0.111111
      1.000000
      1.000000
      2.555560e-07
      7.900776e-07
      0.000743
      0.000761
      24
      64
      -1.000683e-13
      2.416715e-13
      1.567253e-12
      6.207936e-12
      3.122976e-13
      1.859408e-12
      -5.872395e-15
      3.075290e-12
      1.076685e-13
      6.914915e-14
      2.051733e-12
      9.657654e-16
      -3.796453e-13
      1.341099e-13
      6.455030e-12
      2.988530e-12
      1.671777e-13
      9.261643e-14
      -2.839895e-13
      4.529009e-12
      4.620106e-13
      1.273286e-13
      8.783099e-14
      1.604342e-14
      6.078550e-24

      2
      53024
      470337
      1
      0
      0.100000
      0
      0
      5
      6
      5
      6
      1
      1
      1.000000
      1
      1
      2
      0.377964
      0.408248
      0.786213
      0.154303
      1.164177
      1.194461
      1.711219e-06
      9.779807e-07
      0.000750
      0.000754
      30
      30
      -5.739548e-11
      1.508032e-10
      8.174490e-12
      1.178467e-10
      2.465994e-11
      1.264218e-12
      -3.662229e-13
      2.117166e-11
      1.974638e-11
      1.799748e-11
      1.650662e-11
      7.129315e-13
      -1.200755e-12
      1.549722e-12
      1.029112e-11
      1.103299e-11
      2.430948e-11
      4.198859e-11
      -2.078009e-12
      3.596208e-12
      3.750138e-11
      1.377155e-11
      2.457679e-11
      4.199957e-11
      8.810981e-21

      3
      1417605
      1230379
      1
      0
      0.588235
      0
      0
      22
      29
      36
      45
      17
      30
      18.915117
      1
      1
      2
      0.182574
      0.164399
      0.346973
      0.030015
      0.529547
      0.511372
      4.447768e-07
      5.524609e-07
      0.000830
      0.000861
      638
      1620
      -9.182749e-18
      4.864470e-17
      1.293195e-15
      1.701462e-17
      3.402242e-17
      -2.861893e-18
      -1.092711e-17
      6.139269e-17
      1.663824e-15
      2.099197e-17
      4.253749e-17
      -2.691626e-18
      -2.423041e-18
      2.264568e-17
      6.880258e-16
      9.507225e-18
      2.106043e-17
      -3.131570e-19
      -3.476597e-18
      3.792054e-17
      1.187459e-15
      1.661522e-17
      3.820082e-17
      -3.136377e-19
      2.975379e-30

      4
      896938
      1135133
      1
      0
      0.000000
      0
      0
      1
      10
      2
      10
      0
      0
      0.000000
      1
      1
      10
      0.301511
      0.577350
      0.878862
      0.174078
      1.180373
      1.456212
      2.132984e-07
      1.394701e-06
      0.000735
      0.000769
      10
      20
      -2.077938e-15
      6.781134e-15
      2.228023e-15
      6.800924e-17
      1.920533e-16
      1.210472e-17
      -2.163061e-15
      1.936589e-14
      9.044099e-14
      6.999872e-14
      1.347763e-15
      1.197486e-16
      -2.191116e-17
      1.929638e-16
      8.779132e-16
      6.690169e-16
      1.076853e-17
      8.507623e-19
      -7.727098e-16
      4.763126e-15
      1.038215e-13
      6.545172e-15
      1.729064e-14
      1.693182e-15
      4.389916e-28

```python
#for test dataset
s1,s2,s3,s4,s5,s6=df_final_test['svd_u_s_1'],df_final_test['svd_u_s_2'],df_final_test['svd_u_s_3'],df_final_test['svd_u_s_4'],df_final_test['svd_u_s_5'],df_final_test['svd_u_s_6']
s7,s8,s9,s10,s11,s12=df_final_test['svd_v_s_1'],df_final_test['svd_v_s_2'],df_final_test['svd_v_s_3'],df_final_test['svd_v_s_4'],df_final_test['svd_v_s_5'],df_final_test['svd_v_s_6']

d1,d2,d3,d4,d5,d6=df_final_test['svd_u_d_1'],df_final_test['svd_u_d_2'],df_final_test['svd_u_d_3'],df_final_test['svd_u_d_4'],df_final_test['svd_u_d_5'],df_final_test['svd_u_d_6']
d7,d8,d9,d10,d11,d12=df_final_test['svd_v_d_1'],df_final_test['svd_v_d_2'],df_final_test['svd_v_d_3'],df_final_test['svd_v_d_4'],df_final_test['svd_v_d_5'],df_final_test['svd_v_d_6']
```

```python
svd_dot=[]
for i in range(len(np.array(s1))):
    a=[]
    b=[]
    a.append(np.array(s1[i]))
    a.append(np.array(s2[i]))
    a.append(np.array(s3[i]))
    a.append(np.array(s4[i]))
    a.append(np.array(s5[i]))
    a.append(np.array(s6[i]))
    a.append(np.array(s7[i]))
    a.append(np.array(s8[i]))
    a.append(np.array(s9[i]))
    a.append(np.array(s10[i]))
    a.append(np.array(s11[i]))
    a.append(np.array(s12[i]))
    b.append(np.array(d1[i]))
    b.append(np.array(d2[i]))
    b.append(np.array(d3[i]))
    b.append(np.array(d4[i]))
    b.append(np.array(d5[i]))
    b.append(np.array(d6[i]))
    b.append(np.array(d7[i]))
    b.append(np.array(d8[i]))
    b.append(np.array(d9[i]))
    b.append(np.array(d10[i]))
    b.append(np.array(d11[i]))
    b.append(np.array(d12[i]))
    svd_dot.append(np.dot(a,b))

df_final_test['svd_dot']=svd_dot
```

```python
df_final_test.head()
```

      source_node
      destination_node
      indicator_link
      jaccard_followers
      jaccard_followees
      cosine_followers
      cosine_followees
      num_followers_s
      num_followers_d
      num_followees_s
      num_followees_d
      inter_followers
      inter_followees
      adar_index
      follows_back
      same_comp
      shortest_path
      weight_in
      weight_out
      weight_f1
      weight_f2
      weight_f3
      weight_f4
      page_rank_s
      page_rank_d
      katz_s
      katz_d
      prefer_Attach_followers
      prefer_Attach_followees
      svd_u_s_1
      svd_u_s_2
      svd_u_s_3
      svd_u_s_4
      svd_u_s_5
      svd_u_s_6
      svd_u_d_1
      svd_u_d_2
      svd_u_d_3
      svd_u_d_4
      svd_u_d_5
      svd_u_d_6
      svd_v_s_1
      svd_v_s_2
      svd_v_s_3
      svd_v_s_4
      svd_v_s_5
      svd_v_s_6
      svd_v_d_1
      svd_v_d_2
      svd_v_d_3
      svd_v_d_4
      svd_v_d_5
      svd_v_d_6
      svd_dot

      0
      848424
      784690
      1
      0
      0.000000
      0
      0
      6
      14
      6
      9
      1
      0
      0.000000
      1
      1
      2
      0.258199
      0.377964
      0.636163
      0.097590
      0.894362
      1.014128
      6.557971e-07
      0.000002
      0.000754
      0.000786
      84
      54
      -9.987980e-12
      2.283706e-13
      1.439885e-10
      6.136172e-13
      4.188169e-13
      5.983462e-15
      -1.026186e-11
      5.316560e-13
      5.843125e-10
      1.166046e-13
      2.253352e-11
      3.220432e-15
      -2.148850e-13
      1.883264e-13
      5.904787e-11
      2.701538e-12
      4.341613e-13
      5.535503e-14
      -9.994077e-10
      5.791927e-10
      3.512349e-07
      2.486660e-09
      2.771140e-09
      1.727694e-12
      2.083217e-17

      1
      1344580
      342611
      1
      0
      0.012500
      0
      0
      50
      26
      56
      25
      1
      1
      1.285097
      1
      1
      2
      0.192450
      0.132453
      0.324903
      0.025491
      0.517353
      0.457357
      3.740596e-06
      0.000003
      0.000934
      0.000835
      1300
      1400
      -1.041012e-10
      2.450910e-10
      1.636445e-06
      1.744189e-10
      2.403377e-10
      1.761182e-12
      -2.127380e-11
      1.010533e-09
      1.199321e-08
      1.693617e-09
      3.864187e-11
      1.945994e-12
      -3.144714e-11
      3.702433e-10
      1.013765e-07
      6.055064e-11
      2.346810e-09
      1.941724e-11
      -4.075337e-12
      8.025003e-12
      4.438231e-10
      1.995940e-11
      2.923692e-10
      1.085116e-13
      1.967248e-14

      2
      1294891
      1447581
      1
      0
      0.000000
      0
      0
      6
      11
      5
      13
      0
      0
      0.000000
      1
      1
      3
      0.288675
      0.408248
      0.696923
      0.117851
      0.985599
      1.105172
      4.271048e-07
      0.000001
      0.000755
      0.000774
      66
      65
      -2.936992e-11
      9.718611e-12
      5.022905e-11
      6.553552e-12
      1.321321e-11
      3.854840e-14
      -5.969443e-14
      7.834291e-11
      3.476005e-12
      1.887946e-13
      1.554217e-11
      1.678442e-15
      -1.587702e-12
      3.182815e-11
      1.402614e-11
      7.735058e-12
      4.660135e-12
      8.114806e-16
      -2.966961e-13
      3.001932e-11
      5.479215e-12
      1.403040e-12
      2.755603e-10
      1.063588e-15
      3.472117e-21

      3
      1802128
      1634556
      1
      0
      0.037736
      0
      0
      46
      14
      38
      17
      1
      2
      2.308604
      1
      1
      2
      0.258199
      0.160128
      0.418327
      0.041345
      0.676526
      0.578455
      2.738619e-06
      0.000001
      0.000921
      0.000786
      644
      646
      -2.623130e-12
      7.000651e-08
      4.281443e-06
      4.331846e-09
      3.224032e-11
      8.991486e-12
      -1.146981e-11
      9.191004e-11
      9.061251e-06
      2.877213e-10
      1.644661e-11
      5.363448e-12
      -1.485085e-11
      7.087963e-09
      4.194574e-05
      2.988299e-08
      1.145909e-11
      5.002974e-10
      -2.225020e-12
      3.048860e-12
      5.197992e-07
      5.928659e-12
      1.166038e-12
      1.579162e-13
      6.059860e-11

      4
      1505897
      1828850
      1
      0
      0.020408
      0
      0
      41
      19
      23
      27
      1
      1
      0.574593
      1
      1
      2
      0.223607
      0.204124
      0.427731
      0.045644
      0.651338
      0.631855
      3.403581e-06
      0.000003
      0.000898
      0.000809
      779
      621
      -2.507284e-11
      1.413282e-11
      2.109213e-05
      1.787929e-12
      5.404061e-12
      1.931283e-12
      -4.699055e-11
      3.834546e-11
      2.136433e-05
      1.609714e-12
      5.680389e-11
      1.977754e-12
      -2.147159e-11
      6.603848e-12
      2.194823e-05
      5.864301e-12
      1.323759e-09
      7.844954e-13
      -1.830513e-11
      5.911760e-12
      6.590570e-05
      3.453278e-12
      1.231581e-11
      7.614798e-13
      1.897133e-09

```python
hdf = pd.HDFStore(os.path.join(data_path_gold,'storage_sample_stage4.h5'))
hdf.put('train_df',df_final_train, format='table', data_columns=True)
hdf.put('test_df',df_final_test, format='table', data_columns=True)
hdf.close()
```

```python
!git status
```

    On branch main
    Your branch is up to date with 'origin/main'.

    Untracked files:
      (use "git add <file>..." to include in what will be committed)

    	data/gold/

    nothing added to commit but untracked files present (use "git add" to track)

```python
!git add .
!git commit -m 'added gold data layer'
```

    [main da0d36f] added gold data layer
     6 files changed, 0 insertions(+), 0 deletions(-)
     create mode 100644 data/gold/katz.p
     create mode 100644 data/gold/page_rank.p
     create mode 100644 data/gold/storage_sample_stage1.h5
     create mode 100644 data/gold/storage_sample_stage2.h5
     create mode 100644 data/gold/storage_sample_stage3.h5
     create mode 100644 data/gold/storage_sample_stage4.h5

```python
!git push origin main
```

    Counting objects: 10, done.
    Delta compression using up to 2 threads.
    Compressing objects: 100% (10/10), done.
    Writing objects: 100% (10/10), 102.44 MiB | 5.19 MiB/s, done.
    Total 10 (delta 4), reused 0 (delta 0)
    remote: Resolving deltas: 100% (4/4), completed with 1 local object.
    remote: warning: GH001: Large files detected. You may want to try Git Large File Storage - https://git-lfs.github.com.
    remote: warning: See http://git.io/iEPt8g for more information.
    remote: warning: File data/gold/storage_sample_stage4.h5 is 95.79 MB; this is larger than GitHub's recommended maximum file size of 50.00 MB
    To https://github.com/sparsh-ai/reco-tut-ffr.git
       9ebae1e..da0d36f  main -> main


# Multi-Class Classification - Language Classification

This notebook implements the method presented in Goldberg's [2017] book "Neural Network Methods for Natural Language Processing". It shows the steps you need to go through in order to successfully train a classifier, and it should also, so I hope, illustrate the notational differences between Goldberg and standard machine learning literature.

$NOTE$: There is no cross-validation etc. to find optimal parameters. This is simply to show how multi-class classification works. This will be part of a tutorial session and all other concepts will be explained there.

Author: Phillip Ströbel

## Getting and cleaning the data

The data consists of downloaded Wikipedia articles (see `urls.txt`) in German, English, French, Spanish, Italian and Finnish (instead of "O" in Goldberg). The data is in HTML, so we need to some preprocessing to get the text out of it. We also restrict ourselfes to the characters from a to z in the alphabet (as described in Goldberg). In this fashion, we get rid of all the Umlauts (ä, ö, ü) and all other characters with diacritics (as, e.g., the é or ç in French). Note however, that if these characters ocurring in bigrams would probably be good features. In some way, we still keep the information "special character" by not fully deleting the character, but by replacing it by the dollar sign "\\$". Furthermore, we replace all punctuation marks and digits by dollar signs as well. As such, all special characters, digits, and punctuation marks are mapped to "\\$". The space will be replaced by an underscore "\_". We then represent each langauge by 28 characters, as is suggested by Goldberg.

### Cleaning HTML
We first strip the HTML to get only the text of the Wikipedia page.

#### Get the html files

```python
import re
import numpy as np
from bs4 import BeautifulSoup
from urllib.request import urlopen
from collections import defaultdict

seed = np.random.seed(seed=200)  # set a seed for random, so results are reproducible

article_dict = defaultdict(lambda: defaultdict(str))

regex = r'[\n ]{2,}'
pattern = re.compile(regex)

urls = open('urls.txt', 'r').readlines()

for url_index, url in enumerate(urls):
    language = url[8:10]
    doc_id = 'doc_%d' % url_index
    html = urlopen(url.strip()).read()
    soup = BeautifulSoup(html, 'html.parser')
    raw = soup.body.get_text()  # only get text from the text body (this excludes headers and should exclude navigation bars)
    raw = re.sub(pattern, ' ', raw)  # replace multiple breaks and spaces by only one space
    raw = re.sub(r'\n', ' ', raw)  # replace every line break with a space
    article_dict[language][doc_id] = raw.lower()  # assign each text to its language and lower all uppercase characters
```

### Preprocessing --> prepare the text
replace special characters and digits

```python
preprocessed_dict = defaultdict(lambda: defaultdict(str))

abc = r'[a-z]'
abc_pattern = re.compile(abc)

for lang, doc in article_dict.items():
    for doc, text in doc.items():
        for char in text:
            if re.match(abc_pattern, char):
                preprocessed_dict[lang][doc] += char
            elif re.match(' ', char):
                preprocessed_dict[lang][doc] += '_'
            else:
                preprocessed_dict[lang][doc] += '$'
```

### Count bigrams --> Feature extraction

The distribution of bigrams will be our only feature. We could extend this by taking into account other n-grams.

```python
charset = 'abcdefghijklmnopqrstuvwxyz$_'  # define the character set we want to use
```

```python
from itertools import combinations_with_replacement, permutations

def bigrams(text):
    """
    Function to extract bigrams from text and calculate their distribution
    :param text: text string
    :return: dictionary containing bigrams as keys, and the normalised count as values
    """
    combs = combinations_with_replacement(charset, 2)
    perms = permutations(charset, 2)
    bigram_dict = dict()

    for comb in set(list(combs) + list(perms)):
        bigram_dict[''.join(comb)] = 0

    doc_length = len(text)

    for i in range(0, len(text)-1):
        bigram = text[i] + text[i+1]
        bigram_dict[bigram] += 1

    for bigram, count in bigram_dict.items():
        bigram_dict[bigram] = count/doc_length

    return bigram_dict
```

### Put data into pandas dataframe
The pandas dataframe allows us to conveniently represent all the data we need in one table. So let's do this. But first we need to extract the features.

```python
bigram_dict_full = defaultdict(lambda: defaultdict(dict))

for lang, doc in preprocessed_dict.items():
    for doc, text in sorted(doc.items()):
        bigram_dict = bigrams(text)
        bigram_dict_full[lang][doc] = bigram_dict
```

```python
import pandas as pd

col_names = ['y'] + sorted(bigram_dict_full['en']['doc_0'].keys())
my_df = dict()

for col in col_names:
    my_df[col] = list()

df = pd.DataFrame(my_df)

for lang, doc in bigram_dict_full.items():
    for key, value in doc.items():
        df_obj = value
        df_obj['y'] = lang
        df = df.append(df_obj, ignore_index=True)

df.head()

```

      $$
      $_
      $a
      $b
      $c
      $d
      $e
      $f
      $g
      $h
      ...
      zq
      zr
      zs
      zt
      zu
      zv
      zw
      zx
      zy
      zz

      0
      0.070389
      0.034548
      0.002063
      0.000317
      0.000249
      0.000295
      0.001020
      0.000136
      0.000113
      0.001065
      ...
      0.0
      0.0
      0.000000
      0.0
      0.000023
      0.0
      0.000000
      0.0
      0.0
      0.0

      1
      0.047840
      0.025302
      0.000809
      0.000243
      0.000135
      0.000067
      0.000890
      0.000067
      0.000121
      0.000850
      ...
      0.0
      0.0
      0.000000
      0.0
      0.000013
      0.0
      0.000013
      0.0
      0.0
      0.0

      2
      0.064450
      0.032426
      0.000541
      0.001359
      0.000357
      0.000392
      0.000841
      0.000092
      0.000081
      0.001106
      ...
      0.0
      0.0
      0.000000
      0.0
      0.000127
      0.0
      0.000012
      0.0
      0.0
      0.0

      3
      0.070677
      0.029689
      0.000320
      0.000137
      0.000412
      0.000457
      0.001189
      0.000549
      0.000046
      0.000549
      ...
      0.0
      0.0
      0.000000
      0.0
      0.000000
      0.0
      0.000000
      0.0
      0.0
      0.0

      4
      0.046610
      0.034304
      0.000789
      0.000168
      0.000168
      0.000505
      0.000751
      0.000233
      0.000065
      0.001009
      ...
      0.0
      0.0
      0.000013
      0.0
      0.000000
      0.0
      0.000000
      0.0
      0.0
      0.0

5 rows × 785 columns

```python
df.shape
```

    (60, 785)

Now we split the data into the label vector \begin{equation}\mathbf{y}\end{equation} and a training data matrix \begin{equation}\mathbf{X}\end{equation}. But first, we shuffle the df and split it into a training and a test set.

Split the data into a train set and a test set

```python
train = df.sample(frac=0.9, random_state=seed)
test = df.drop(train.index)
```

Define the different sets

```python
# for training
y_train = train.y
X_train = train.drop('y', axis=1)

# for testing
y_test = test.y
X_test = test.drop('y', axis=1)
```

Check the shapes

```python
print('Training samples shape: ', X_train.shape)
print('Training labels shape: ', y_train.shape)
print('Test samples shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
```

    Training samples shape:  (54, 784)
    Training labels shape:  (54,)
    Test samples shape:  (6, 784)
    Test labels shape:  (6,)

Moreover, it is necessary for many machine learning tasks to standardise the data. Our aim is for each feature to be represented by a column vector in which values have zero mean and unit variance.

```python
def normalise_matrix(matrix, mean_and_std=None):
    """
    normalises the data matrix (normalise each datapoint to zero mean and unit variance.)
    :param matrix: input matrix
    :param mean_and_std: provide mean and std as tuples in list for normalisation of test data
    :return: normalised matrix and list consisting of tuples containing mean and std
    """
    normalised = np.ones(matrix.shape)

    if mean_and_std == None:

        mean_std_list = list()

        for col_index, col in enumerate(matrix):
            mean = matrix[col].mean()
            std = matrix[col].std()
            mean_std_list.append((mean, std))
            for row_index, item in enumerate(matrix[col]):
                try:
                    normalised[row_index, col_index] = (item - mean)/std
                except ZeroDivisionError:
                    normalised[row_index, col_index] = 0.0
        return normalised, mean_std_list
    else:
        for col_index, col in enumerate(matrix):
            for row_index, item in enumerate(matrix[col]):
                try:
                    mean = mean_and_std[col_index][0]
                    std = mean_and_std[col_index][1]
                    normalised[row_index, col_index] = (item - mean)/std
                except ZeroDivisionError:
                    normalised[row_index, col_index] = 0.0
        return normalised
```

normalise the train and test set (use values from training set for standardisation of test set). in sklearn, there are simple methods which help you with this!

```python
train_norm, train_mean_std = normalise_matrix(X_train)
test_norm = normalise_matrix(X_test, train_mean_std)
```

We should binarise our labels, although libraries like sklearn can also deal with non-numeric data

```python
from sklearn import preprocessing

lb = preprocessing.LabelBinarizer()
lb.fit(['en', 'fr', 'de', 'it', 'es', 'fi'])
```

    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)

```python
lb.classes_
```

    array(['de', 'en', 'es', 'fi', 'fr', 'it'], dtype='<U2')

We do this for both our training and test labels:

```python
y_train = lb.transform(y_train)
y_test = lb.transform(y_test)
```

Labels are now one-hot encoded:

```python
y_train[0]
```

    array([0, 0, 0, 0, 0, 1])

We almost have everything now. However, we need to take care of the bias and the weight matrix. The hypothesis ŷ is given by:
\begin{equation}
\mathbf{\hat{y}}=\mathbf{x}\cdot\mathbf{W}+\mathbf{b}
\end{equation}
We can achieve this by appending 1 to each feature vector x, and the whole weight vector b to the weight matrix W. This is called the bias trick. Note that the dimensions of X_train change, and that the weight matrix W will have match the dimensions (same number of rows as X has columns).

```python
bias_vector = np.ones([train_norm.shape[0], 1])
X_train = np.append(train_norm, bias_vector, axis=1)
X_train.shape
```

    (54, 785)

```python
# initialise weight matrix with small weights

np.random.seed(seed=200)

W = np.random.randn(X_train.shape[1], len(lb.classes_)) * 0.0001
#W = np.zeros([X.shape[1], len(lb.classes_)])
```

We see that the dimensions are right. The dot product of a specific row from X_train and the weight matrix W constitutes a forward pass and calculates the score for each class.

```python
W.shape
```

    (785, 6)

```python
X_train.shape
```

    (54, 785)

```python
X_train[5:6].dot(W)
```

    array([[-0.00165842, -0.00558468, -0.00562853, -0.00551135, -0.00191775,
             0.0010696 ]])

We see that the values for the highest score of the dot product is not the score of the true label. Our aim is to change this by implementing a support vector classifier.

```python
X_train[5:6].dot(W).max(axis=1)
```

    array([0.0010696])

```python
X_train[5:6].dot(W)[0][y_train[5:6].argmax()]
```

    -0.005511349692839673

Important: we follow kind of a naive implementation. The aim is to be able to understand what is going on!

In order to quantify how good (or how bad) our weight matrix W can predict the data in our training set, we need to implement a loss function. Here we take a go at the hinge loss, which tries to predict the correct class with a margin of at least one to all other classes (or in this case, like presented in Goldberg, to the class which does not equal the true class, but which scores highest). In my understanding, this is a one-vs-one approach (true class vs. class with highest score (but doesn't equal the true class)).

```python
def hinge_loss(x, y, W, index):
    """
    Calculates the loss of a single data point by taking the prediction of the correct value and the the prediction of
    the value of next highest score, following Crammer and Singer (2001)
    :param x: sample point x as a vector
    :param y: correct label y for x as a vector
    :param W: weight matrix
    :param index: column index of data matrix X
    :return: loss
    """
    loss = 0
    y_index = y[index].argmax()
    y_value = x.dot(W)[y_index]
    y_hat_max_value = np.delete(x.dot(W), y_index).max()
    #for j in range(0, y.shape[1]):  # in case we wanted to classify against all other classes (one-vs-all) --> currently one-vs-one
        #if j == y_index:
            #continue
    loss += max(0, 1 - (y_value - y_hat_max_value))
    return loss
```

With matrix multiplication, we could get all the scores at once. In the following, however, we focus on an approach which takes sample by sample and calculates the loss and the gradients.

```python
scores = X_train.dot(W)  # simple matrix multiplication to get all scores
```

```python
scores
```

    array([[-5.55398818e-04,  3.37949002e-03, -8.40643524e-05,
            -2.72327890e-03,  2.27223898e-03, -4.19751279e-03],
           [-3.02864621e-03, -8.57414552e-04, -4.44178360e-03,
            -4.11820898e-03,  3.05471255e-03,  1.43402301e-03],
           [ 2.68524769e-03,  2.54146957e-03,  1.71621466e-03,
             3.04851690e-03,  5.60148080e-04,  1.58296171e-03],
           [ 6.07786877e-03,  9.80210433e-04,  1.45588397e-03,
             4.25217812e-03, -6.79212250e-04,  8.68427653e-04],
           [-2.66124536e-03,  8.88080362e-04,  3.02651856e-04,
            -7.39309752e-04,  3.83831939e-03,  4.96306015e-06],
           [-1.65841868e-03, -5.58467584e-03, -5.62853211e-03,
            -5.51134969e-03, -1.91774985e-03,  1.06960255e-03],
           [ 6.68489444e-04, -9.52709157e-04,  6.06902896e-03,
             2.99064977e-03, -2.74790493e-03,  5.99410083e-05],
           [-1.09604347e-03,  2.34331758e-03, -1.70226576e-03,
            -3.43223327e-03,  6.39593485e-04,  3.13845305e-03],
           [ 4.01153457e-03, -1.75777546e-03,  4.76723649e-03,
             1.34746827e-04,  3.88965791e-04,  1.54363000e-03],
           [ 9.03308560e-04, -1.50523336e-03,  2.04904889e-03,
             3.65883895e-03,  1.74633377e-03,  3.62567453e-03],
           [ 4.52825975e-03, -3.36132303e-03, -2.09312431e-03,
            -2.56750485e-04, -2.85020477e-03,  1.97404022e-03],
           [ 2.86929292e-05, -3.12982777e-03,  2.51737000e-03,
             1.61146964e-03,  1.16598467e-03,  1.38131041e-03],
           [ 1.25468042e-03,  3.55457144e-03,  2.04847053e-03,
            -7.80614265e-04,  1.55386126e-04,  1.48975012e-03],
           [-1.84322518e-03, -2.47633666e-03, -4.98749106e-03,
            -6.05062610e-03,  2.11161810e-03,  2.19335207e-03],
           [ 2.10508964e-03,  5.52673541e-03,  3.21037747e-03,
            -2.41858395e-03,  1.40176148e-03,  7.89224485e-04],
           [ 1.35362921e-03, -1.54690976e-03,  1.84040171e-03,
             4.11198799e-03, -1.09150846e-04, -8.22979741e-05],
           [-2.83882035e-03,  3.11551345e-04,  1.66067287e-03,
            -3.36640806e-03, -6.32316236e-04,  2.01833056e-03],
           [ 4.10264110e-03, -3.92537847e-03, -2.27597716e-03,
            -2.44793088e-03, -4.39975576e-03,  3.52977165e-03],
           [-1.48527951e-03,  3.66382385e-04,  1.35215662e-03,
            -1.50648057e-03, -1.57596236e-03, -5.50518226e-03],
           [-9.72366285e-04,  4.78814103e-03,  2.58910669e-03,
             5.39394220e-03, -1.57293616e-03,  4.22924061e-03],
           [-2.88692861e-03,  1.52110310e-04,  4.99086604e-04,
            -1.33806624e-03,  7.22355055e-04, -2.76405320e-04],
           [-6.23605884e-04, -1.39092401e-03,  1.88068480e-03,
             1.90631752e-03,  3.75719249e-03, -2.99704478e-04],
           [-2.60698050e-03,  8.18836268e-05, -1.70551676e-03,
             5.59892296e-04, -9.25462194e-04, -2.44649279e-03],
           [ 2.40158754e-04,  5.52718652e-03,  2.97087697e-03,
             1.81675300e-03,  5.84908771e-04,  3.47649932e-03],
           [-2.85837995e-04,  5.37796488e-03,  8.43814550e-04,
            -1.65335972e-04,  2.57437251e-03, -6.16078555e-03],
           [-1.58338925e-03, -1.20432639e-03, -6.76867686e-04,
            -6.73685825e-04, -2.45493779e-03, -2.31849650e-03],
           [ 4.60197097e-03, -2.91921895e-03,  2.75858197e-03,
            -3.73946909e-03, -2.34702825e-03,  4.46608037e-03],
           [ 7.29854071e-04,  1.48284915e-03,  2.70595662e-03,
             1.97892452e-04,  7.04919414e-04,  6.74455460e-04],
           [-7.37362575e-03, -5.55209757e-03, -3.89684395e-03,
            -5.83786550e-03,  1.59977384e-04,  3.58215542e-03],
           [-7.53444678e-04,  3.86701704e-03, -7.58640423e-04,
             2.73705550e-03,  1.99547972e-03, -1.47925254e-03],
           [ 6.72226578e-04,  2.67331981e-03,  8.42467197e-04,
             5.05579807e-03, -1.63158415e-03, -1.77346576e-03],
           [ 2.53123290e-03, -2.97967462e-03, -3.26974881e-03,
            -5.35584605e-03,  2.19329177e-04, -2.84576780e-03],
           [ 6.43803319e-03,  3.57852190e-03, -6.84739654e-04,
             4.26289510e-03, -2.42325371e-03,  3.33888212e-03],
           [-1.12512363e-03, -4.10571823e-03,  1.28758455e-03,
             2.28303126e-03,  2.40930636e-03,  2.86519536e-04],
           [-9.11245238e-04,  6.30418204e-04, -1.23619257e-03,
             1.08606840e-03,  4.70362341e-04, -2.66840432e-03],
           [ 3.72506413e-04,  2.53446537e-03,  6.10840306e-03,
             2.06334938e-03,  1.43158279e-03, -3.92986162e-03],
           [ 7.27886403e-03, -1.90580867e-03,  4.19118836e-03,
             2.70185242e-03, -2.24546200e-04,  5.94045133e-03],
           [-4.32271571e-04, -8.04898991e-04,  2.62491977e-04,
             5.10430757e-03, -3.67055110e-03, -3.62994102e-03],
           [-6.35702001e-04, -1.44506161e-03, -2.43002807e-03,
             8.26612928e-04, -2.79540717e-03, -3.87367022e-03],
           [-1.01093710e-03, -1.39941771e-04, -2.51291947e-04,
             4.51631096e-03, -1.52482374e-03, -2.35623262e-03],
           [ 3.49168328e-03,  1.47389333e-03, -1.45000577e-03,
            -3.06382997e-04, -9.73322780e-04,  3.65063085e-03],
           [-4.28282440e-05,  1.93962830e-03,  3.20790645e-03,
            -4.92906061e-04, -2.65847573e-04,  8.51443525e-05],
           [-1.91249278e-03, -1.55674671e-03, -3.78584722e-03,
            -5.57076226e-03,  4.93095817e-03,  3.46342033e-03],
           [-2.53666613e-03,  1.06093719e-03, -1.60635503e-03,
            -1.82611733e-03, -1.07202843e-04,  2.37305550e-03],
           [-2.24231882e-03, -4.25484236e-03, -2.92661727e-03,
            -3.76271706e-03, -2.63426990e-03,  1.54798272e-03],
           [-3.92393104e-03, -4.37583119e-03, -6.83100648e-03,
            -4.80530124e-03,  1.17665768e-03,  4.24925863e-03],
           [-6.03832528e-03, -6.13488162e-03,  1.06133942e-03,
            -3.82860031e-04,  7.70334022e-04, -1.33933247e-03],
           [-4.36028898e-03, -7.31474705e-04, -4.12771197e-03,
            -2.63947413e-03,  5.49441884e-04, -9.97584426e-04],
           [ 2.05081700e-03, -2.13252788e-04, -2.41023521e-04,
            -8.56090121e-04,  4.01033782e-04, -4.06884224e-03],
           [ 1.07529014e-03,  3.37191215e-03,  9.30435452e-04,
             4.48896151e-04, -1.60049308e-03, -4.92705985e-04],
           [-1.64570617e-03, -3.36795974e-04, -5.04819172e-04,
             2.31189299e-03, -2.98629563e-03, -1.09210064e-03],
           [ 1.51681385e-03,  1.63894965e-03,  6.73197669e-04,
             1.40719752e-03,  8.77669363e-04, -4.56386402e-03],
           [-6.17202456e-03, -1.56389107e-03,  1.63903722e-03,
             4.45352358e-03, -2.18417294e-03, -3.54034279e-03],
           [ 5.14020346e-03,  3.66308904e-03, -6.19451688e-04,
             5.54488663e-04,  1.30402766e-03, -4.29022322e-03]])

```python
def gradient(X, y, W):
    """
    compute the gradient
    :param X: data matrix (train)
    :param y: the corresponding
    :param W: weight matrix
    :return: loss and Jacobian dW with all gradients
    """
    dW = np.zeros(W.shape)

    total_loss = 0.0

    for index, x in enumerate(X):
        y_index = y[index].argmax()
        y_value = x.dot(W)[y_index]
        y_hat_max_value = np.delete(x.dot(W), y_index).max()
        loss = max(0, 1 - (y_value - y_hat_max_value))
        total_loss += loss
        y_hat_max_index = np.delete(x.dot(W), y_index).argmax() + 1
        if loss > 0:  # not sure whether we need this if statement
            dW[:, y_hat_max_index] += x.transpose()
            dW[:, y_index] -= x.transpose()

    total_loss /= X.shape[0]
    dW /= X.shape[0]

    return total_loss, dW

```

```python
def gradient_descent(X, y, W, eta, steps):
    """
    Perform gradient descent for a number of times with a fixed learning rate eta
    :param X: data matrix
    :param y: labels
    :param W: weight matrix
    :param eta: learning rate
    :param steps: number of times gradient descent should be performed
    :return: learned representation matrix W_learned
    """
    W_learned = W.copy()

    for step in range(0, steps):
        loss, dW = gradient(X, y, W_learned)
        print(loss)
        W_learned = W_learned - eta * dW

    return W_learned

```

```python
W_star = gradient_descent(X_train, y_train, W, eta=0.001, steps=100)
```

    1.0038170867499712
    0.9458004956913564
    0.9029223134272765
    0.8359050724406546
    0.7627383071817009
    0.7002533175250243
    0.6363839674437564
    0.5601080098980816
    0.4977957071945216
    0.43727328194478265
    0.39969043635331086
    0.37194985032759115
    0.3451396585114108
    0.31993737904591485
    0.29542868779422654
    0.2730317394318695
    0.25051221058149853
    0.22653748291157821
    0.20419158204055793
    0.18030176579112206
    0.1613555932139755
    0.14161625629003335
    0.12595610133530682
    0.11406584874399786
    0.10426680552175424
    0.09156041227255908
    0.08373285352546704
    0.07634078453206532
    0.07264716918242302
    0.06919024286641753
    0.06600104960037959
    0.06336474174747486
    0.062469866350450105
    0.06050885893178353
    0.05983604994039867
    0.05923939378809413
    0.05776841144964399
    0.05749401804990851
    0.05721962465017302
    0.05714119484868436
    0.054972663633005285
    0.05427993779004683
    0.05406309086590985
    0.05353799074850154
    0.053318813705064985
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394
    0.053161296647773394

### Testing
Let's test if our learned representation of the data is any good at classifying the data in the test set. Of course we need the bias in our test set as well!

```python
bias_vector_test = np.ones([X_test.shape[0], 1])
X_test = np.append(test_norm, bias_vector_test, axis=1)
```

```python
for index, x in enumerate(X_test.dot(W_star)):
    pred = x.argmax()
    true_label = y_test[index].argmax()
    print(pred, true_label)
```

    3 3
    5 5
    5 5
    4 4
    0 0
    0 0

Not too bad! But Goldberg mentioned something about regularisation, so we should take this into account as well!

```python
def gradient_reg(X, y, W, lam):
    """
    compute the gradient
    :param X: data matrix (train)
    :param y: the corresponding
    :param W: weight matrix
    :param lam: reguliser lambda
    :return: Jacobian dW with all gradients
    """
    dW = np.zeros(W.shape)

    total_loss = 0.0

    for index, x in enumerate(X):
        y_index = y[index].argmax()
        y_value = x.dot(W)[y_index]
        y_hat_max_value = np.delete(x.dot(W), y_index).max()
        loss = max(0, 1 - (y_value - y_hat_max_value))
        total_loss += loss
        y_hat_max_index = np.delete(x.dot(W), y_index).argmax() + 1
        if loss > 0:  # not sure whether we need this if statement
            dW[:, y_hat_max_index] += x.transpose()
            dW[:, y_index] -= x.transpose()

    loss /= X.shape[0]
    dW /= X.shape[0]

    total_loss += lam * np.linalg.norm(W, 2)
    dW += lam * W

    return total_loss, dW

def gradient_descent_reg(X, y, W, eta, steps):
    """
    Perform gradient descent for a number of times with a fixed learning rate eta
    :param X: data matrix
    :param y: labels
    :param W: weight matrix
    :param eta: learning rate
    :param steps: number of times gradient descent should be performed
    :return: learned representation matrix W_learned
    """
    W_learned = W.copy()

    for step in range(0, steps):
        loss, dW = gradient_reg(X, y, W_learned, 2)
        print(loss)
        W_learned = W_learned - eta * dW

    return W_learned

```

```python
W_star_reg = gradient_descent_reg(X_train, y_train, W, eta=0.001, steps=20)
```

    54.20758211976921
    51.076752717461495
    48.764681892306726
    45.150246857914915
    41.20739764734916
    37.842593970072826
    34.40372589873632
    30.297678814172144
    26.947244821759636
    23.67920299683294
    21.65552984702191
    20.159673247871645
    18.719435368574114
    17.37361158137452
    16.031237446541766
    14.79426200478653
    13.69071268963535
    12.341406359158503
    11.25382383570278
    9.978812350016465

```python
for index, x in enumerate(X_test.dot(W_star_reg)):
    pred = x.argmax()
    true_label = y_test[index].argmax()
    print(pred, true_label)
```

    3 3
    5 5
    5 5
    4 4
    0 0
    0 0

If we look at the two different weight matrices (one regularised, the other not), we notice the following:

```python
W_star[0:5]
```

    array([[-5.78647135e-04,  1.28346720e-03, -1.93445596e-03,
             2.80640163e-03, -2.44218375e-04, -1.20728113e-03],
           [-7.96955403e-04,  7.22211518e-04, -3.49303253e-03,
            -3.51679433e-03,  5.01353061e-03,  2.24429131e-03],
           [-1.74911317e-03, -1.79858176e-03,  3.92019550e-03,
            -4.73300532e-03,  5.45948384e-03, -7.59202032e-04],
           [ 3.90212020e-03, -1.66440595e-03, -1.09472119e-03,
            -4.97761960e-04,  9.07647146e-05, -8.39547695e-04],
           [-1.87037704e-04, -1.13292487e-03,  9.40532198e-04,
            -5.72550923e-03,  7.10109402e-03, -9.25386999e-04]])

```python
W_star_reg[0:5]
```

    array([[-5.20655827e-04,  1.05672338e-03, -1.40708426e-03,
             2.39033138e-03, -4.62027423e-04, -9.33267750e-04],
           [-5.93592079e-04,  4.90021476e-04, -2.30015460e-03,
            -2.06140822e-03,  3.31698330e-03,  1.31967699e-03],
           [-1.52572058e-03, -1.39847125e-03,  2.97273629e-03,
            -3.91436072e-03,  4.52360052e-03, -3.21388872e-04],
           [ 3.38964849e-03, -1.44861974e-03, -9.56067745e-04,
            -6.55413811e-04,  7.09559278e-05, -5.03024378e-04],
           [-2.05402557e-04, -9.42166370e-04,  6.01204214e-04,
            -4.90855116e-03,  5.93320763e-03, -4.08228655e-04]])

## By the way ...
### In scikit-learn it's much easier to implement this :-)

```python
from sklearn.svm import LinearSVC
clf = LinearSVC(random_state=0, multi_class='crammer_singer', loss='hinge')
clf.fit(X_train, train.y)
```

    LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
         intercept_scaling=1, loss='hinge', max_iter=1000,
         multi_class='crammer_singer', penalty='l2', random_state=0,
         tol=0.0001, verbose=0)

```python
clf.predict(X_test)
```

    array(['en', 'fi', 'fi', 'de', 'fr', 'fr'], dtype=object)

```python
test.y
```

    4     en
    12    fi
    16    fi
    26    de
    41    fr
    42    fr
    Name: y, dtype: object

We see that with our naive implementation, we do not much worse than with scikit's. scikit's implementation is of course much more elaborate and uses the vectorised operation and possibly other optimisation techniques in order to make its SVM (or SVC) better.

```python

```