```python
'''
MIT License

Copyright (c) 2021 Valentin Gorbunov

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
```

    '\nMIT License\n\nCopyright (c) 2021 Valentin Gorbunov\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the "Software"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n'

## Environment Set Up

```python
# Install packages not in Colab by default
!pip install vaderSentiment
!pip install textstat
!pip install flwr
```

    Requirement already satisfied: vaderSentiment in /usr/local/lib/python3.7/dist-packages (3.3.2)
    Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from vaderSentiment) (2.23.0)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (2.10)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (2021.10.8)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (1.24.3)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (3.0.4)
    Requirement already satisfied: textstat in /usr/local/lib/python3.7/dist-packages (0.7.3)
    Requirement already satisfied: pyphen in /usr/local/lib/python3.7/dist-packages (from textstat) (0.12.0)
    Requirement already satisfied: flwr in /usr/local/lib/python3.7/dist-packages (0.18.0)
    Requirement already satisfied: importlib-metadata<2.0.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.7.0)
    Requirement already satisfied: numpy<2.0.0,>=1.19.0 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.21.6)
    Requirement already satisfied: grpcio<=1.43.0,>=1.27.2 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.43.0)
    Requirement already satisfied: protobuf<4.0.0,>=3.12.1 in /usr/local/lib/python3.7/dist-packages (from flwr) (3.17.3)
    Requirement already satisfied: google<3.0.0,>=2.0.3 in /usr/local/lib/python3.7/dist-packages (from flwr) (2.0.3)
    Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from google<3.0.0,>=2.0.3->flwr) (4.6.3)
    Requirement already satisfied: six>=1.5.2 in /usr/local/lib/python3.7/dist-packages (from grpcio<=1.43.0,>=1.27.2->flwr) (1.15.0)
    Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<2.0.0,>=1.4.0->flwr) (3.8.0)

```python
# Import packages
import pickle
import sys
import string
import re
import nltk
import seaborn
import os
import time
import warnings
import random
import threading
import scipy
import json
import nltk.stem.porter
import pandas as pd
import numpy as np
import flwr as fl
import matplotlib.pyplot as plt
from matplotlib import rcParams
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import SelectFromModel
from sklearn.pipeline import make_pipeline
from sklearn.metrics import classification_report, confusion_matrix, log_loss, precision_recall_fscore_support
from sklearn.model_selection import train_test_split, StratifiedKFold
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer as VS
from textstat.textstat import *
from typing import Tuple, Union, List, Dict, Optional
from flwr.server.criterion import Criterion
from flwr.server.client_proxy import ClientProxy
from flwr.common import Weights, FitRes, Parameters, Scalar, parameters_to_weights, weights_to_parameters
from google.colab import drive
from multiprocessing import Process, Manager
from multiprocessing.managers import ListProxy
from math import ceil
from statistics import mean
from datetime import datetime
from functools import reduce
import textwrap
%matplotlib inline
```

```python
# Mount external storage
drive.mount('/content/drive')
```

    Mounted at /content/drive

```python
# Configure global search paths
HOME_DIR = "/content/drive/MyDrive/"
LOGGING_FOLDERS_DIR = HOME_DIR + "Results_Folders/"
```

## Centralized Manual Pipeline

```python
'''
Centralized Manual Pipeline Adapted From:

Title: Automated Hate Speech Detection and the Problem of Offensive Language.ipynb
Authors: Davidson et al.
Date: Mar 29, 2019
Availability: https://github.com/t-davidson/hate-speech-and-offensive-language/blob/master/src/Automated%20Hate%20Speech%20Detection%20and%20the%20Problem%20of%20Offensive%20Language.ipynb

'''
```

    '\nAdapted from:\n\nTitle: Automated Hate Speech Detection and the Problem of Offensive Language.ipynb\nAuthors: Davidson et al.\nDate: Mar 29, 2019\nAvailability: https://github.com/t-davidson/hate-speech-and-offensive-language/blob/master/src/Automated%20Hate%20Speech%20Detection%20and%20the%20Problem%20of%20Offensive%20Language.ipynb\n\n'

### Preliminary Data Analysis

```python
# Deserialize Pandas DataFrame from pickle file
df = pd.read_pickle(open(HOME_DIR + "labeled_data.p", mode='rb'))
```

```python
# Describe content, size and format of the data
df
```

      count
      hate_speech
      offensive_language
      neither
      class
      tweet

      0
      3
      0
      0
      3
      2
      !!! RT @mayasolovely: As a woman you shouldn't...

      1
      3
      0
      3
      0
      1
      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...

      2
      3
      0
      3
      0
      1
      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...

      3
      3
      0
      2
      1
      1
      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...

      4
      6
      0
      6
      0
      1
      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...

      ...
      ...
      ...
      ...
      ...
      ...
      ...

      25291
      3
      0
      2
      1
      1
      you's a muthaf***in lie &#8220;@LifeAsKing: @2...

      25292
      3
      0
      1
      2
      2
      you've gone and broke the wrong heart baby, an...

      25294
      3
      0
      3
      0
      1
      young buck wanna eat!!.. dat nigguh like I ain...

      25295
      6
      0
      6
      0
      1
      youu got wild bitches tellin you lies

      25296
      3
      0
      0
      3
      2
      ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...

24783 rows × 6 columns

      <button class="colab-df-convert" onclick="convertToInteractive('df-8c6a7709-226d-4af6-ae5e-23e518398cb2')"
              title="Convert this dataframe to an interactive table."
              style="display:none;">

  <svg xmlns="http://www.w3.org/2000/svg" height="24px"viewBox="0 0 24 24"
       width="24px">
    <path d="M0 0h24v24H0V0z" fill="none"/>
    <path d="M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z"/><path d="M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z"/>
  </svg>
      </button>

```python
# Generate descriptive statistics
df.describe()
```

      count
      hate_speech
      offensive_language
      neither

      count
      24783.000000
      24783.000000
      24783.000000
      24783.000000

      mean
      3.243473
      0.280515
      2.413711
      0.549247

      std
      0.883060
      0.631851
      1.399459
      1.113299

      min
      3.000000
      0.000000
      0.000000
      0.000000

      25%
      3.000000
      0.000000
      2.000000
      0.000000

      50%
      3.000000
      0.000000
      3.000000
      0.000000

      75%
      3.000000
      0.000000
      3.000000
      0.000000

      max
      9.000000
      7.000000
      9.000000
      9.000000

      <button class="colab-df-convert" onclick="convertToInteractive('df-a7bc4a59-d993-4306-b6b2-dc2cef808552')"
              title="Convert this dataframe to an interactive table."
              style="display:none;">

  <svg xmlns="http://www.w3.org/2000/svg" height="24px"viewBox="0 0 24 24"
       width="24px">
    <path d="M0 0h24v24H0V0z" fill="none"/>
    <path d="M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z"/><path d="M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z"/>
  </svg>
      </button>

```python
# Get feature names
df.columns
```

    Index(['count', 'hate_speech', 'offensive_language', 'neither', 'class',
           'tweet'],
          dtype='object')

Key:

count = number of CrowdFlower users who coded each tweet (at least 3; more users coded a tweet when judgments were determined to be unreliable by CF)

hate_speech = number of CF users who coded the tweet as hate speech

offensive_language = number of CF users who coded the tweet as offensive but not hate speech

neither = number of CF users who coded the tweet as neither offensive nor hate speech

class = class label determined by majority rule:

* 0 : hate speech
* 1 : offensive  language
* 2 : neither

tweet = tweet text

```python
# Plot distribution of samples across classes
labels = df['class']
bins = ['offensive', 'neither', 'hate speech']

y_pos = np.arange(len(bins))
value_counts = labels.value_counts()

plt.bar(y_pos, value_counts, align='center', color=['g', 'r', '#1f77b4'])

plt.xticks(y_pos, bins)
plt.xlabel("Class Label")
plt.ylabel("Frequency")
plt.title("Frequency of Class Labels")

plt.show()
```

The bar chart shows the imbalanced nature of the classification problem; most tweets containing "hateful" words, as defined by Hatebase, were coded as offensive by CF coders. More tweets were considered neither hate speech nor offensive than were considered hate speech.

### Data Preprocessing

```python
# Select data for preprocessing and transformation
tweets=df.tweet
```

```python
# Create list of stopwords
nltk.download('stopwords')
stopwords = nltk.corpus.stopwords.words("english")
other_exclusions = ["#ff", "ff", "rt"]
stopwords.extend(other_exclusions)
```

    [nltk_data] Downloading package stopwords to /root/nltk_data...
    [nltk_data]   Unzipping corpora/stopwords.zip.

```python
# Create reusable stemmer
stemmer = nltk.stem.porter.PorterStemmer()
```

```python
# Define functions for preprocessing data
def preprocess(text_string):
    """
    Accepts a text string and replaces:
    1) urls with URLHERE
    2) lots of whitespace with one instance
    3) mentions with MENTIONHERE

    This allows us to get standardized counts of urls and mentions
    Without caring about specific people mentioned
    """
    space_pattern = '\s+'
    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
        '[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    mention_regex = '@[\w\-]+'
    parsed_text = re.sub(space_pattern, ' ', text_string)
    parsed_text = re.sub(giant_url_regex, '', parsed_text)
    parsed_text = re.sub(mention_regex, '', parsed_text)
    return parsed_text

def tokenize(tweet):
    """Removes punctuation & excess whitespace, sets to lowercase,
    and stems tweets. Returns a list of stemmed tokens."""
    tweet = " ".join(re.split("[^a-zA-Z]*", tweet.lower())).strip()
    tokens = [stemmer.stem(t) for t in tweet.split()]
    return tokens

def basic_tokenize(tweet):
    """Same as tokenize but without the stemming"""
    tweet = " ".join(re.split("[^a-zA-Z.,!?]*", tweet.lower())).strip()
    return tweet.split()
```

### Feature Engineering

####Automatic Feature Extraction

```python
# Construct tfidf matrix and get relevant scores
vectorizer = TfidfVectorizer(
    tokenizer=tokenize,
    preprocessor=preprocess,
    ngram_range=(1, 3),
    stop_words=stopwords,
    use_idf=True,
    smooth_idf=False,
    norm=None,
    decode_error='replace',
    max_features=10000,
    min_df=5,
    max_df=0.75
    )

tfidf = vectorizer.fit_transform(tweets).toarray()
vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names_out())}
idf_vals = vectorizer.idf_
idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores
```

    /usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:401: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['b', 'c', 'e', 'f', 'g', 'h', 'j', 'l', 'n', 'p', 'r', 'u', 'v', 'w'] not in stop_words.
      % sorted(inconsistent)

```python
# Get POS tags for tweets and save as a string
nltk.download('averaged_perceptron_tagger')
tweet_tags = []
for t in tweets:
    tokens = basic_tokenize(preprocess(t))
    tags = nltk.pos_tag(tokens)
    tag_list = [x[1] for x in tags]
    tag_str = " ".join(tag_list)
    tweet_tags.append(tag_str)
```

    [nltk_data] Downloading package averaged_perceptron_tagger to
    [nltk_data]     /root/nltk_data...
    [nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.

```python
# Construct POS TF matrix and get vocab dict
pos_vectorizer = TfidfVectorizer(
    tokenizer=None,
    lowercase=False,
    preprocessor=None,
    ngram_range=(1, 3),
    stop_words=None,
    use_idf=False,
    smooth_idf=False,
    norm=None,
    decode_error='replace',
    max_features=5000,
    min_df=5,
    max_df=0.75,
    )

pos = pos_vectorizer.fit_transform(pd.Series(tweet_tags)).toarray()
pos_vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names_out())}
```

#### Manual Feature Construction

```python
# Create reusable sentiment analyzer
sentiment_analyzer = VS()
```

```python
# Define functions for devising features
def count_twitter_objs(text_string):
    """
    Accepts a text string and replaces:
    1) urls with URLHERE
    2) lots of whitespace with one instance
    3) mentions with MENTIONHERE
    4) hashtags with HASHTAGHERE

    This allows us to get standardized counts of urls and mentions
    Without caring about specific people mentioned.

    Returns counts of urls, mentions, and hashtags.
    """
    space_pattern = '\s+'
    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
        '[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    mention_regex = '@[\w\-]+'
    hashtag_regex = '#[\w\-]+'
    parsed_text = re.sub(space_pattern, ' ', text_string)
    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)
    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)
    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)
    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))

def other_features(tweet):
    """This function takes a string and returns a list of features.
    These include Sentiment scores, Text and Readability scores,
    as well as Twitter specific features"""
    sentiment = sentiment_analyzer.polarity_scores(tweet)

    words = preprocess(tweet) # Get text only

    syllables = textstat.syllable_count(words)
    num_chars = sum(len(w) for w in words)
    num_chars_total = len(tweet)
    num_terms = len(tweet.split())
    num_words = len(words.split())
    avg_syl = round(float((syllables+0.001))/float(num_words+0.001),4)
    num_unique_terms = len(set(words.split()))

    # Modified FK grade, where avg words per sentence is just num words/1
    FKRA = round(float(0.39 * float(num_words)/1.0) + float(11.8 * avg_syl) - 15.59,1)
    # Modified FRE score, where sentence fixed to 1
    FRE = round(206.835 - 1.015*(float(num_words)/1.0) - (84.6*float(avg_syl)),2)

    twitter_objs = count_twitter_objs(tweet)
    retweet = 0
    if "rt" in words:
        retweet = 1
    features = [FKRA, FRE,syllables, avg_syl, num_chars, num_chars_total, num_terms, num_words,
                num_unique_terms, sentiment['neg'], sentiment['pos'], sentiment['neu'], sentiment['compound'],
                twitter_objs[2], twitter_objs[1],
                twitter_objs[0], retweet]
    return features

def get_feature_array(tweets):
    feats=[]
    for t in tweets:
        feats.append(other_features(t))
    return np.array(feats)
```

```python
# Construct features
other_feats = get_feature_array(tweets)
```

### Feature Selection

```python
# Join automatically extracted features and manually constructed features together
X = np.concatenate([tfidf,pos,other_feats],axis=1)
```

```python
# Get dimensions of the data
X.shape
```

    (24783, 4023)

The data is high dimensional. There is an exponential decrease in predictive power for each additional variable in a multivariate model, known as the curse of dimensionality. Therefore, the dimensionality of the data should be reduced.

### Model Training

```python
# Encode class labels
y = df['class'].astype(int)
```

```python
# Reduce dimensionality of data. Specify solver as 'liblinear' because default solver changed to 'lbfgs' in 0.22.
feature_transformer = SelectFromModel(LogisticRegression(class_weight='balanced',penalty="l1", C=0.01, solver='liblinear'))

X_ = feature_transformer.fit_transform(X,y)
```

```python
# Get reduced dimensions of the data
X_.shape
```

    (24783, 465)

```python
# Generate train/test split of 9/1. Initialise with random seed for reproducability.
X_train, X_test, y_train, y_test = train_test_split(X_, y, random_state=42, test_size=0.1, stratify=y)
```

As classification problem is imbalanced, it's desirable for the train/test split to preserve the same proportions of examples in each class as observed in the original dataset [(Brownlee, 2020)](https://machinelearningmastery.com/train-test-split-for-evaluating-machine-learning-algorithms//).

```python
# Create model. Change hyperparameters to ensure control variables remain constant. Change solver to the new default solver, lbfgs.
# Increase `max_iter` as lbfgs takes longer to converge. Change framework used for multi class classification to multinomial.
# See comments in `start_flower_client` for details.
classifier = LogisticRegression(class_weight='balanced',penalty='l2', C=0.01, max_iter=20000, multi_class="multinomial")
```

```python
# Train model
model = classifier.fit(X_train, y_train)
```

### Model Testing

```python
# Test model
y_preds = model.predict(X_test)
```

```python
# Generate test report
report = classification_report(y_test, y_preds)
print(report.rsplit("\n",2)[0])
```

                  precision    recall  f1-score   support

               0       0.24      0.69      0.36       143
               1       0.96      0.74      0.84      1920
               2       0.61      0.86      0.72       416

        accuracy                           0.76      2479
       macro avg       0.61      0.77      0.64      2479

There is a class-imbalance in the dataset; the hate speech class (0) and neither class (2) are most important to get right, and they are under-represented. Offensive language class (1) is over-represented. Therefore, macro average is preferable over weighted average, which is misleading [(Khalusova, 2021)](https://www.mariakhalusova.com/posts/2019-04-17-ml-model-evaluation-metrics-p2/#micro--macro--weighted-averaged-precision).

F1-score is used rather than AUROC because AUROC averages over all possible discrimination thresholds, which is misleading in the case of class-imbalanced data [(2018, Almardeny)](https://stackoverflow.com/questions/44172162/f1-score-vs-roc-auc). In contrast, F-1 score maintains a balance between Precision and Recall.

```python
# Define function for plotting confusion matrix
def plot_confusion_matrix(y, y_preds):

  matrix = confusion_matrix(y,y_preds)
  matrix_proportions = np.zeros((3,3))
  for i in range(0,3):
    matrix_proportions[i,:] = matrix[i,:]/float(matrix[i,:].sum())
  names=['Hate','Offensive','Neither']
  confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)
  plt.figure(figsize=(5,5))
  seaborn.heatmap(confusion_df,annot=True,annot_kws={"size": 12},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')
  plt.ylabel(r'True categories',fontsize=14)
  plt.xlabel(r'Predicted categories',fontsize=14)
  plt.tick_params(labelsize=12)
```

```python
# Plot confusion matrix
plot_confusion_matrix(y_test,y_preds)
```

```python
# Plot distribution of estimates. Set the density parameter to True to normalize bin heights so that the integral of the histogram is 1.
# The resulting histogram is an approximation of the probability density function. This facilitates comparison between the distribution
# of estimates and true distribution.
plt.hist(y_preds, density=True)
plt.show()
```

```python
# Plot true distribution
plt.hist(y, density=True)
plt.show()
```

# Design of Single-Machine Simulation of Federated Learning System

We use a controlled environment that is suitable for experiments. We assume
a synchronous update scheme that proceeds in rounds of
communication. There is a fixed set of $K$ clients, each
with a fixed local dataset. At the beginning of each round, the server
sends the current global model parameters to selected clients. Each selected client then trains its model on its local dataset, and sends an
update to the server. The server aggregates these updates and applies them
to the global model, and the cycle repeats [(McMahan et al., 2017)](https://arxiv.org/pdf/1602.05629.pdf). In our simulation, of cross-silo FL, all clients are selected for each round of federated learning.

## Federated Evaluation

There are two main approaches to evaluate models in a federated learning system: centralized evaluation and federated evaluation. In centralized evaluation, the aggregated model is validated on the server-side, and the validation set is sampled from the population's probability distribution. In federated evaluation, the aggregated model is validated on the client-side, and the validation set is sampled from the same probability distribution as the client's training set [(Adap GmbH, 2022)](https://flower.dev/docs/evaluation.html).

We focus on federated evaluation as it's typically used in real-world applications.

```python
# Initialize global experiment config
DATA_PARTITIONING_STRATEGY = ""
DATA_PARTITIONING_RANDOM_SEED = 0
DIRICHLET_DISTRIBUTION_PARAM = 10
NUMBER_OF_CLIENTS = 10
MIN_AVAILABLE_CLIENTS = NUMBER_OF_CLIENTS
NUMBER_OF_FL_ROUNDS = 50
LOCAL_EPOCHS = ceil(1500 / NUMBER_OF_FL_ROUNDS)
MIN_FIT_CLIENTS = NUMBER_OF_CLIENTS
FRACTION_FIT = 1.0
FRACTION_FIT_RANDOM_SEED = 1
MIN_EVAL_CLIENTS = NUMBER_OF_CLIENTS
FRACTION_EVAL = 1.0
EVAL_STRATEGY = "federated"
DATA_SHARING = False
NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY = 10
NUMBER_OF_VALIDATION_FOLDS = 10
NUMBER_OF_REPEATS = NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY * NUMBER_OF_VALIDATION_FOLDS
LOG_DIR = ""
```

Key:

DATA_PARTITIONING_STRATEGY - strategy for distributing samples across clients

DATA_PARTITIONING_RANDOM_SEED - seed to initialize random data partition generator, set for experimental control and reproducability

DIRICHLET_DISTRIBUTION_PARAM - distribution parameter of the Dirichlet distribution, used in particular data partitioning strategies to control degree of imbalance

NUMBER_OF_CLIENTS - total number of clients participating in the FL

MIN_AVAILABLE_CLIENTS - minimum number of clients that must be connected to the server before FL can begin

NUMBER_OF_FL_ROUNDS - number of FL rounds, set to a sufficiently large number so that the performance of the FL system can be observed

LOCAL_EPOCHS - number of complete passes over local training data performed by clients during a training round, limited by the maximum number of iterations taken for the solvers to converge and the number of FL rounds, so that the performance of the FL system can be observed

MIN_FIT_CLIENTS - minimum number of clients used during training

FRACTION_FIT - fraction of random clients selected to participate in a training round

FRACTION_FIT_RANDOM_SEED - seed for initializing the random sampling used to select clients for participation in a training round, set for experimental control and reproducability

MIN_EVAL_CLIENTS - minimum number of clients used during evaluation

FRACTION_EVAL - fraction of random clients selected to participate in an evaluation round

EVAL_STRATEGY - there are two main approaches to evaluate models in federated learning systems: centralized (server-side) evaluation and federated (client-side) evaluation

DATA_SHARING - whether or not to use a data-sharing strategy [(Zhao et al., 2018)](https://arxiv.org/pdf/1806.00582.pdf) to improve training on non-IID data

NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY - number of observations drawn using each data partitioning strategy

NUMBER_OF_VALIDATION_FOLDS - number of folds used in cross-validation

NUMBER_OF_REPEATS - total number of experiment repeats

LOG_DIR - folder in which to log experiments

## Utility Functions

```python
LogRegParams = Tuple[np.ndarray, np.ndarray]

def get_model_parameters(model: LogisticRegression) -> LogRegParams:
#Returns the paramters of a sklearn LogisticRegression model.
    params = (model.coef_, model.intercept_)
    return params

def set_model_params(model: LogisticRegression, params: LogRegParams) -> LogisticRegression:
#Sets the parameters of a sklean LogisticRegression model.
    model.coef_ = params[0]
    model.intercept_ = params[1]
    return model

def set_initial_params(model: LogisticRegression):
#Sets initial parameters as zeros Required since model params are
#uninitialized until model.fit is called.
#But server asks for initial parameters from clients at launch. Refer
#to sklearn.linear_model.LogisticRegression documentation for more
#information.

    if DATA_SHARING:
      model.classes_ = warm_up_model.classes_
      model.coef_ = warm_up_model.coef_
      model.intercept_ = warm_up_model.intercept_
    else:
      n_classes = 3
      n_features = X_.shape[1]  # Number of features in dataset after feature selection by transformer

      model.classes_ = np.array([i for i in range(n_classes)])
      model.coef_ = np.zeros((n_classes, n_features))
      model.intercept_ = np.zeros((n_classes,))

def get_partitions(x, y):

    '''
    Adapted from:

    Title: NIID-Bench
    Authors: Li et al.
    Date: Nov 17, 2021
    Availability: https://github.com/Xtra-Computing/NIID-Bench/blob/main/utils.py

    '''

    np.random.seed(DATA_PARTITIONING_RANDOM_SEED)

    n_train = y.shape[0]

    if DATA_PARTITIONING_STRATEGY == "iid":
        idxs = np.random.permutation(n_train)
        batch_idxs = np.array_split(idxs, NUMBER_OF_CLIENTS)
        net_dataidx_map = {i: batch_idxs[i] for i in range(NUMBER_OF_CLIENTS)}
    elif DATA_PARTITIONING_STRATEGY == "noniid-distribution-based-label-imbalance":
        min_size = 0
        min_require_size = 10
        K = 3
        N = y.shape[0]
        net_dataidx_map = {}

        while min_size < min_require_size:
            idx_batch = [[] for _ in range(NUMBER_OF_CLIENTS)]

            for k in range(K):
                idx_k = np.where(y == k)[0]
                np.random.shuffle(idx_k)
                proportions = np.random.dirichlet(np.repeat(DIRICHLET_DISTRIBUTION_PARAM, NUMBER_OF_CLIENTS))
                proportions = np.array([p * (len(idx_j) < N / NUMBER_OF_CLIENTS) for p, idx_j in zip(proportions, idx_batch)])
                proportions = proportions / proportions.sum()
                proportions = (np.cumsum(proportions) * len(idx_k)).astype(int)[:-1]
                idx_batch = [idx_j + idx.tolist() for idx_j, idx in zip(idx_batch, np.split(idx_k, proportions))]
                min_size = min([len(idx_j) for idx_j in idx_batch])

        for j in range(NUMBER_OF_CLIENTS):
            np.random.shuffle(idx_batch[j])
            net_dataidx_map[j] = idx_batch[j]
    elif DATA_PARTITIONING_STRATEGY == "noniid-quantity-based-label-imbalance-1" or DATA_PARTITIONING_STRATEGY == "noniid-quantity-based-label-imbalance-2":
        num = int(DATA_PARTITIONING_STRATEGY[38]) # number of distinct labels per client
        K = 3 # total number of labels
        times = [0 for _ in range(K)] # holds the number of parties to which each label ID is assigned
        contain = [[] for _ in range(NUMBER_OF_CLIENTS)] # holds the buckets of num label IDs assigned to each party

        for i in range(K): # ensure each label ID is assigned
            bucket = contain[np.random.randint(NUMBER_OF_CLIENTS)]
            while len(bucket) == num:
                bucket = contain[np.random.randint(NUMBER_OF_CLIENTS)]
            bucket.append(i)
            times[i] += 1

        for bucket in contain: # randomly assign num different distinct label IDs to each party
            j = len(bucket)
            while j < num:
                ind = np.random.randint(K)
                if ind not in bucket:
                    bucket.append(ind)
                    times[ind] += 1
                    j += 1

        net_dataidx_map ={i: np.ndarray(0, dtype=np.int64) for i in range(NUMBER_OF_CLIENTS)}

        # For the samples of each label, randomly and equally divide them into the parties
        # which own the label, such that there is no overlap between the samples of different parties
        for i in range(K):
            idx_k = np.where(y == i)[0]
            np.random.shuffle(idx_k)
            split = np.array_split(idx_k,times[i])
            ids = 0
            for j in range(NUMBER_OF_CLIENTS):
                if i in contain[j]:
                    net_dataidx_map[j]=np.append(net_dataidx_map[j],split[ids])
                    ids += 1

    elif DATA_PARTITIONING_STRATEGY == "iid-quantity-skew":
        idxs = np.random.permutation(n_train)
        min_size = 0
        while min_size < 10:
            proportions = np.random.dirichlet(np.repeat(DIRICHLET_DISTRIBUTION_PARAM, NUMBER_OF_CLIENTS))
            proportions = proportions/proportions.sum()
            min_size = np.min(proportions*len(idxs))
        proportions = (np.cumsum(proportions)*len(idxs)).astype(int)[:-1]
        batch_idxs = np.split(idxs, proportions)
        net_dataidx_map = {i: batch_idxs[i] for i in range(NUMBER_OF_CLIENTS)}

    partitioned_train_set = [(x[net_dataidx_map[idx]], pd.Series(y[net_dataidx_map[idx]])) for idx in range(NUMBER_OF_CLIENTS)]

    return partitioned_train_set

def get_train_partitions(x_train, y_train):
  partitioned_train_set = []

  if DATA_SHARING:
    partitioned_train_set = get_partitions(X_train_client, y_train_client.to_numpy())
    # a random α portion of G is merged with the data of each client
    partitioned_train_set = [(np.concatenate((X_train_partition, X_train_shared)), pd.concat((y_train_partition, y_train_shared))) for X_train_partition, y_train_partition in partitioned_train_set]
  else:
    partitioned_train_set = get_partitions(x_train, y_train)

  return partitioned_train_set

def get_test_partitions(x_test, y_test, partitioned_train_set):
  partitioned_test_set = []

  if DATA_PARTITIONING_STRATEGY == "noniid-distribution-based-label-imbalance" or DATA_PARTITIONING_STRATEGY == "iid-quantity-skew":
    # Grody estimate of distribution

    np.random.seed(DATA_PARTITIONING_RANDOM_SEED)

    net_dataidx_map = {i: np.ndarray(0, dtype=np.int64) for i in range(NUMBER_OF_CLIENTS)}

    total_label_freqs = 0

    if DATA_SHARING:
      total_label_freqs = y_train_client.value_counts()
    else:
      total_label_freqs = y_train.value_counts()

    for i in range(3):
      idx_i = np.where(y_test == i)[0]
      np.random.shuffle(idx_i)

      indices = []

      offset = 0

      for j in range(NUMBER_OF_CLIENTS - 1):
        partition_label_proportion = partitioned_train_set[j][1].value_counts().reindex(range(3), fill_value=0)[i] / total_label_freqs[i]
        partition_label_freq = int(len(idx_i )*partition_label_proportion)
        split_index = offset + partition_label_freq
        indices.append(split_index)
        offset = split_index

      split = np.array_split(idx_i, indices)

      for j in range(NUMBER_OF_CLIENTS):
        net_dataidx_map[j]=np.append(net_dataidx_map[j],split[j])

    partitioned_test_set = [(x_test[net_dataidx_map[idx]], pd.Series(y_test[net_dataidx_map[idx]])) for idx in range(NUMBER_OF_CLIENTS)]
  else:
    partitioned_test_set = get_partitions(x_test, y_test)

  return partitioned_test_set
```

## Server Functions

```python
def fit_round(rnd: int) -> Dict:
  """Send fit round number to client."""

  return {"rnd": rnd}

def evaluate_round(rnd: int) -> Dict:
  """Send evaluate round number to client."""

  return {"rnd": rnd}

# get_eval_fn is a closure factory function that creates a closure consisting of the inner function evaluate together with its enclosing environment.
# The closure can modify the mutable objects aka dynamic enclosing state, such as model, within its enclosing state
def get_eval_fn(model: LogisticRegression, y_preds: ListProxy, test_accs: ListProxy):
    """Return an evaluation function for server-side evaluation."""

    # The `evaluate` function will be called after every training round.
    def evaluate(parameters: fl.common.Weights):
        # print("Beginning centralized evaluation...")
        # print("Updating global model with the latest parameters...")
        # Update model with the latest parameters
        set_model_params(model, parameters)
        # print("Predicting class labels for samples in X_test...")
        # Predict class labels for samples in X_test
        predictions = model.predict(X_test)
        y_preds.append(predictions)
        # Calculate probability estimates
        # print("Calculating probability estimates...")
        prob_estimates = model.predict_proba(X_test)
        # Calculate cross-entropy loss
        # print("Calculating cross-entropy loss...")
        loss = log_loss(y_test, prob_estimates)
        # print("Calculating the mean accuracy on the test data and labels...")
        # Calculate the mean accuracy on the test data and labels.
        accuracy = model.score(X_test, y_test)
        test_accs.append(accuracy)
        # print("Finished centralized evaluation.")
        return loss, {"accuracy": accuracy}

    return evaluate
```

```python
def my_aggregate(results: List[Tuple[Weights, int]]) -> Weights:
    """Compute weighted average."""
    # Calculate the total number of examples used during training
    num_examples_total = sum([num_examples for _, num_examples in results])

    # print("Parameters received from each client:")
    # for parameters, num_examples in results:
      # print(f"Num_examples: {num_examples}")
      # print(f"Shape of coeffs: {parameters[0].shape}")
      # print(f"Coeffs: {parameters[0]}")
      # print(f"Shape of biases: {parameters[1].shape}")
      # print(f"Biases: {parameters[1]}")

    # Create a list of weights, each multiplied by the related number of examples
    weighted_parameters = [
        [parameter * num_examples for parameter in parameters] for parameters, num_examples in results
    ]

    # print(f"Weighted parameters: {weighted_parameters}")

    # print(f"Zipped weighted parameters: {list(zip(*weighted_parameters))}")

    # Compute average parameters
    weights_prime = [
        reduce(np.add, layer_updates) / num_examples_total
        for layer_updates in zip(*weighted_parameters)
    ]

    return weights_prime
```

```python
def start_flower_server(y_preds: ListProxy, test_accs: ListProxy):

  if EVAL_STRATEGY == "centralized":
    model = LogisticRegression()
    set_initial_params(model)

# Sample fraction of available clients for the next round
# Minimum number of clients to be sampled for the next round
# Minimum number of clients that need to be connected to the server before a round of federated learning can start. If few clients
# are connected then the server will wait until more connect before sampling clients
# There are two main approaches to evaluate models in flower, centralized (server-side) evaluation and federated (client-side) evaluation.
# Centralized and federated evaluation are mutually exclusive.
# All built-in strategies support centalized evaluation by providing an evaluation function during
# initialization. An evaluation function is any function that can take the current global model parameters as input
# and return evaluation results. The current server implementation calls eval_fn AFTER parameter aggregation
# (training round).
# Customizing the existing FedAvg strategy with the callback function fit_round, bound to eval_fn, to pass variable
# configuration values to the client each training round. The provided function will be called by the strategy
# and must return a dictionary of arbitrary configuration key-value pairs that will be sent to the client
# (received in its own client.fit function).

  class MyClientManager(fl.server.SimpleClientManager):
    def __init__(self) -> None:
      super().__init__()
      random.seed(FRACTION_FIT_RANDOM_SEED)

  client_manager = MyClientManager()

  class MyStrategy(fl.server.strategy.FedAvg):
    def aggregate_fit(self, rnd: int, results: List[Tuple[ClientProxy, FitRes]], failures: List[BaseException]) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
        """Aggregate fit results using weighted average."""
        if not results:
            return None, {}

        # Do not aggregate if there are failures and failures are not accepted
        if not self.accept_failures and failures:
            return None, {}

        for i in range(len(results)):
          client, fit_res = results[i]

        # Convert results
        weights_results = [(parameters_to_weights(fit_res.parameters), fit_res.num_examples)
            for client, fit_res in results
        ]

        average_weights = my_aggregate(weights_results)

        # print(f"Average parameters after aggregation: {average_weights}, number of coeffs: {average_weights[0].shape}, number of biases: {average_weights[1].shape}")

        return weights_to_parameters(average_weights), {}

  strategy = MyStrategy()

  # strategy = fl.server.strategy.FedAvg()

  strategy.min_available_clients = MIN_AVAILABLE_CLIENTS
  strategy.fraction_fit = FRACTION_FIT
  strategy.min_fit_clients = MIN_FIT_CLIENTS
  strategy.on_fit_config_fn = fit_round
  strategy.fraction_eval = FRACTION_EVAL
  strategy.min_eval_clients = MIN_EVAL_CLIENTS

  if (EVAL_STRATEGY == "federated"):
    strategy.on_evaluate_config_fn = evaluate_round
  else:
    strategy.eval_fn = get_eval_fn(model, y_preds, test_accs)

  server = fl.server.Server(client_manager=client_manager, strategy=strategy)

  fl.server.start_server("0.0.0.0:80", server=server, config={"num_rounds": NUMBER_OF_FL_ROUNDS})
```

## Client Functions

```python
def start_flower_client(X_train: np.ndarray, y_train: np.ndarray, X_test: np.ndarray, y_test: np.ndarray, id: str, y_preds: ListProxy, test_accs: ListProxy):
  pid = os.getpid()
  print(f"initializing Client{id} with pid {pid}")

  # Create LogisticRegression Model
  # max_iter: way to limit the number of epochs. One epoch means that each sample in the training dataset has had an opportunity
  # to update the internal model parameters. The solver iterates until convergence (determined by ‘tol’) or this number of iterations.
  # i.e., the learning will be stopped either when the solver iterates max_iter times or when the cost between two iterations changes less than tol,
  # whichever comes first. From sklearn.linear_model.LogisticRegression documentation we can see that ‘tol’ is 0.0001 by default.
  # warm_start: prevent refreshing weights when fitting, this is required for federated learning
  # As a result of the fact that warm_start is required, the variable solver must be changed because liblinear does not support warm start
  # Framework used for multi class classification is multinomial.
  # This is necessary because of the non-iid data in federated learning scenarios; in a multinomial framework, learning 𝛽𝑖 for a particular class i
  # immediately affects our model for other classes j, through the joint model. Consequently, The training algorithm uses log loss aka cross-entropy loss
  model = LogisticRegression(penalty='l2', max_iter=LOCAL_EPOCHS, warm_start=True, class_weight='balanced', C=0.01)

  # Setting initial model parameters
  set_initial_params(model)

  # Define Flower client
  class MyClient(fl.client.NumPyClient):
    def get_parameters(self): # -> List[numpy.ndarray]
      return get_model_parameters(model)

    # Training happens in fit method and can be configured from the server side.
    # Model parameters can also be evaluated during training. Client.fit can return arbitrary evaluation results as a dictionary
    def fit(self, parameters, config):
      # print(f"Client{id} with pid {pid} is sampled in training rnd {config['rnd']}")
      set_model_params(model, parameters)
      # Ignore convergence failure due to low local epochs
      with warnings.catch_warnings():
          warnings.simplefilter("ignore")
          model.fit(X_train, y_train)
      # print(f"Training finished for round {config['rnd']}")
      # print(f"Model params is a 2 tuple, first element contains coefficients/weights, second element contains intercept")
      # print(f"Number of coefficients/weights: {get_model_parameters(model)[0].shape}")
      # print(f"Model params: {get_model_parameters(model)}")
      return get_model_parameters(model), len(X_train), {}

    # Federated evaluation happens in the evaluate method and can be configured from the server side.
    def evaluate(self, parameters, config):
      # print(f"Client{id} with pid {pid} is sampled in evaluation rnd {config['rnd']}")

      # Update model with the latest parameters
      set_model_params(model, parameters)

      # Remind model of classes in the population
      model.classes_ = np.array([0, 1, 2])

      # Predict class labels for samples in X_test
      predictions = model.predict(X_test)

      y_preds.append(predictions)

      # Calculate probability estimates
      prob_estimates = model.predict_proba(X_test)

      # Calculate cross-entropy loss
      loss = log_loss(y_test, prob_estimates, labels=[0,1,2])

      # Calculate the mean accuracy on the test data and labels.
      accuracy = model.score(X_test, y_test)
      test_accs.append(accuracy)

      return loss, len(X_test), {"accuracy": accuracy}

  client = MyClient()

  # Start Flower client
  fl.client.start_numpy_client("0.0.0.0:80", client=client)
```

# Test Bench

## Test Bench Functions

```python
# Define function for getting a time stamp
def get_time_stamp():
  return datetime.today().strftime('%d_%m_%Y__%H_%M_%S')
```

```python
def evaluate_local_models(partitioned_train_set, partitioned_test_set):
  local_models_y_preds = []

  for i in range(NUMBER_OF_CLIENTS):
    (X_train_partition, y_train_partition) = partitioned_train_set[i]
    model = classifier.fit(X_train_partition, y_train_partition)

    (X_test_partition, y_test_partition) = partitioned_test_set[i]
    y_preds = model.predict(X_test_partition)
    local_models_y_preds.append(y_preds)

  return local_models_y_preds
```

```python
# Define function for running simulation with global config
def run_sim(partitioned_train_set, partitioned_test_set=None):
  with Manager() as manager:
    # Holds all child processes
    processes = []

    # List proxy for gathering class labels predicted during centralized evaluation
    centralized_y_preds = manager.list()

    # List proxy for gathering test accuracies from centralized evaluation
    centralized_test_accs = manager.list()

    # Holds all the list proxies for gathering class labels predicted during federated evaluation
    federated_y_preds = []

    # Holds all the list proxies for gathering test accuracies from federated evaluation
    federated_test_accs = []

    # Start Flower server
    server_process = Process(target=start_flower_server, args=(centralized_y_preds, centralized_test_accs))
    server_process.start()
    processes.append(server_process)

    # Block the script here for a second or two so the server has time to start
    time.sleep(2)

    # Start Flower clients
    if EVAL_STRATEGY == "centralized":
      for i in range(NUMBER_OF_CLIENTS):
        y_preds = manager.list()
        test_accs = manager.list()
        (X_train_partition, y_train_partition) = partitioned_train_set[i]
        client_process = Process(target=start_flower_client, args=(X_train_partition, y_train_partition, X_test, y_test, str(i), y_preds, test_accs))
        client_process.start()

        federated_y_preds.append(y_preds)
        federated_test_accs.append(test_accs)
        processes.append(client_process)

        # Block the script here for a second to ensure clients register with server in-order (helps control fraction fit variable)
        time.sleep(1)
    else:
      for i in range(NUMBER_OF_CLIENTS):
        y_preds = manager.list()
        test_accs = manager.list()
        (X_train_partition, y_train_partition) = partitioned_train_set[i]

        (X_test_partition, y_test_partition) = partitioned_test_set[i]

        client_process = Process(target=start_flower_client, args=(X_train_partition, y_train_partition, X_test_partition, y_test_partition, str(i), y_preds, test_accs))
        client_process.start()

        federated_y_preds.append(y_preds)
        federated_test_accs.append(test_accs)
        processes.append(client_process)

        # Block the script here for a second to ensure clients register with server in-order (helps control fraction fit variable)
        time.sleep(1)

    # Wait for processes to finish
    for p in processes:
      p.join()

    # Dereference proxies and return results
    return {"centralized_y_preds": centralized_y_preds[:], "centralized_test_accs": centralized_test_accs[:],
            "federated_y_preds": [y_preds[:] for y_preds in federated_y_preds], "federated_test_accs": [test_accs[:] for test_accs in federated_test_accs]}
```

```python
def get_label_freqs(partitioned_train_set, relative=False):
  label_freqs = [y_train.value_counts(normalize=relative).rename("y_train")]

  for i in range(NUMBER_OF_CLIENTS):
    label_freqs.append(partitioned_train_set[i][1].value_counts(normalize=relative).rename("client" + str(i)))

  label_freqs_table = pd.concat(label_freqs, axis=1).fillna(0)

  if relative:
    return label_freqs_table.astype("float64")
  else:
    return label_freqs_table.astype("int64")

def plot_distribution_of_labels(partitioned_train_set, filepath=None):
  # Resize figure
  plt.figure(figsize=(rcParams["figure.figsize"][0]*1.4, rcParams["figure.figsize"][1]))

  number_of_label_across_partitions = pd.concat([partitioned_train_set[i][1].value_counts().sort_index() for i in range(NUMBER_OF_CLIENTS)], axis=1).fillna(0).astype("int64").to_numpy()

  max_number_of_labels = max([len(partitioned_train_set[i][1]) for i in range(NUMBER_OF_CLIENTS)])

  labs = [str(i) for i in range(NUMBER_OF_CLIENTS)]

  plt.barh(labs, number_of_label_across_partitions[0])
  plt.barh(labs, number_of_label_across_partitions[1], left=number_of_label_across_partitions[0], color='g')
  plt.barh(labs, number_of_label_across_partitions[2], left=number_of_label_across_partitions[0]+number_of_label_across_partitions[1], color='r')

  plt.title("Distribution of Labels")
  plt.xlabel("Number of Local Data Samples")
  plt.ylabel("Client ID")

  labelIDs=['hate speech','offensive','neither']

  plt.legend(labelIDs)

  plt.xlim([0, max_number_of_labels + 300])

  if filepath:
    plt.ioff()
    plt.savefig(filepath, bbox_inches='tight')
    plt.close()
  else:
    plt.show()

def plot_performance_of_local_models(precision_recall_fscores_of_local_models, precision_recall_fscores_of_global_model, classification_metric, filepath=None):
  classification_metrics = {'Precision': 0, 'Recall': 1, 'F1-score': 2}

  labs = [str(i) for i in range(NUMBER_OF_CLIENTS)]

  scaling_parameter = 5.5
  x = np.arange(0, scaling_parameter*len(labs), scaling_parameter)  # the label locations
  width = 2.2  # the width of the bars

  fig, ax = plt.subplots(figsize=(rcParams["figure.figsize"][0]*1.8, rcParams["figure.figsize"][1] * 1.5))
  rects1 = ax.bar(x - width/2, precision_recall_fscores_of_local_models[classification_metrics[classification_metric]], width, label='Local Model')
  rects2 = ax.bar(x + width/2, precision_recall_fscores_of_global_model[classification_metrics[classification_metric]], width, label='Global Model')

  # Add some text for labels, title and custom x-axis tick labels, etc.
  ax.set_ylabel(classification_metric)
  ax.set_xlabel('Client ID')
  ax.set_title(classification_metric + ' of Local Models Against That of Global Model')
  ax.set_xticks(x)
  ax.set_xticklabels(labs)
  ax.legend()

  def autolabel(rects):
    '''
    Copied from:

    Title: Matplotlib
    Authors: John Hunter et al.
    Date: 2012
    Availability: https://matplotlib.org/3.2.2/gallery/lines_bars_and_markers/barchart.html

    '''

    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
      height = round(rect.get_height(), 2)
      ax.annotate('{}'.format(height),
                  xy=(rect.get_x() + rect.get_width() / 2, height),
                  xytext=(0, 3),  # 3 points vertical offset
                  textcoords="offset points",
                  ha='center', va='bottom')

  autolabel(rects1)
  autolabel(rects2)

  plt.ylim([0, 1])

  if filepath:
    plt.ioff()
    plt.savefig(filepath)
    plt.close()
  else:
    plt.show()

def get_measures_of_dispersion(precision_recall_fscores):
  return [np.std(classification_metric) for classification_metric in precision_recall_fscores]

def precision_recall_fscore(y_true, y_pred):
  return precision_recall_fscore_support(y_true, y_pred, average="macro", zero_division=1)[:-1]

def get_precision_recall_fscores_of_local_models(local_models_y_test, local_models_y_preds):
  return list(zip(*[precision_recall_fscore(local_model_y_test, local_model_y_preds) for local_model_y_test, local_model_y_preds in zip(local_models_y_test, local_models_y_preds)]))

def get_mean_precision_recall_fscores_of_local_models(precision_recall_fscores_of_local_models):
  return [mean(metric) for metric in precision_recall_fscores_of_local_models]

def get_utility_measure_on_distributions(aggr_model_precision_recall_fscore, avg_local_model_precision_recall_fscore):
  arrays = [["Precision", "Precision", "Recall", "Recall", "F1-score", "F1-score"],
          ["Aggr.", "Local", "Aggr.", "Local", "Aggr.", "Local"]]
  tuples = list(zip(*arrays))
  index = pd.MultiIndex.from_tuples(tuples, names=["Distribution", ""])

  utility_measure = pd.DataFrame([[round(i, 2) for sub in zip(aggr_model_precision_recall_fscore, avg_local_model_precision_recall_fscore) for i in sub]], index=[DATA_PARTITIONING_STRATEGY], columns=index)
  return utility_measure

def plot_convergence_behaviour_from_federated_evaluation(federated_test_accs, filepath=None):
  # Place legend outside axes
  axes_legend_bbox = (1.03, 0.6)

  # Resize figure
  plt.figure(figsize=(rcParams["figure.figsize"][0] + axes_legend_bbox[0] - 1, rcParams["figure.figsize"][1]))

  plt.plot([mean(test_accs) for test_accs in zip(*federated_test_accs)])
  plt.ylim([0, 1])
  plt.xlabel("Round")
  plt.ylabel("Avg Test Acc")
  plt.title("Convergence Behaviour")
  plt.legend(bbox_to_anchor = axes_legend_bbox)

  if filepath:
    plt.ioff()
    plt.savefig(filepath, bbox_inches='tight')
    plt.close()
  else:
    plt.show()

def plot_convergence_behaviour_from_centralized_evaluation(centralized_test_accs, filepath=None):
  # Place legend outside axes
  axes_legend_bbox = (1.26, 0.6)

  # Resize figure
  plt.figure(figsize=(rcParams["figure.figsize"][0] + axes_legend_bbox[0] - 1, rcParams["figure.figsize"][1]))

  plt.plot(centralized_test_accs, label= 'FedShare' if DATA_SHARING else 'FedAvg')
  plt.ylim([0, 1])
  plt.xlabel("Round")
  plt.ylabel("Test Acc")
  plt.title("Convergence Behaviour")
  plt.legend(bbox_to_anchor = axes_legend_bbox)

  if filepath:
    plt.ioff()
    plt.savefig(filepath, bbox_inches='tight')
    plt.close()
  else:
    plt.show()

def report_classification_macro_averages(y_test, y_preds):
  report = classification_report(y_test, y_preds)
  rows = report.split("\n")
  headings = " ".join(rows[0].split(" ")[:-1])
  macro_averages = " ".join(rows[7].split(" ")[:-1])
  return "\n".join((headings, macro_averages))
```

## Design of Experimental Validation

### Distributed Optimization Problem

Our federated learning system has the finite sum objective with generic form:

\begin{equation}
\min_{w \in \mathbb{R}^d} f(w)
\qquad \text{where} \qquad
f(w) \overset{\text{def}}{=} \frac{1}{n} \sum_{i=1}^n f_i(w).
\end{equation}

where $n$ is the number of data samples and $w$ is the model parameters

We take $f_i(w) = \ell(x_i, y_i; w)$, that is, the log loss of the prediction on sample $(x_i, y_i)$ made with coefficients $w$. We assume there are $K$ clients over which the data is partitioned, with $\mathcal{P}_k$ the set of samples on client $k$, with $n_k = |\mathcal{P}_k|$. Thus, we can re-write the system's objective as:

\begin{equation}
\min_{w \in \mathbb{R}^d} \sum_{k = 1}^K \frac{n_k}{n} F_k(w)
\qquad \text{where} \qquad
F_k(w) \overset{\text{def}}{=} \frac{1}{n_k} \sum_{i \in \mathcal{P}_k} f_i(w).
\end{equation}

If the partition $\mathcal{P}_k$ is formed by distributing the samples over the clients uniformly at random, then the f**IID assumption** made by distributed optimization algorithms holds[(Zhao et al., 2018)](https://arxiv.org/pdf/1806.00582.pdf).

We use the distribution setting where the IID assumption holds as a basline for our experiments.

### Federated Optimization Problem

The optimization problem implicit in federated learning is known as the federated optimization problem, drawing a contrast to the distributed optimization problem. The key properties that differentiate federated optimization are: unbalanced data (different clients can hold different amounts of data) and that the IID assumption does not hold.

Here we focus on horizontal federated learning, where the clients share the same feature space but different sample space (in contrast to vertical federated learning, where the clients have different feature and sample spaces).

An existing study gives a taxonomy of non-IID data distribution cases [(Kairouz et al., 2021)](https://arxiv.org/pdf/1912.04977.pdf). Consider the local data distribution $P(x_i, y_i)$. We can rewrite $P(x_i, y_i)$ as $P(x_i | y_i)P(y_i)$ and $P(y_i | x_i)P(x_i)$ so that we can more precisely show the differences between cases. The study gives 5 different non-IID cases:

(1) label distribution skew (i.e., $P(y_i)$ is different among clients)

(2) feature distribution skew (i.e., $P(x_i)$ is different among clients)

(3) same labels but different features (i.e., $P(x_i|y_i)$ is different among clients)

(4) same features but different labels (i.e., $P(y_i|x_i)$ is different among
clients)

(5) quantity skew (i.e., $P(x_i, y_i)$ is the same but the amount of data is different among clients)

Here, (3) is mainly related to vertical FL (the clients share the same sample IDs but different features). (4) is not applicable in most FL studies, which assume same ground truths . (2) can be simulated on a dataset by adding different levels of noise to different partitions. In computer vision, these transformations can be done on the fly, using data generators e.g. for Gaussian noise. However, this isn't the case with NLP due to the grammatical structure of the text; a new carefully augmented dataset must be generated beforehand [(Shahul ES, 2021)](https://neptune.ai/blog/data-augmentation-nlp).

Thus, we consider (5) quantity skew and (1) label distribution skew as possible non-IID data distribution cases in our experiments.

### Unbalancedness (Quantity skew)

The size of the local dataset, $|{\mathcal{P}}_k|$, varies across clients. Although, the iid assumption still holds. We use a Dirichlet distribution to allocate different amounts of data samples to each client. We sample $q ∼ Dir_N (β)$, and allocate a $q_j$ proportion of the total data samples to ${\mathcal{P}}_j$ . The distribution parameter, $β>0$, can be used to control the degree of imbalance. The smaller $β$ is, the more imbalanced the distribution [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf).

This scenario is relevant as irl some organisations will generate more data than others, leading to varying amounts of local training data.

### Label Distribution Skew: Distribution-based label imbalance

Each client is allocated a proportion of the samples of each label according to a Dirichlet distribution. Dirichlet distribution is an appropriate
choice to simulate imbalanced data distributions in the real-world [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf). Specifically, we sample $p_k ∼ Dir_N(β)$ and allocate a $p_{k,j}$ proportion of the instances of class $k$ to client $j$. We fix to constrain the scope of the study. This partitioning strategy has been used to simulate real world data distributions in many recent FL studies [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf).

### Label Distribution Skew: Quantity-based label imbalance

Each client owns data samples of a fixed subset of total labels. Such a setting is also used in other studies, [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf). We use a general partitioning strategy to set the number of labels that each client has; in the context of our study, this is either 1 or 2.

### Statistical Resampling

We use 10-fold Cross-Validation to sufficiently randomize the train/test split, to reduce sampling error. Further, we select 10 observations from each distribution setting to reduce selection bias. The data partitioning random seed is set to the current observation number. Therefore, we perform 100 repeats of the experiment per distribution setting.

## Test Bench Driver Code

```python
data_partitioning_strategies = ["iid", "iid-quantity-skew", "noniid-distribution-based-label-imbalance", "noniid-quantity-based-label-imbalance-2", "noniid-quantity-based-label-imbalance-1"]
```

```python
logging_folders = ["log_29_03_2022__21_13_51_iid/", "log_29_03_2022__22_02_06_iid-quantity-skew/", "log_29_03_2022__22_02_56_noniid-distribution-based-label-imbalance/", "log_29_03_2022__23_14_51_noniid-quantity-based-label-imbalance-2/", "log_29_03_2022__23_15_21_noniid-quantity-based-label-imbalance-1/"]
```

```python
use_data_sharing = [False, False, False, True, True]
```

```python
classification_metrics = ['precision', 'recall', 'f1-score']
aggregated_measures = ['value', 'std']
```

```python
# Cross-validator
skf = StratifiedKFold(n_splits=NUMBER_OF_VALIDATION_FOLDS, shuffle=True, random_state=42)

current_repetition = 1

for data_partitioning_strategy, logging_folder, data_sharing_flag in zip(data_partitioning_strategies, logging_folders, use_data_sharing):
  # Overrides to global experiment config
  DATA_PARTITIONING_STRATEGY = data_partitioning_strategy
  LOG_DIR = LOGGING_FOLDERS_DIR + logging_folder
  DATA_SHARING = data_sharing_flag

  summed_federated_test_accs = np.zeros(NUMBER_OF_FL_ROUNDS)

  aggregated_classification_macro_averages_of_locally_trained_models_on_local_data = {classification_metric: {aggregated_measure: []
                                                                                for aggregated_measure in aggregated_measures}
                                                                                for classification_metric in classification_metrics}
  aggregated_classification_macro_averages_of_federated_model_on_local_data = {classification_metric: {aggregated_measure: []
                                                                                for aggregated_measure in aggregated_measures}
                                                                               for classification_metric in classification_metrics}

  for random_seed in range(0, NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY):

    DATA_PARTITIONING_RANDOM_SEED = random_seed

    summed_precision_recall_fscores_of_local_models_across_validation_folds = np.zeros((3, NUMBER_OF_CLIENTS))
    summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.zeros((3, NUMBER_OF_CLIENTS))

    for train_index, test_index in skf.split(X_, y):

      print(f'Repetition number {current_repetition}...')

      X_train, X_test = X_[train_index], X_[test_index]
      y_train, y_test = y.iloc[train_index], y.iloc[test_index]

      if DATA_SHARING:
        # Partition train set into client partition and holdout partition G, where β = 2.5%. Initialise with random seed for reproducability.
        X_train_client, X_train_holdout, y_train_client, y_train_holdout = train_test_split(X_train, y_train, random_state=42, test_size=0.025, stratify=y_train)

        # A warm-up model is trained on G
        warm_up_model = classifier.fit(X_train_holdout, y_train_holdout)

        # Create a random α portion of G to be merged with the data of each client, where α is 10%
        X_train_shared, _, y_train_shared, _ = train_test_split(X_train_holdout, y_train_holdout, random_state=42, train_size=0.1, stratify=y_train_holdout)

      # Split train set into NUMBER_OF_CLIENTS partitions.
      partitioned_train_set = get_train_partitions(X_train, y_train.to_numpy())

      # Split test set into NUMBER_OF_CLIENTS partitions.
      partitioned_test_set = get_test_partitions(X_test, y_test.to_numpy(), partitioned_train_set)

      # Evaluate local models
      local_models_y_preds = evaluate_local_models(partitioned_train_set, partitioned_test_set)

      local_models_y_test = [y_test_partition for X_test_partition, y_test_partition in partitioned_test_set]

      precision_recall_fscores_of_local_models = get_precision_recall_fscores_of_local_models(local_models_y_test, local_models_y_preds)

      avg_local_model_precision_recall_fscore = get_mean_precision_recall_fscores_of_local_models(precision_recall_fscores_of_local_models)

      measures_of_dispersion = get_measures_of_dispersion(precision_recall_fscores_of_local_models)

      summed_precision_recall_fscores_of_local_models_across_validation_folds = np.add(summed_precision_recall_fscores_of_local_models_across_validation_folds, precision_recall_fscores_of_local_models)

      for i, classification_metric in enumerate(classification_metrics):
        aggregated_classification_macro_averages_of_locally_trained_models_on_local_data[classification_metric]['value'].append(avg_local_model_precision_recall_fscore[i])
        aggregated_classification_macro_averages_of_locally_trained_models_on_local_data[classification_metric]['std'].append(measures_of_dispersion[i])

      # Run sim with global config
      sim_results = run_sim(partitioned_train_set, partitioned_test_set)

      federated_y_preds = sim_results["federated_y_preds"]

      summed_federated_test_accs = np.add(summed_federated_test_accs, [mean(test_accs) for test_accs in zip(*sim_results["federated_test_accs"])])

      ultimate_precision_recall_fscores_of_federated_model_on_local_data = get_precision_recall_fscores_of_local_models(local_models_y_test, [client[-1] for client in federated_y_preds])

      avg_ultimate_federated_model_precision_recall_fscore = get_mean_precision_recall_fscores_of_local_models(ultimate_precision_recall_fscores_of_federated_model_on_local_data)

      measures_of_dispersion = get_measures_of_dispersion(ultimate_precision_recall_fscores_of_federated_model_on_local_data)

      summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.add(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, ultimate_precision_recall_fscores_of_federated_model_on_local_data)
      np.add(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, ultimate_precision_recall_fscores_of_federated_model_on_local_data)

      for i, classification_metric in enumerate(classification_metrics):
        aggregated_classification_macro_averages_of_federated_model_on_local_data[classification_metric]['value'].append(avg_ultimate_federated_model_precision_recall_fscore[i])
        aggregated_classification_macro_averages_of_federated_model_on_local_data[classification_metric]['std'].append(measures_of_dispersion[i])

      current_repetition += 1

    avg_precision_recall_fscores_of_local_models_across_validation_folds = np.divide(summed_precision_recall_fscores_of_local_models_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)
    avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.divide(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)
    np.divide(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)

    classification_macro_averages_filepath = LOG_DIR + DATA_PARTITIONING_STRATEGY + '_seed_' + str(DATA_PARTITIONING_RANDOM_SEED) + 'ultimate_precision_recall_fscores_of_models_across_10_validation_folds.json'
    json.dump([{"avg_precision_recall_fscores_of_local_models_across_validation_folds": avg_precision_recall_fscores_of_local_models_across_validation_folds.tolist(),
                "stds": np.std(avg_precision_recall_fscores_of_local_models_across_validation_folds, axis=1).tolist(),
                "means": np.mean(avg_precision_recall_fscores_of_local_models_across_validation_folds, axis=1).tolist()},
           {"avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds":
            avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds.tolist(),
            "stds": np.std(avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, axis=1).tolist(),
            "means": np.mean(avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, axis=1).tolist()}], open(classification_macro_averages_filepath, 'w'), indent=4)

  avg_federated_test_accs = np.divide(summed_federated_test_accs, NUMBER_OF_REPEATS)

  avg_classification_macro_averages_of_locally_trained_models_on_local_data = {classification_metric: {aggregated_measure: {'avg': np.mean(averages), '95% CI': [np.percentile(averages, 2.5),
                                                                                np.percentile(averages, 97.5)]}
                                                                                for aggregated_measure, averages in aggregated_measures.items()}
                                                                              for classification_metric, aggregated_measures in
                                                                              aggregated_classification_macro_averages_of_locally_trained_models_on_local_data.items()}

  avg_classification_macro_averages_of_federated_model_on_local_data = {classification_metric: {aggregated_measure: {'avg': np.mean(averages), '95% CI': [np.percentile(averages, 2.5),
                                                                                np.percentile(averages, 97.5)]}
                                                                                for aggregated_measure, averages in aggregated_measures.items()}
                                                                        for classification_metric, aggregated_measures in
                                                                        aggregated_classification_macro_averages_of_federated_model_on_local_data.items()}

  federated_test_accs_filepath = LOG_DIR+f"{DATA_PARTITIONING_STRATEGY}_federated_test_accs.json"
  json.dump({'federated_test_accs': avg_federated_test_accs.tolist()}, open(federated_test_accs_filepath, 'w'), indent=4)

  classification_macro_averages_filepath = LOG_DIR + f'classification_macro_averages_federated_evaluation.json'
  json.dump([{'avg_classification_averages_local_models': avg_classification_macro_averages_of_locally_trained_models_on_local_data},
            {'avg_classification_averages_federated_eval': avg_classification_macro_averages_of_federated_model_on_local_data}], open(classification_macro_averages_filepath, 'w'), indent=4)
```