```python
import numpy as np
import sympy as sp
import svg.path as svg
import matplotlib.pyplot as plt
import xml.dom.minidom as md
import matplotlib.animation as animation

from scipy.fftpack import fft
from scipy.fftpack import ifft
from scipy.integrate import quad_vec
from matplotlib.patches import Circle
from matplotlib.animation import FuncAnimation
from svg.path import Path, Line, Arc, CubicBezier, QuadraticBezier, Close, parse_path
from xml.dom.minidom import parse

%matplotlib notebook
```

```python
# Get SVG file path
filepath = "./MikuCurveTest.svg"
svg_file = md.parse(filepath)

# Use libray to phase SVG file.
rootNode = svg_file.documentElement
svg_path_node = rootNode.getElementsByTagName("path")

# Open with xml and phase
for node in svg_path_node:
    if not node.hasAttribute("d"):
        raise AttributeError()
    else:
        svg_path_temp = node.getAttribute("d")
        break

    svg_path_temp = [str(i) for i in svg_path_temp]

# Remove path with line
curve_path = parse_path(svg_path_temp)
for path in curve_path:
    if type(path) == svg.path.Line:
        curve_path.remove(path)
```

```python
# Complex number get real and imag.
def complexToReal(num: complex):
    x = round(float(num.real), 2)
    y = round(float(num.imag), 2)
    return (x, y)

# Get bezier curve control point from path.
def getControlPoint(path):

    p0 = complexToReal(path.start)
    p1 = complexToReal(path.control1)
    p2 = complexToReal(path.control2)
    p3 = complexToReal(path.end)

    return p0, p1, p2, p3

# Check bezier curve continutiy(optional, debug).
def continuityCheck(curve_path):
    count = 0
    p3_temp = None
    for path in curve_path:

        if type(path) == svg.path.CubicBezier:
            p0, p1, p2, p3 = getControlPoint(path)
        else:
            count += 1
            continue

        if p3_temp != None and p3_temp != p0:
            print(f"At count {count} {path_temp} and {path} is not continuity.")
            return False
        count += 1

        p3_temp, path_temp = p3, path
    return True

# Simplify bezeir Curve as: f(t) = A*t^3 + B*t^2 + C*t + D .
def calcConst(path, axis):

    # B(t) = at^3 + bt^2 + ct + d
    p0, p1, p2, p3 = getControlPoint(path)

    if axis == "x":
        p0, p1, p2, p3 = p0[0], p1[0], p2[0], p3[0]
    elif axis == "y":
        p0, p1, p2, p3 = p0[1], p1[1], p2[1], p3[1]
    elif axis == "xy":
        tp0 = str(p0[0]) + "+" + str(p0[1]) + "j"
        tp1 = str(p1[0]) + "+" + str(p1[1]) + "j"
        tp2 = str(p2[0]) + "+" + str(p2[1]) + "j"
        tp3 = str(p3[0]) + "+" + str(p3[1]) + "j"
        p0, p1, p2, p3 = complex(tp0), complex(tp1), complex(tp2), complex(tp3)
    else:
        raise ValueError("Only support x, y or xy.")

    a = p3 - 3*p2 + 3*p1 - p0
    b = 3*(p2 - 2*p1 +p0)
    c = 3*(p1 - p0)
    d = p0

    return a, b, c, d

# Cubic bezier curve formula function
def cubicBezierCurve(p0, p1, p2, p3, t):

    x = 1 * p0[0] * ((1 - t) ** 3) * (t ** 0) + 3 * p1[0] * ((1 - t) ** 2) * (t ** 1) + 3 * p2[0] * ((1 - t) ** 1) * (t ** 2) + 1 * p3[0] * ((1 - t) ** 0) * (t ** 3)
    y = 1 * p0[1] * ((1 - t) ** 3) * (t ** 0) + 3 * p1[1] * ((1 - t) ** 2) * (t ** 1) + 3 * p2[1] * ((1 - t) ** 1) * (t ** 2) + 1 * p3[1] * ((1 - t) ** 0) * (t ** 3)

    return (x, y)

# Print config function(offset, rotation).
def printCubicBezierCurves(t, path, mirror = None, offset = (0, 0)):

    """
    mirror = None : Primitive function
    mirror = "x"  : Function symmetric about x axis
    mirror = "y"  : Function symmetric about y axis
    mirror = "xy" : Function symmetric about origin point
    """

    p0, p1, p2, p3 = getControlPoint(path)
    x, y = cubicBezierCurve(p0, p1, p2, p3, t)

    if mirror == None: return (x + offset[0], y + offset[1])
    elif mirror == "x": return (-x + offset[0], y + offset[1])
    elif mirror == "y": return (x + offset[0], -y + offset[1])
    elif mirror == "xy": return (-x + offset[0], -y + offset[1])
```

```python
# Sampling rate of single part of all bezier curve
SAMPLING_RATE = 100

plt.figure(figsize = (10, 10))
plt.title("SVG File Image", fontsize = 20)
plt.tick_params(labelsize = 20)

np_point_list = []
curve_const_list, curve_const_list_x, curve_const_list_y = [], [], []
point_list_x, point_list_y, point_list = [], [], []
t = np.linspace(0, 1, SAMPLING_RATE)

# Discretization curve to point and save.
for path in curve_path:
    if type(path) == svg.path.Move:
        pass

    elif type(path) == svg.path.Line:
        pass

    elif type(path) == svg.path.CubicBezier:
        x, y = printCubicBezierCurves(t, path, "y")

        for i in range(len(x)):
            point_list_x.append(x[i])
            point_list_y.append(y[i])

        plt.plot(x, y)

    elif type(path) == svg.path.Close:
        break

    else:
        raise TypeError(f"Unexpected type: {type(path)}")

array_x, array_y = np.array(point_list_x), np.array(point_list_y)
plt.show()

curve_count = int(sum([type(i) == svg.path.CubicBezier for i in curve_path]))
print(f"Total {curve_count} Cubic Bezier Curves.")
```

    <IPython.core.display.Javascript object>

    Total 368 Cubic Bezier Curves.

```python
# Circle count = CIRCLE from [-order] to [order]
#        with length = 2 * order + 1
order = 500
array_x = array_x - np.mean(array_x)
array_y = array_y - np.mean(array_y)
array_t = np.linspace(0, 2*np.pi, len(array_x))
```

```python
# Calulate the parameter C

C = []

def f(t, array_t, array_x, array_y):
    return np.interp(t, array_t, array_x + 1j*array_y)

for n in range(-order, order + 1):
    coef = 1/(2*np.pi) * quad_vec(lambda t: f(t, array_t, array_x, array_y)*np.exp(-n*t*1j), 0, 2*np.pi, limit=100, full_output=1)[0]
    C.append(coef)

np.save('/C.npy', np.array(C))
```

```python
# Sort C with index 0, 1, -1, 2, -2, 3, -3, ...
def sort_coeff(coeffs):
    new_coeffs, index = [], []
    new_coeffs.append(coeffs[order])
    index.append(0)
    for i in range(1, order+1):
        new_coeffs.extend([coeffs[order+i],coeffs[order-i]])
        index.extend([i,-i])
    return new_coeffs, index

coeffs_list, lable = sort_coeff(C)
coeffs = np.array(coeffs_list)

x_coeffs = np.real(coeffs)
y_coeffs = np.imag(coeffs)
```

```python
def getCircle(start, end, sr = 100):
    a, b, c, d = start[0], start[1], end[0], end[1]
    r = ((a - c)**2 + (b - d)**2) ** 0.5
    theta = np.linspace(0, 2 * np.pi, sr)
    x = r * np.cos(theta) + a
    y = r * np.sin(theta) + b
    return x, y

def drawCircle(start, end, sr = 100):
    x, y = getCircle(start, end, sr)
    return plt.plot(x, y, color="blue", linewidth = 1, alpha = 0.3)

def getLine(start, end):
    line_x = np.linspace(start[0], end[0], 100)
    line_y = np.linspace(start[1], end[1], 100)
    return line_x, line_y

def drawLine(start, end):
    x, y = getLine(start, end)
    return plt.plot(x, y)

def calcRadius(t_index, t_list = array_t, para_C = coeffs, lable = np.array(lable)):
    t = t_list[t_index]
    vector = para_C * np.exp(lable * 1j * t)
    vector_x, vector_y = vector.real, vector.imag
    return (vector_x, vector_y)
```

```python
# Calculate vector
temp_vector_list = []
vector = []
for i in range(len(array_t)):
    temp_vector_list.append(calcRadius(i))
temp_vector = np.array(temp_vector_list)

for i in range(len(temp_vector)):
    vector.append(temp_vector[i].T)
vector = np.array(vector)
```

```python
# Get FFT fit result(sum all vector)
vector_x = vector[:, :, 0]
vector_y = vector[:, :, 1]
x_sum = np.sum(vector_x, axis = 1, keepdims = True)
y_sum = np.sum(vector_y, axis = 1, keepdims = True)
```

```python
# Print FFT fit result
plt.figure(figsize=(10, 10))
plt.plot(x_sum, y_sum)
plt.show()
```

    <IPython.core.display.Javascript object>

```python
# Calulate vector into point set.
point_vector = np.zeros(vector.shape)

for t in range(array_t.shape[0]):
    temp_sum_x, temp_sum_y = 0.0, 0.0
    for i in range(vector[t, :, :].shape[0]):
        point_vector[t][i] = [temp_sum_x, temp_sum_y]
        temp_sum_x += vector[t][i][0]
        temp_sum_y += vector[t][i][1]
```

```python
# Create start-end tuple from point set.
def getPointList(t):
    pointList = []
    tempList = point_vector[t]
    for i in range(len(tempList) - 1):
        pointList.append([tempList[i], tempList[i+1]])
    return pointList

# Calculate animate xy-limit.
def calcXYlimit(array):
    squ_max = abs(float(np.max(array)))
    squ_min = abs(float(np.min(array)))
    squ_lim = max(squ_max, squ_min)
    return (squ_lim // 50 + 1) * 50
```

```python
# Create Animation with plt
fig, ax = plt.subplots(figsize=(6,6))

lines, circles = [], []
point_x, point_y = [], []

lim = calcXYlimit(np.array([x_sum, y_sum]))
ax.set(xlim = [-lim, lim], ylim = [-lim, lim])

def initalize():
    start_end_list = getPointList(0)

    global scatter
    scatter = ax.plot([], [])[0]

    for i in range(len(start_end_list)):
        lines.append(ax.plot([], [], alpha=0.7)[0])
        circles.append(ax.plot([], [], alpha=0.7)[0])

def update(data):
    start_end_list = getPointList(data)

    for i in range(len(start_end_list)):

        line_x, line_y = getLine(start_end_list[i][0], start_end_list[i][1])
        lines[i].set_data(line_x, line_y)

        circle_x, circle_y = getCircle(start_end_list[i][0], start_end_list[i][1])
        circles[i].set_data(circle_x, circle_y)

    point_x = x_sum[0 : data, 0]
    point_y = y_sum[0 : data, 0]

    scatter.set_data(point_x, point_y)

ani = FuncAnimation(fig, update, init_func = initalize, frames = range(len(x_sum)), interval = 1)
ani.save("movie.mp4", fps = 120, writer="ffmpeg")
# ani.save('animation.gif', writer='imagemagick', fps = 20, )
plt.show()
```

    <IPython.core.display.Javascript object>


# Text Message Spam Detection

## The Business Use Case

You are the CEO of a new email service company trying to attract capital for your next growth stage. Most of the private equity firms you have spoken to want to see a user base of at least 100,000 users prior to commiting. Due to your superb marketing team, you estimate that each day you attract 1000 new users. Typically, 10% of all email messages are spam, and an average user receives 50 emails per day.

One of your data scientists says he can provide you a model with **90% accuracy** in classifying spam / ham. Another data scientist says she can build a model that is only **80% accurate** but has **100% recall**. A third data scientist says he can build a model that has **80% accuracy** with **100% precision**.

Which of the above models would you pick to model? Why?

```python
import pandas as pd
data = pd.read_csv("spam-sms.csv", encoding='latin-1')
data.shape
```

```python
Y = data["class"].values
X = data["text"].values
```

# Bayes Rule

$$
\begin{equation}
P(A|B) = \frac{P(B|A)P(A)}{P(B)}
\end{equation}
$$

For our purposes, we will redefine this as

$$
\begin{equation}
P(spam|text) = \frac{P(text|spam)P(spam)}{P(text)}
\end{equation}
$$

Here,

- **prior** means before seeing any new text (evidence). Our impression of the likelihood of certain words appearing before new evidence is introduced.
- **text** is the new message (evidence) being introduced that we want to classify as either spam or ham. Let's say we have a new text message `When are you coming home? I'm hungry.`.

- $P(text)$ is the **prior likelihood** of seeing a particular text message with that exact combination of words. For instance, `P("the car is")` will be significantly higher than `P("Downstream supply chain agents")`, especially in the **context of text messages**.
- $P(spam)$ is the likelihood that any text message will be spam. This is computed in our dataset:
```python
p_spam = sum(data["class"] == "spam") / len(data)
p_ham = 1 - p_spam # since there are only two classes
```
- $P(text|spam)$ is our **likelihood**. More specifically, the likelihood of this text message given that it is a piece of spam. It is saying, *let's assume that this message is spam. Knowing that, how likely is it that we'll find this particular combination of words in the text message?*

In order to quickly get our likelihoods, we'll need to create a **likelihood table**:

```python
spam_data = data[data["class"] == "spam"]
ham_data = data[data["class"] == "ham"]
print(f"The shape of the spam_data is {spam_data.shape}. The shape of the ham_data is {ham_data.shape}.")
```

```python
from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer()
vectorizer.fit(data["text"].values)

# create the vocabulary list
vocabulary = set(vectorizer.get_feature_names())
```

```python
likelihood_table
```

```python
# Create the vocabulary list
import spacy, string
# nlp = spacy.load('en') <-- if spacy is giv
from nltk import word_tokenize

likelihood_table = pd.DataFrame(columns=["spam", "ham"], index=list(vocabulary)).fillna(0)

# populate the spam column in our likelihood table
for i, sentence in enumerate(spam_data["text"].values):
    for token in word_tokenize(sentence):
        if token.lower() not in likelihood_table.index:
            likelihood_table.loc[token.lower(), "spam"] = 1
        else:
            likelihood_table.loc[token.lower(), "spam"] += 1
likelihood_table.fillna(0, inplace=True)
```

```python
# populate the spam column in our likelihood table
for i, sentence in enumerate(ham_data["text"].values):
    for token in nlp(sentence):
        if token.text.lower() not in likelihood_table.index:
            likelihood_table.loc[token.text.lower(), "ham"] = 1
        else:
            likelihood_table.loc[token.text.lower(), "ham"] += 1
likelihood_table.fillna(0, inplace=True)
```

```python
likelihood_table.head()
```

What about words like `won`?
```python
likelihood_table.loc["won"]
```
Output:
```
spam    73.0
ham      0.0
Name: won, dtype: float64
```

What is $P(w = won|c = ham)$? If even one of the words' class-conditional probabilities is 0, then the entire likelihood will be zero, since the likelihood is simply the product of all the words' individual likelihoods.

# Additive Smoothing Techniques

We can define a new likelihood for the word `won`:

$$
\begin{equation}
P_{new}(w = won | c = ham) = \frac{N_{ham, won} + \alpha}{N_{ham} + \alpha d}
\end{equation}
$$

Here, $N_{ham, won}$ is the number of times `won` appears in a text message that is classified as ham, and $N_{ham}$ is simply the total number of messages that are classified as ham.

```python

```

## Simple Optimizations to Improve Naive Bayes Probabilistic Models for Text Classification

- to may be useful to simply create a simple **co-occurence matrix**, and **run a correlation analysis** on the features (words). If certain words have extremely high correlations, you may wish to take them out, or fuse them into a single entity.
- apply smoothing techniques to handle **out-of-vocabulary test words**
- **ensemble techniques like bagging / boosting** do **not** help. There isn't any "variation" in a Naive Bayes model. Given the same trained corpus $C$, and a new text message $m$, a Naive Bayes model will always output the same prediction.

# Representing Words as Probabilities

We can represent a sentence (a sequence of words) mathematically as

$$
\begin{equation}
w = \{{w_0, w_1, w_2, \dots,w_{s-1}}\}
\end{equation}
$$

Here, **$s$** represents the total number of words in the sentence. **$w_{0}$** represents the first word in the sentence, **$w_{1}$** represents the second word in the sentence, and so on.

# Exercise:

`Older people, like everyone else, can benefit from accessing ride-sharing, but many are not comfortable with smart-phones.`

You can ignore punctuation and capitalization for now.

1. What is $s$?
2. What is $w_4$? What is $w_6$?
3. What is $V$ (this corpus' vocabulary size, assuming this is the only sentence in the corpus)? You can do this the hard way, by counting manually.

```python
sentence = "Older people, like everyone else, can benefit from accessing ride-sharing, but many are not comfortable with smart-phones"

import re # the most efficient, concise way (less readable)
vocabulary = set([re.sub(r'[^\w\s]','',word).lower() for word in sentence.split()])
print("The size of the vocabulary is {} words".format(vocabulary))
```

# Independence

In statistics, two events are independent if the outcome of one event does not affect the probability of the outcomes of another event.

You will also often see this written as
$$
\begin{equation}
P(A,B) = P(A) * P(B)
\end{equation}
$$

In other words, an event A is independent of event B if the **probability of event A and event B happening together** is equal to **the probability of event A multplied by the probability of event B**.

# Bigram Model

A bigram is a group of two tokens (frequently words) that are treated as one distinct entity. For instance, the distinct bigrams in the sentence `I am home now` would be
```python
bigrams = [
    ("I", "am"),
    ("am", "home"),
    ("home", "now")
]
```

### Exercise:
Write a Python function to find all the bigrams in the sentence
`In recent years, Johnson & Johnson has been focusing more on its high-margin pharmaceutical segment via acquisitions.`

**Hints**:
- split the sentence into a list of individual words (`my_sentence.split()`)
- remove punctuation
- lowercase all the letters
- use a for loop to iterate through this list, getting the **i-th** and **i + 1-th** elements of the list

**Challenge**:
Generalize this function to work with `n-grams`.

## Language Model

Are words in a sentence conditionally independent from each other? In other words, does knowing that the first word `The` change your belief in the likelihood of the second word that follows?

Which of the following sentences is more likely?

```python
sentence_A = "Jack went to Wal-Mart."
sentence_B = "at and the be of I"
```
Notice that all the words in sentence B come from [Wikipedia most common words in the English language](https://en.wikipedia.org/wiki/Most_common_words_in_English). Yet we intuitively know that the sentence is nonsensical and is unlikely to be seen in natural language.

We can express the likelihood of a sentence $w$ as $p(w)$, and define it as
$$
\begin{equation}
p(w) = \prod_{i=0}^{s}p(w_{i+1}|w_{i})
\end{equation}
$$

If we want to generalize this to an **N-Gram** model:

$$
\begin{equation}
p(w) = \prod_{i=0}^{s}p(w_i |w_{i-n+1}, w_{i-n+2}, \dots, w_{i})
\end{equation}
$$
$$
\begin{equation}
p(w) = \prod_{i=0}^{s}p(w_i | w_{i-n}^{i})
\end{equation}
$$

Here, $w_{i-n}^{i} = w_{i-n+1}, w_{i-n+2}, \dots, w_{i}$.

## Model Evaluation: Choosing n in an n-Gram Model

- the larger the dataset, and by implication, the more rich the corpus, the larger the n we can likely try.
- in practice, $n = 2$, $n = 3$, $n = 4$ work well. A larger $n$ tends to begin to overfit (and may be computationally extremely expensive). Remember the **bias-variance** tradeoff:

Here, as $n \rightarrow \infty$, model complexity increases dramatically.
- **tune $n$ based on the performance of the downstream model**: usually n-gram models are the first step in a broader sentiment analysis prediction model, or topic modelling model, recommendation system, or sequence-to-sequence translation task.

### Perplexity

Look again at the definition of likelihood for a particular sentence:

$$
\begin{equation}
L = p(w) = \prod_{i=0}^{s}p(w_i | w_{i-n}^{i})
\end{equation}
$$

Here, $w_{i-n}^{i}$ stands for $w_{i-n}, w_{i-n+1}, ..., w_{i-1}$ (if you have bi-gram (`n=2`) model, then you would have $w_{i-2}, w_{i-1}$)

Is it reasonable to compare two sentences, one with $s=4$ (sentence length of 4 words) with the one below in Sentence B?

##### Sentence A:
> *I love to eat.*

##### Sentence B:
> *My escort was an exceptionally genial sixty-seven-year-old man named Don Seely, an electrical engineer who said that he was between jobs and using the unwanted free time to volunteer his services to the Northern Kentucky Tea Party, the rally’s host organization, as a Webmaster.*

Answer: **No**. A common way of quantifying the likelihood of your n-gram models, accounting for different sizes of test corpuses, is to use **perplexity**. Remember that our likelihood of seeing a particular sentence is

$$
\begin{equation}
P = \frac{1}{\sqrt[N]{p(w)}}
\end{equation}
$$
$N$ is the length of all the words in the test sentence. We typically use perplexity, instead of simply likelihood, as the overall model evaluation metric, because in general, **in order to compare two different models**, they should be using the same test corpus / vocabulary.

```python
import numpy as np
import matplotlib.pyplot as plt

def perplexity(likelihood, N):
    return 1 / (likelihood ** (1/N))

x = []
y = []
for i in range(5, 500):
    x.append(i)
    y.append(perplexity(likelihood=.204, N=i)) # an example likelihood of .004

plt.plot(x, y)
plt.xlabel("N")
plt.ylabel("Perplexity")
plt.show()
```

## Dealing with Out-of-Vocabulary Words

Let's pretend that our training corpus is

> *This is mistaken logic. It is true that a high variance and low bias model can preform well in some sense.*

Our test corpus is

> *This **is not** true.*

Assuming a bi-gram model is used, what is the **perplexity** of our model?

We don't actually need to count each bi-gram. **The answer is infinity**. Why?
$
\begin{equation}
p(w_i = not | w_{i-1} = is) = 0
\end{equation}
$

What you can do instead:

* Look at the **frequency distribution** of words in your corpus
* Decide upon some **threshold cutoff**, where every word below that threshold frequency will be converted into an `UNKNOWN` token. Now, whenever a new word appears that is out of vocabulary, you simply convert it into `UNKNOWN` and run the tests as usual.

week3/Probabilities and N-Gram Language Models.ipynb


# Realization of Non-Recursive Filters

*This jupyter notebook is part of a [collection of notebooks](../index.ipynb) on various topics of Digital Signal Processing. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## Segmented Convolution

In many applications one of the signals of a convolution is much longer than the other. For instance when filtering a speech signal $x_L[k]$ of length $L$ with a room impulse response $h_N[k]$ of length $N \ll L$. In such cases the [fast convolution](fast_convolution.ipynb), as introduced before, does not bring a benefit since both signals have to be zero-padded to a total length of at least $N+L-1$. Applying the fast convolution may then even be impossible in terms of memory requirements or overall delay. The filtering of a signal which is captured in real-time is also not possible by the fast convolution.

In order to overcome these limitations, various techniques have been developed that perform the filtering on limited portions of the signals. These portions are known as partitions, segments or blocks. The respective algorithms are termed as *segmented* or *block-based* algorithms. The following section introduces two techniques for the segmented convolution of signals. The basic concept of these is to divide the convolution $y[k] = x_L[k] * h_N[k]$ into multiple convolutions operating on (overlapping) segments of the signal $x_L[k]$.

### Overlap-Add Algorithm

The [overlap-add algorithm](https://en.wikipedia.org/wiki/Overlap%E2%80%93add_method) is based on splitting the signal $x_L[k]$ into non-overlapping segments $x_p[k]$ of length $P$

\begin{equation}
x_L[k] = \sum_{p = 0}^{L/P - 1} x_p[k - p \cdot P]
\end{equation}

where the segments $x_p[k]$ are defined as

\begin{equation}
x_p[k] = \begin{cases} x_L[k + p \cdot P] & \text{ for } k=0,1,\dots,P-1 \\ 0 & \text{ otherwise} \end{cases}
\end{equation}

Note that $x_L[k]$ might have to be zero-padded so that its total length is a multiple of the segment length $P$. Introducing the segmentation of $x_L[k]$ into the convolution yields

\begin{align}
y[k] &= x_L[k] * h_N[k] \\
&= \sum_{p = 0}^{L/P - 1} x_p[k - p \cdot P] * h_N[k] \\
&= \sum_{p = 0}^{L/P - 1} y_p[k - p \cdot P]
\end{align}

where $y_p[k] = x_p[k] * h_N[k]$. This result states that the convolution of $x_L[k] * h_N[k]$ can be split into a series of convolutions $y_p[k]$ operating on the samples of one segment only. The length of $y_p[k]$ is $N+P-1$. The result of the overall convolution is given by summing up the results from the segments shifted by multiples of the segment length $P$. This can be interpreted as an overlapped superposition of the results from the segments, as illustrated in the following diagram

The overall procedure is denoted by the name *overlap-add* technique. The convolutions $y_p[k] = x_p[k] * h_N[k]$ can be realized efficiently by the [fast convolution](fast_convolution.ipynb) using zero-padding and fast Fourier transformations (FFTs) of length $M \geq P+N-1$.

A drawback of the overlap-add technique is that the next input segment is required to compute the result for the actual segment of the output. For real-time applications this introduces an algorithmic delay of one segment.

#### Example

The following example illustrates the overlap-add algorithm by showing the (convolved) segments and the overall result.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sig

L = 64  # length of input signal
N = 8  # length of impulse response
P = 16  # length of segments

# generate input signal
x = sig.triang(L)
# generate impulse response
h = sig.triang(N)

# overlap-add convolution
xp = np.zeros((L//P, P))
yp = np.zeros((L//P, N+P-1))
y = np.zeros(L+P-1)
for p in range(L//P):
    xp[p, :] = x[p*P:(p+1)*P]
    yp[p, :] = np.convolve(xp[p,:], h, mode='full')
    y[p*P:(p+1)*P+N-1] += yp[p, :]
y = y[0:N+L]

# plot signals
plt.figure(figsize = (10,2))

plt.subplot(121)
plt.stem(x)
for n in np.arange(L//P)[::2]:
    plt.axvspan(n*P, (n+1)*P-1, facecolor='g', alpha=0.5)
plt.title(r'Signal $x[k]$ and segments')
plt.xlabel(r'$k$')
plt.ylabel(r'$x[k]$')
plt.axis([0, L, 0, 1])

plt.subplot(122)
plt.stem(h)
plt.title(r'Impulse response $h[k]$')
plt.xlabel(r'$k$')
plt.ylabel(r'$h[k]$')
plt.axis([0, L, 0, 1])

for p in np.arange(L//P):
    plt.figure(figsize = (10,2))

    plt.stem(np.concatenate((np.zeros(p*P), yp[p, :])))
    plt.title(r'Result of segment $p=%d$' %(p))
    plt.xlabel(r'$k$')
    plt.ylabel(r'$y_%d[k - %d P]$' %(p,p))
    plt.axis([0, L+P, 0, 4])

plt.figure(figsize = (10,2))
plt.stem(y)
plt.title(r'Result $y[k] = x[k] * h[k]$')
plt.xlabel(r'$k$')
plt.ylabel(r'$y[k]$')
plt.axis([0, L+P, 0, 4]);
```

**Exercises**

* Change the length `N` of the impulse response and the length `P` of the segments. What changes?
* What influence have these two lengths on the numerical complexity of the overlap-add algorithm?

Solution: The parameters `N` and `P` influence the overlap in the output and the total number of segments. The number of overlapping samples of two consecutive output segments $y_p[k]$ and $y_{p+1}[k]$ is given as $N-1$, and the total number of segments as $\frac{L}{P}$. The segmented convolution requires $\frac{L}{P}$ linear convolutions of length $P+N-1$ each. The numerical complexity is mainly determined by the overall number of multiplications which is given as $\frac{L}{P} (P+N-1)^2$. For fixed $L$ and $N$, the optimum segment length is computed by finding the minimum in terms of multiplications. It is given as $P=N-1$.

### Overlap-Save Algorithm

The [overlap-save](https://en.wikipedia.org/wiki/Overlap%E2%80%93save_method) algorithm, also known as *overlap-discard algorithm*, follows a different strategy as the overlap-add technique introduced above. It is based on an overlapping segmentation of the input $x_L[k]$ and application of the periodic convolution for the individual segments.

Lets take a closer look at the result of the periodic convolution $x_p[k] \circledast h_N[k]$, where $x_p[k]$ denotes a segment of length $P$ of the input signal and $h_N[k]$ the impulse response of length $N$. The result of a linear convolution $x_p[k]* h_N[k]$ would be of length $P + N -1$. The result of the periodic convolution of period $P$ for $P > N$ would suffer from a circular shift (time aliasing) and superposition of the last $N-1$ samples to the beginning. Hence, the first $N-1$ samples are not equal to the result of the linear convolution. However, the remaining $P- N + 1$ do so.

This motivates to split the input signal $x_L[k]$ into overlapping segments of length $P$ where the $p$-th segment overlaps its preceding $(p-1)$-th segment by $N-1$ samples

\begin{equation}
x_p[k] = \begin{cases}
x_L[k + p \cdot (P-N+1) - (N-1)] & \text{ for } k=0,1, \dots, P-1 \\
0 & \text{ otherwise}
\end{cases}
\end{equation}

The part of the circular convolution $x_p[k] \circledast h_N[k]$ of one segment $x_p[k]$ with the impulse response $h_N[k]$ that is equal to the linear convolution of both is given as

\begin{equation}
y_p[k] = \begin{cases}
x_p[k] \circledast h_N[k] & \text{ for } k=N-1, N, \dots, P-1 \\
0 & \text{ otherwise}
\end{cases}
\end{equation}

The output $y[k]$ is simply the concatenation of the $y_p[k]$

\begin{equation}
y[k] = \sum_{p=0}^{\frac{L}{P-N+1} - 1} y_p[k - p \cdot (P-N+1) + (N-1)]
\end{equation}

The overlap-save algorithm is illustrated in the following diagram

For the first segment $x_0[k]$, $N-1$ zeros have to be appended to the beginning of the input signal $x_L[k]$ for the overlapped segmentation. From the result of the periodic convolution $x_p[k] \circledast h_N[k]$ the first $N-1$ samples are discarded, the remaining $P - N + 1$ are copied to the output $y[k]$. This is indicated by the alternative notation *overlap-discard* used for the technique. The periodic convolution can be realized efficiently by a FFT/IFFT of length $P$.

#### Example

The following example illustrates the overlap-save algorithm by showing the results of the periodic convolutions of the segments. The discarded parts are indicated by the red background.

```python
L = 64  # length of input signal
N = 8  # length of impulse response
P = 24  # length of segments

# generate input signal
x = sig.triang(L)
# generate impulse response
h = sig.triang(N)

# overlap-save convolution
nseg = (L+N-1)//(P-N+1) + 1
x = np.concatenate((np.zeros(N-1), x, np.zeros(P)))
xp = np.zeros((nseg, P))
yp = np.zeros((nseg, P))
y = np.zeros(nseg*(P-N+1))

for p in range(nseg):
    xp[p, :] = x[p*(P-N+1):p*(P-N+1)+P]
    yp[p, :] = np.fft.irfft(np.fft.rfft(xp[p, :]) * np.fft.rfft(h, P))
    y[p*(P-N+1):p*(P-N+1)+P-N+1] = yp[p, N-1:]
y = y[0:N+L]

plt.figure(figsize = (10,2))

plt.subplot(121)
plt.stem(x[N-1:])
plt.title(r'Signal $x[k]$')
plt.xlabel(r'$k$')
plt.ylabel(r'$x[k]$')
plt.axis([0, L, 0, 1])

plt.subplot(122)
plt.stem(h)
plt.title(r'Impulse response $h[k]$')
plt.xlabel(r'$k$')
plt.ylabel(r'$h[k]$')
plt.axis([0, L, 0, 1])

for p in np.arange(nseg):
    plt.figure(figsize = (10,2))
    plt.stem(yp[p, :])
    plt.axvspan(0, N-1+.5, facecolor='r', alpha=0.5)
    plt.title(r'Result of periodic convolution of $x_%d[k]$ and $h_N[k]$' %(p))
    plt.xlabel(r'$k$')
    plt.axis([0, L+P, 0, 4])

plt.figure(figsize = (10,2))
plt.stem(y)
plt.title(r'Result $y[k] = x[k] * h[k]$')
plt.xlabel(r'$k$')
plt.ylabel(r'$y[k]$')
plt.axis([0, L+P, 0, 4]);
```

**Exercise**

* Change the length `N` of the impulse response and the length `P` of the segments. What changes?
* How many samples of the output signal $y[k]$ are computed per segment for a particular choice of these two values?
* What would be a good choice for the segment length `P` with respect to the length `N` of the impulse response?

Solution: Decreasing the segment length $P$ or increasing the length of the impulse response $N$ decreases the number of valid output samples per segment which is given as $P-N+1$. The computation of $L$ output samples requires $\frac{L}{P-N+1}$ cyclic convolutions of length $P$ each. Regarding the total number of multiplications, an optimal choice for the segment length is $P = 2 N - 2$.

### Practical Aspects and Extensions

* For both the overlap-add and overlap-save algorithm the length $P$ of the segments influences the lengths of the convolutions, FFTs and the number of output samples per segment. The segment length is often chosen as

    * $P=N$ for overlap-add and
    * $P = 2 N$ for overlap-save.

For both algorithms this requires FFTs of length $2 N$ to compute $P$ output samples. The overlap-add algorithm requires $P$ additional additions per segment in comparison to overlap-save.

* For real-valued signals $x_L[k]$ and impulse responses $h_N[k]$ real-valued FFTs lower the computational complexity significantly. As alternative, the $2 N$ samples in the FFT can be distributed into the real and complex part of a FFT of length $N$ [[Zölzer](../index.ipynb#Literature)].

* The impulse response can be changed in each segment in order to simulate time-variant linear systems. This is often combined with an overlapping computation of the output in order to avoid artifacts due to instationarities.

* For long impulse responses $h_N[k]$ or low-delay applications, algorithms have been developed which base on an additional segmentation of the impulse response. This is known as *partitioned convolution*.

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Digital Signal Processing - Lecture notes featuring computational examples, 2016-2018*.


Handwritten digit classification using multivariate linear regression.

  Robert Martin
  July 3, 2020

In this notebook we develop a least squares model that will classify handwritten digits. This is a task more suited to convolutional neural networks. In fact such networks have been shown to have a 99% accuracy rate. Our aim is to demonstrate that this classification can be accomplished using a simple least squares model*. Also this exercise will afford us a better understanding of the least squares method of building models. We will demonstrate that an accuracy rate of near 90% can be achieved using this method.

More on terminology.

Simple linear regression involves finding a linear equation or linear regression model with a single independent variable and a single dependent variable. A multiple regression model is one that attempts to predict a dependent variable  based on the value of multiple independent variables. For example we could attempt to predict the selling price of a house based on the number of bedrooms and square footage of the house. This is an example of two independent variables. Multivariate or multi target regression is the term used when there are multiple dependent variables or outputs. For this exercise the set of dependent variables is the set of 784 pixel values for the digit image. There are 10 target variables, namely the 10 digits. As the target variables are categorical (is the image digit a given digit value, yes or no) we are performing multi-target classification here.

```python
# Libraries

import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
%config InlineBackend.figure_format ='retina'
import pandas as pd
```

Load the data

We use a dataset of 60,000 small square 28×28 pixel grayscale images of handwritten single digits between 0 and 9 to train the model. A dataset of 10,000 images will be used in testing the model. The MNIST dataset is used.  https://en.wikipedia.org/wiki/MNIST_database

```python
pwd
```

    '/Users/rmartin/Dropbox/MyPython/notebooks'

```python
# load the data
training_df = pd.read_csv('mnist_train.csv.gz',header = None,compression='gzip')
test_df = pd.read_csv('mnist_test.csv.gz',header = None,compression='gzip')
training_data = training_df.values
test_data = test_df.values
```

```python
(training_data.shape , test_data.shape)
```

    ((60000, 785), (10000, 785))

We see above that training_data is a 60,000 x 785 matrix.  Each row has 784 pixel values for the image and one value for the label.  The labels are stored in the first entry of each row.  We unpack this data below.

```python
# unpack the training images and symbols
Xtest, labelstest = (test_data[:,1:],test_data[:,0])
X , labels = (training_data[:,1:],training_data[:,0])
(X.shape,labels.shape)
```

    ((60000, 784), (60000,))

From above we see the matrix "X" contains 60,000 images of (784 = 28 x 28) pixels. The array, "labels" contains the 60,000 labels or target values for each image.

Let us display the digit corresponding row 5 in X.

```python
digit = X[5].reshape(28,28)
plt.imshow(digit,cmap='gray')
plt.grid()
```

```python
# We see the digit "2" let us check the label and see if it matches the image
print(labels[5])  # yes it does see the "2" below
```

    2

The linear regression model is shown below.  $X$ is the design matrix. The rows of $X$ are our inputs or "feature vectors". $\vec{\mathbf{\beta}}$ is the parameter vector, what we are solving for. $\vec{\mathbf{y}}$ is the observation vector containing the labels for each input

\begin{align}
X  \vec{\mathbf{\beta}}  = \vec{\mathbf{y}}   \\
\end{align}

This system is overdetermined containing 60,000 equation and 785 unknown parameters. We will solve by using the standard normal equations used in least squares.

\begin{align}
\vec{\mathbf{\beta}}  =   (X^T X)^{-1} X^T  \vec{\mathbf{y}} \\
\end{align}

The vector of observations $\vec{\mathbf{y}}$ contains digit values 0 through 9.  The model will have difficuly predicting these values as is.  Instead imagine for a moment that we are just trying to determine if a digit is a zero or not. So we reduce our problem to a simpler problem. Just determine if the digit is a zero.  To do this we change our labele to y = 0 if the digit is not a zero and y = 1 if the digit is a zero. The numpy command below will create the new y.

```python
y = np.where(labels==0,1,0) # this command looks "where" the labels values are 0 and replace
# with a 1 else replace with a 0.
print(f'First 10 label values are {labels[:10]}')
print(f'First 10 y values are {y[:10]}')
# see the comparison of the original vector of labels and y. The second label is a zero so we see a 1 in
# corresponding postion for y
```

    First 10 label values are [5 0 4 1 9 2 1 3 1 4]
    First 10 y values are [0 1 0 0 0 0 0 0 0 0]

```python
# We need to append a column of zeros to the design matrix X
X = np.hstack((X,np.ones((60000,1))))
```

```python
# compute the parameters for the model, again we are only detecting digit zero for now
# this involves solving a sytem of 60,000 equations and 785 unknowns
beta = np.linalg.pinv(X.T@X)@X.T@y
beta.shape
```

    (785,)

Test the zero digit detector.

```python
# Get 100 random digits from the test set
index_values = np.random.choice(range(10000),100,replace=False)
digits = Xtest[index_values]
# append column of 1's
digits = np.hstack((digits,np.ones((100,1))))
# predict output for these digits
predict_values =(digits @ beta).round()

```

```python
# which digits do we predict are zero, below will give the position of the zeros.
np.where(predict_values==1)
```

    (array([ 8, 15, 23, 34, 39, 40, 43, 51, 57, 59, 68, 69, 72, 90, 92]),)

```python
# Which digits are zero, below will give the position of the zeros.
# The accuracy is good butt a big deal as we were just testing between
# two possiblities. Is the digit zero or not.
np.where(labelstest[index_values]==0)
```

    (array([15, 19, 34, 39, 40, 43, 51, 57, 59, 68, 69, 70, 72, 90]),)

Ten digit classification.

To accomplish 10 digit classification we combine 10 linear regressors, each similar to the on above, in parallel. This will involve using one-hot-encoding for our labels. Each label instead of being represented as a single digit will be represented by an array of 10 bits all zero value excecpt a "1" in the position corresponding to the digit value.  For example the digit 3 will be encoded [0,0,0,1,0,0,0,0,0,0]. We start counting from zero. When predicting a digit value the model will output a 10 bit string.  The postion that has the largest value is the predicted digit.
For example if the mode output the array [.22,.01,1.2,.44,.10,.25,.45,.76,.30,.55] then we predict the digit is a "2". Our parameter vector $\vec{\mathbf{\beta}}$ will not be a matrix of size 785 x 10.

```python
# one hot encode the labels
labels = labels.astype(int)
y = np.zeros((60000,10))
for k in range(60000):
    y[k,labels[k]]=1
```

```python
# display the first 10 labels
y[:10,:]
# below we see the first label is a digit "5"
# for the zero digit detector we just used the first column below. Now we process all 10 columns in parallel
```

    array([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
           [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
           [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],
           [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
           [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],
           [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
           [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
           [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
           [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
           [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]])

Compute the parameter matrix $\vec{\mathbf{\beta}}$

```python
beta_matrix = np.linalg.pinv(X.T@X)@X.T@y
```

We have just solve a sytem of 60000 equations in 785 unknowns.  Actually we have simultaneously solved 10 such systems.

```python
beta_matrix.shape
```

    (785, 10)

Test the model with 100 random digits.

```python
# predict output for the random digits chosen above
predict_values =(digits @ beta_matrix)
```

```python
predict_values.shape
```

    (100, 10)

```python
outcomes = np.argmax(predict_values,axis=1)
outcomes[:20]
# below we see the prediction for the first 20 digits
```

    array([1, 8, 3, 4, 3, 5, 4, 1, 0, 1, 6, 5, 6, 9, 7, 0, 6, 6, 7, 0])

```python
print('     ',labelstest[index_values][:20])
```

          [1 8 3 6 3 6 9 1 8 1 6 5 6 9 3 0 6 6 7 0]

Compute the accuracy.  Several runs give an average accuracy between 85 - 90 %

```python
sum(outcomes==labelstest[index_values])/100
```

    0.86

```python
# save the parameter matrix for later use
np.save("beta_matrix.npy",beta_matrix)
```

```python
%load_ext watermark

# python, ipython, packages, and machine characteristics
%watermark -v -m -p pandas,numpy,matplotlib,watermark

# date
print (" ")
%watermark -u -n -t -z
```

    The watermark extension is already loaded. To reload it, use:
      %reload_ext watermark
    CPython 3.7.6
    IPython 7.12.0

    pandas 1.0.1
    numpy 1.15.0
    matplotlib 3.1.3
    watermark 2.0.2

    compiler   : Clang 4.0.1 (tags/RELEASE_401/final)
    system     : Darwin
    release    : 19.5.0
    machine    : x86_64
    processor  : i386
    CPU cores  : 8
    interpreter: 64bit

    last updated: Mon Jul 13 2020 21:41:29 EDT


# Part 2: Link Prediction on a Knowledge Graph

Link prediction is the task of predicting missing connections (or links) between two nodes in a (directed) graph. By convention, we call those two nodes the subject $s$ and the object $o$. In most of the cases, we are considering typed edges which means that the edges (ie the relations) have a label, noted as $r$. An example triplet in the context of social networks could be $\Big(s="John Doe", r="follows", o="Karen Smith"\Big)$. Link prediction is as the heart of many applications with Knowledge Graphs (KG). For instance, link prediction can be used to propose to a user to connect/follow another one.

In this hands-on tutorial, we propose to play with the FB15k-237 dataset.  This dataset contains textual  relations  that were extracted  from  200  million  sentences  in  the  ClueWeb12  corpus  coupled  with Freebase mention annotations and include textual links of all co-occurring entities from the KB set. The FB15k-237 dataset has 15'000 nodes and 272'000 edges. 17'000 are used for validation and 20'000 are used for testing.

## Objectives:

- Sharpen your understanding of Graph Neural Networks with the R-GCN model
- Use the high-level API of the DGL library
- Train a link prediction task: predicting missing links in a Knowledge Graph (KG)

Disclaimer: This hands-on exercise is inspired by the official DGL tutorial introducing the Relational-Graph Convolutional Network (R-GCN) available [here](https://github.com/dmlc/dgl/tree/master/examples/pytorch/rgcn) .

## A.) Dataloading with DGL:

### Load FB15k-237 dataset

We can directly use the `dgl.data.knowledge_graph` module that allows to load several Knowledge Graphs, eg `FB15k-237`.

### Pre-processing

Working with KG requires some data manipulation to extract the test graph out of the full knowledge graph

```python
from utils.graph import *
from dgl.data.knowledge_graph import load_data

# 1. load KG
data = load_data("FB15k-237")

# 2. extract meta info
num_nodes = data.num_nodes
train_data = data.train
valid_data = torch.LongTensor(data.valid)
test_data = torch.LongTensor(data.test)
num_rels = data.num_rels

# 3. build test graph
test_graph, test_rel, test_norm = build_test_graph(num_nodes, num_rels, train_data)
test_deg = test_graph.in_degrees(range(test_graph.number_of_nodes())).float().view(-1,1)
test_node_id = torch.arange(0, num_nodes, dtype=torch.long).view(-1, 1)
test_rel = torch.from_numpy(test_rel)
test_norm = node_norm_to_edge_norm(test_graph, torch.from_numpy(test_norm).view(-1, 1))

# build adj list and calculate degrees for sampling
adj_list, degrees = get_adj_and_degrees(num_nodes, train_data)

```

## B.) Designing the model

### High-level model design

- Encode each node using a Graph Neural Network that can operates on directed node- and edges- labeled graph. An example is to use the relational GCN (R-GCN)
- Decode using a tensor factorization method to derive scores for each candidate triplet. An example is to use the  DistMult.

### Model formulation & implementation

#### R-GCN

We make use of the high-level API `dgl.nn.pytorch.conv` that allows to directly load GNN layers. In particular, this task is based on the R-GCN model. The R-GCN proposes to update each node as:
\begin{equation}
h(v)^{(k)} = \sigma \Big( \sum_{r \in R} \sum_{u \in N(v)^r} \frac{1}{c(v)^r} W_r^{(k)} h(u)^{(k-1)} + W_0^{(k)} h(v)^{(k-1)}   \Big)
\end{equation}

#### DistMult

The probability of a triplet defined by subject, object and a relation is computed using the `DistMult` as:
\begin{equation}
f(s, r, o) = e^T_s R_r e_o
\end{equation}
where $e_s$ and $e_o$ are the final node embeddings of the subject and object respectively, ie $e_s = h_s^{(k=K)}$

```python
import torch
import torch.nn as nn
from dgl.data.knowledge_graph import load_data
from dgl.nn.pytorch import RelGraphConv

class RGCN(nn.Module):
    def __init__(self, num_nodes, h_dim, out_dim, num_rels, num_bases,
                 num_hidden_layers=1, dropout=0,
                 use_self_loop=False):
        super(RGCN, self).__init__()
        self.num_nodes = num_nodes
        self.h_dim = h_dim
        self.out_dim = out_dim
        self.num_rels = num_rels
        self.num_bases = None if num_bases < 0 else num_bases
        self.num_hidden_layers = num_hidden_layers
        self.dropout = dropout
        self.use_self_loop = use_self_loop

        # create rgcn layers
        self.build_model()

    def build_model(self):
        self.layers = nn.ModuleList()
        # i2h
        i2h = self.build_input_layer()
        if i2h is not None:
            self.layers.append(i2h)
        # h2h
        for idx in range(self.num_hidden_layers):
            h2h = self.build_hidden_layer(idx)
            self.layers.append(h2h)

    def build_input_layer(self):
        return EmbeddingLayer(self.num_nodes, self.h_dim)

    def build_hidden_layer(self, idx):
        act = F.relu if idx < self.num_hidden_layers - 1 else None
        return RelGraphConv(self.h_dim, self.h_dim, self.num_rels, "bdd",
                self.num_bases, activation=act, self_loop=True,
                dropout=self.dropout)

    def build_output_layer(self):
        return None

    def forward(self, g, h, r, norm):
        for layer in self.layers:
            h = layer(g, h, r, norm)
        return h

class EmbeddingLayer(nn.Module):
    def __init__(self, num_nodes, h_dim):
        super(EmbeddingLayer, self).__init__()
        self.embedding = torch.nn.Embedding(num_nodes, h_dim)

    def forward(self, g, h, r, norm):
        return self.embedding(h.squeeze())

class LinkPredict(nn.Module):
    def __init__(self, in_dim, num_rels, h_dim=500, num_bases=100,
                 num_hidden_layers=2, dropout=0.2, reg_param=0.01):
        super(LinkPredict, self).__init__()
        self.rgcn = RGCN(in_dim, h_dim, h_dim, num_rels * 2, num_bases,
                         num_hidden_layers, dropout)
        self.reg_param = reg_param
        self.w_relation = nn.Parameter(torch.Tensor(num_rels, h_dim))

    def calc_score(self, embedding, triplets):
        # DistMult
        s = embedding[triplets[:,0]]
        r = self.w_relation[triplets[:,1]]
        o = embedding[triplets[:,2]]
        score = torch.sum(s * r * o, dim=1)
        return score

    def forward(self, g, h, r, norm):
        return self.rgcn.forward(g, h, r, norm)

    def regularization_loss(self, embedding):
        return torch.mean(embedding.pow(2)) + torch.mean(self.w_relation.pow(2))

    def get_loss(self, embed, triplets, labels):
        # triplets is a list of data samples (positive and negative)
        # each row in the triplets is a 3-tuple of (source, relation, destination)
        score = self.calc_score(embed, triplets)
        predict_loss = F.binary_cross_entropy_with_logits(score, labels)
        reg_loss = self.regularization_loss(embed)
        return predict_loss + self.reg_param * reg_loss

```

## C.) Define the training and testing loop

### Use classic PyTorch training loop
- Define the model parameters (num layers, GNN dimensions)
- Define the training parameters (optimizer, learning rate, weight decay, number of epochs)

### Define positive and negative samples

In order to train a link prediction system, we need to generate positive and negative triplets $(o, r, s)$ associated to a score (the triplet exists or doesn't exist).

**Positive samples:** During training, the system operates on a subset of the original graph, where we simply randomly drop edges. The dropped relationships are then used to evaluate the system during inference.

**Negative samples:** For each observed example we sample negative ones. We sample by randomly corrupting either the subject or the object of each positive example.

```python
from utils.graph import *
from utils.metrics import *

import torch.nn.functional as F
import torch
from tqdm import tqdm

# declare model
model = LinkPredict(num_nodes, num_rels)

# build optimizer
optimizer = torch.optim.Adam(
    model.parameters(),
    lr=1e-2,
    weight_decay=5e-4
)

epoch = 0
best_mrr = 0
while True:
    model.train()
    epoch += 1

    # perform edge neighborhood sampling to generate training graph and data
    g, node_id, edge_type, node_norm, data, labels = generate_sampled_graph_and_labels(train_data, num_rels, adj_list, degrees)

    # set node/edge feature
    node_id = torch.from_numpy(node_id).view(-1, 1).long()
    edge_type = torch.from_numpy(edge_type)
    edge_norm = node_norm_to_edge_norm(g, torch.from_numpy(node_norm).view(-1, 1))
    data, labels = torch.from_numpy(data), torch.from_numpy(labels)
    deg = g.in_degrees(range(g.number_of_nodes())).float().view(-1, 1)

    embed = model(g, node_id, edge_type, edge_norm)
    loss = model.get_loss(embed, data, labels)
    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) # clip gradients
    optimizer.step()

    print("Epoch {:04d} | Loss {:.4f} | Best MRR {:.4f}".format(epoch, loss.item(), best_mrr))

    optimizer.zero_grad()

    # validation
    if epoch % 500 == 0:
        # perform validation on CPU because full graph is too large
        model.cpu()
        model.eval()
        print("start eval")
        embed = model(test_graph, test_node_id, test_rel, test_norm)
        mrr = calc_mrr(embed, model.w_relation, torch.LongTensor(train_data),
                             valid_data, test_data, hits=[1, 3, 10])
        # save best model
        if mrr < best_mrr:
            if epoch >= 6000:
                break
        else:
            best_mrr = mrr
            torch.save({'state_dict': model.state_dict(), 'epoch': epoch},
                       'checkpoints/model_state_{}.pt'.format(str(epoch)))
print("training done")

```

## D.) Testing

### Metrics
- The system is assessed using:
    - the mean reciprocal rank ($MRR$)
    - Hits at n ($H@n$)

```python
from utils.graph import *
from utils.metrics import *
import torch.nn.functional as F

# 1. load pre-trained network
model = LinkPredict(num_nodes, num_rels)
checkpoint = torch.load('model_state_2000.pt')

# 2. eval and compute MRR
model.cpu()
model.eval()
model.load_state_dict(checkpoint['state_dict'])
print("Using best epoch: {}".format(checkpoint['epoch']))
embed = model(test_graph, test_node_id, test_rel, test_norm)
calc_mrr(embed, model.w_relation, torch.LongTensor(train_data), valid_data, test_data, hits=[1, 3, 10])
```

```python

```

notebooks/Part2 - Link Prediction on a Knowledge Graph.ipynb


# Linear regression

Linear regression is the simplest linear method used for modelling the relationship between the independent variables and the dependent ones. It tries to estimate it by finding a line which is as close as possible to all the data points.

\begin{equation}
y=ax+b
\end{equation}

#### Boston housing example

[Boston housing](https://www.kaggle.com/c/boston-housing) is a very simple dataset built from some statistical data of the houses of Boston suburbs and the median prices (in $1000s) of owner-occupied homes for each zone.

```python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_boston

# Loading the dataset with pandas
boston_data = load_boston()
boston_housing_df = pd.DataFrame(boston_data.data,columns=boston_data.feature_names)
boston_housing_df["MEDV"] = boston_data.target
boston_housing_df.head()
```

      CRIM
      ZN
      INDUS
      CHAS
      NOX
      RM
      AGE
      DIS
      RAD
      TAX
      PTRATIO
      B
      LSTAT
      MEDV

      0
      0.00632
      18.0
      2.31
      0.0
      0.538
      6.575
      65.2
      4.0900
      1.0
      296.0
      15.3
      396.90
      4.98
      24.0

      1
      0.02731
      0.0
      7.07
      0.0
      0.469
      6.421
      78.9
      4.9671
      2.0
      242.0
      17.8
      396.90
      9.14
      21.6

      2
      0.02729
      0.0
      7.07
      0.0
      0.469
      7.185
      61.1
      4.9671
      2.0
      242.0
      17.8
      392.83
      4.03
      34.7

      3
      0.03237
      0.0
      2.18
      0.0
      0.458
      6.998
      45.8
      6.0622
      3.0
      222.0
      18.7
      394.63
      2.94
      33.4

      4
      0.06905
      0.0
      2.18
      0.0
      0.458
      7.147
      54.2
      6.0622
      3.0
      222.0
      18.7
      396.90
      5.33
      36.2

There are several features in the dataset:

* **crim**  per capita crime rate by town.
* **zn** proportion of residential land zoned for lots over 25,000 sq.ft.
* **indus** proportion of non-retail business acres per town.
* **chas** Charles River dummy variable (= 1 if tract bounds river; 0 otherwise).
* **nox** nitrogen oxides concentration (parts per 10 million).
* **rm** average number of rooms per dwelling.
* **age** proportion of owner-occupied units built prior to 1940.
* **dis** weighted mean of distances to five Boston employment centres.
* **rad** index of accessibility to radial highways.
* **tax** full-value property-tax rate per \$10000
* **ptratio** pupil-teacher ratio by town.
* **black** 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town.
* **lstat** lower status of the population (percent).
* **medv** median value of owner-occupied homes in \$1000s.

The target variable $y$ is called *medv*.

#### 2D linear regression

For the simplicity, we'll consider a 2D example and try to predict the *medv* (median value), given *crim* (per capita crime rate). Let's take a look at the data in the first place.

```python
boston_housing_df.plot(x="CRIM", y="MEDV", kind="scatter")

from sklearn.linear_model import LinearRegression

# Create an instance of LinearRegression and find the coeffs
linear_regression = LinearRegression()
linear_regression.fit(X=boston_housing_df[["CRIM"]],
                      y=boston_housing_df["MEDV"])
linear_regression.coef_, linear_regression.intercept_
```

We can try to draw the linear regression $y$ using matplotlib. In the code below we take the MEDV features as $y$ and CRIM as $x$.

```python
# Create a polynomial to be drawn on the plot
coefficients = np.append(linear_regression.coef_,
                         linear_regression.intercept_)
polynomial = np.poly1d(coefficients)

# Calculate the values for a selected range
x_values = np.linspace(0, boston_housing_df["CRIM"].max())
y_values = polynomial(x_values)

# Display a scatter plot: crim vs medv and regressed line
boston_housing_df.plot(x="CRIM", y="MEDV", kind="scatter")
plt.plot(x_values, y_values, color="red", linestyle="dashed")

from sklearn.metrics import mean_squared_error

y_pred = linear_regression.predict(boston_housing_df[["CRIM"]])
y_true = boston_housing_df["MEDV"]
mean_squared_error(y_true, y_pred)
```

### Multidimensional linear regression

An intuitive selection of the possibile predictor did not help to perform a regression of the median value in the area properly. For a low crime rate it looks better, but when it comes to really high crime rate, the predicted value is negative.

For the purposes of selecting predictors, we may consider the variables which have the highest correlation with the target variable.

```python
# Calculate the Pearson correlation coefficients
boston_housing_df.corr()["MEDV"]
```

    CRIM      -0.388305
    ZN         0.360445
    INDUS     -0.483725
    CHAS       0.175260
    NOX       -0.427321
    RM         0.695360
    AGE       -0.376955
    DIS        0.249929
    RAD       -0.381626
    TAX       -0.468536
    PTRATIO   -0.507787
    B          0.333461
    LSTAT     -0.737663
    MEDV       1.000000
    Name: MEDV, dtype: float64

The absolute value of correlation coefficients is highest for *rm* (0.689598) and *lstat* (-0.738600). That means, these values are possibly the best predictors for the target variable and we can consider them in a 3D regression.

```python
from mpl_toolkits.mplot3d import Axes3D

# Display 3D scatter: rm, lstat vs medv
fig = plt.figure()
ax = fig.add_subplot(111, projection="3d")
ax.scatter(boston_housing_df["RM"], boston_housing_df["LSTAT"],
           boston_housing_df["MEDV"], c="blue")
plt.show()
```

Let's find the coefficiencies and build a linear regression instance.

```python
linear_regression = LinearRegression()
linear_regression.fit(X=boston_housing_df[["RM", "LSTAT"]],
                      y=boston_housing_df["MEDV"])
linear_regression.coef_, linear_regression.intercept_
```

    (array([ 5.09478798, -0.64235833]), -1.358272811874489)

For a three-dimensional case we need to calculate three values. We get three coeffiencies that are used to get the $z$ values.

```python
# Calculate coefficients of 2d polynomial
coefficients = np.append(linear_regression.coef_,
                         linear_regression.intercept_)

# Calculate the values for a selected range
x = np.linspace(0, boston_housing_df["RM"].max())
y = np.linspace(0, boston_housing_df["LSTAT"].max())
x_values, y_values = np.meshgrid(x, y)
z_values = coefficients[0] * x_values + coefficients[1] * y_values + coefficients[2]

# Display 3D scatter: rm, lstat vs medv and regressed line
fig = plt.figure()
ax = fig.add_subplot(111, projection="3d")
ax.scatter(boston_housing_df["RM"], boston_housing_df["LSTAT"],
           boston_housing_df["MEDV"], c="blue")
ax.plot_surface(x_values, y_values, z_values, linewidth=0.2,
                color="red", alpha=0.5)
angle=30
ax.view_init(30, angle)
plt.show()
```

The prediction of linear regression can be done with the ``predict`` method. The cost function of a linear regression is calculated with mean squared error function.

```python
y_pred = linear_regression.predict(boston_housing_df[["RM", "LSTAT"]])
y_true = boston_housing_df["MEDV"]
mean_squared_error(y_true, y_pred)
```

    30.51246877729947

# Linear regression under the hood

To understand the method in more details we use a simple example of humans heights and weights values.

```python
heights = np.array([188, 181, 197, 168, 167, 187, 178, 194, 140, 176, 168, 192, 173, 142, 176]).reshape(-1, 1)
weights = np.array([141, 106, 149, 59, 79, 136, 65, 136, 52, 87, 115, 140, 82, 69, 121]).reshape(-1, 1)
```

For comparison, we use the linear regression available in sklearn library.

```python
from sklearn import linear_model
import numpy as np

regr = linear_model.LinearRegression()
regr.fit(heights, weights)
```

    LinearRegression()

As in the previous example, we can plot the slope.

```python
plt.scatter(heights, weights,color='g')
plt.plot(heights, regr.predict(heights),color='k')

plt.show()
```

The coeffieciencies are calculated as:

```python
print(regr.coef_)
print(regr.intercept_)
```

    [[1.61814247]]
    [-180.92401772]

If we calcualte the $y$ value for $x=150$:

```python
plt.scatter(heights, weights,color='g')
plt.plot(heights, regr.predict(heights),color='k')

x= 150
y = 61.797
plt.scatter(x, y,color='r')

plt.show()
```

## Linear regression from scratch

And now we can use the data set to implement the linear regression from scratch.

```python
x = heights.reshape(15,1)
y = weights.reshape(15,1)
```

We should add the bias column before doing the calculation.

```python
x = np.append(x, np.ones((15,1)), axis = 1)
```

The equation for calculating the weights is
\begin{equation}
(X^{T}X)^{-1}X^{T}y
\end{equation}

```python
w = np.dot(np.linalg.inv(np.dot(np.transpose(x),x)),np.dot(np.transpose(x),y))
```

The weitghs we can as the output are exaclty the same as we get using sklearn implementation of the linear regression method.

```python
w
```

    array([[   1.61814247],
           [-180.92401772]])

To make the prediction a bit easier, we can implement a short function where the arguments are:
- inputs - feature $x$ of objects,
- w - weights,
- b - bias.
The calculation is easy and use the linear regression equation $y=wx+b$.

```python
def reg_predict(inputs, w, b):
    results = []
    for inp in inputs:
        results.append(inp*w+b)
    return results
```

Finally, we can plot the predicted values using the function above.

```python
plt.scatter(heights.flatten(), weights.flatten(),color='g')
plt.plot(heights.flatten(), reg_predict(heights.flatten(), w[0], w[1]) ,color='k')

x1 = 150
y = reg_predict([x1], w[0], w[1])[0]
plt.scatter(x1, y,color='r')

plt.show()
```

```python
reg_predict(x.flatten(), w[1], w[0])
```

    [array([-34012.0971882]),
     array([-179.30587525]),
     array([-32745.62906419]),
     array([-179.30587525]),
     array([-35640.41334765]),
     array([-179.30587525]),
     array([-30393.61683388]),
     array([-179.30587525]),
     array([-30212.69281616]),
     array([-179.30587525]),
     array([-33831.17317049]),
     array([-179.30587525]),
     array([-32202.85701104]),
     array([-179.30587525]),
     array([-35097.6412945]),
     array([-179.30587525]),
     array([-25327.74433782]),
     array([-179.30587525]),
     array([-31841.00897561]),
     array([-179.30587525]),
     array([-30393.61683388]),
     array([-179.30587525]),
     array([-34735.79325907]),
     array([-179.30587525]),
     array([-31298.23692246]),
     array([-179.30587525]),
     array([-25689.59237325]),
     array([-179.30587525]),
     array([-31841.00897561]),
     array([-179.30587525])]

# Ridge regression

Ridge regression use a regularizer and the equation is a bit more complex compare to the regular linear regression:
\begin{equation}
 \sum_{i=1}^{M}(y_{i}-\sum_{j=0}^{p}w_{j}\dot x_{ij})^{2} + \lambda\sum_{j=0}^{p}w^{2}_{j}.
\end{equation}
We have an additional parameter $\lambda$ that is known in sklearn as $\alpha$. It's the regularizer that together with $w^{2}_{j}$ is known as the L2 regularizator.

```python
from sklearn.linear_model import Ridge

alpha = 0.1

heights1 = np.asmatrix(np.c_[np.ones((15,1)), heights])

ridge_regression = Ridge(alpha=alpha, fit_intercept=False)
ridge_regression.fit(X=heights1,
                      y=weights)
ridge_regression.coef_, ridge_regression.intercept_
```

    (array([[-101.72397081,    1.16978757]]), 0.0)

Similar to the regular linear regression, the slope can be drawn as below.

```python
plt.scatter(heights, weights,color='g')
plt.plot(heights, ridge_regression.predict(heights1),color='k')

x = 150
y = reg_predict([150], ridge_regression.coef_[0][1], ridge_regression.coef_[0][0])[0]
plt.scatter(x, y,color='r')

y = ridge_regression.coef_[0][1] * 150 + ridge_regression.coef_[0][0]

plt.show()
```

For $x_{1}=150$ the result is a bit different compared to the regression without the regularization.

```python
y = ridge_regression.coef_[0][1] * 150 + ridge_regression.coef_[0][0]
print(y)
```

    73.74416542365165

We can write the equation in a matrix-way as:
\begin{equation}
(X^{T}X+\alpha\dot W)^{-1}X^{T}y
\end{equation}

```python
y = weights
x = np.asmatrix(np.c_[np.ones((15,1)),heights])

I = np.identity(2)
alpha = 0.1

w = np.linalg.inv(x.T*x + alpha * I)*x.T*y
```

The weights are calculated same as in case of sklearn.

```python
w=np.array(w).ravel()
print(w)
```

    [-101.72397081    1.16978757]

The plot looks as below. We see the the slope start with higer $y$ values compared to the regular linear regression.

```python
plt.scatter(heights, weights, color='g')
plt.plot(heights, reg_predict(heights.flatten(), w[1], w[0]),color='k')

x1= 150
y = x1*w[1]+w[0]
plt.scatter(x1, y,color='r')

plt.show()
```

# Lasso regression

Lasso regression uses the L1 regularization. The equation is very similar to the Ridge one, but instead of $w^{2}$ we use the magnitude of $w$.

\begin{equation}
 \sum_{i=1}^{M}(y_{i}-\sum_{j=0}^{p}w_{j}\dot x_{ij})^{2} + \lambda\sum_{j=0}^{p}|w_{j}|.
\end{equation}

```python
from sklearn.linear_model import Lasso

alpha = 0.1

lasso_regression = Lasso(alpha=alpha)
lasso_regression.fit(X=heights,
                      y=weights)
lasso_regression.coef_, lasso_regression.intercept_
```

    (array([1.61776499]), array([-180.8579086]))

```python
plt.scatter(heights, weights,color='g')
plt.plot(heights, lasso_regression.predict(heights),color='k')

plt.show()
```

```python

```