```python
import pandas as pd
import numpy as np
from sympy import *
import matplotlib.pyplot as plt
import math
from pprint import pprint
```

```python
alfa = 0.000097
L = 1
#Relx = 0.5
deltaX = 0.05
deltat = 10
nodos = int(L/deltaX)
Bo = 15 # X = 0
Bl = 0 # X = L
##################################
a = -alfa*deltat/deltaX**2
```

```python
initial = np.empty(nodos)
initial.fill(15)
initial = np.append(initial, 15)
source = np.zeros(len(initial))
zeros = np.zeros(len(initial))
Relax = np.zeros(len(initial))
```

```python
x = np.linspace(start = 0, stop = L, num=nodos+1, endpoint=True)
```

```python
M = np.zeros([len(x), len(x)])
for i in range(len(x)):
    for j in range(len(x)):
        if(i == j):
            if(i==0 or i == len(x)-1):
                M[i,j] = 1
            else:
                M[i,j] = (1-2*a)
        if (i != len(x)-1 and i - 1 == j):
            M[i,j] = a
            #print('2. position ' + str(i) + ',' + str(j) + ' has value ' + str(M[i,j]))
        if (i != 0 and i + 1 == j):
            M[i,j] = a
            #print('4. position ' + str(i) + ',' + str(j) + ' has value ' + str(M[i,j]))
```

```python
M_inv = np.linalg.inv(M)
dfM_inv = pd.DataFrame(M_inv.round(2))
dfM = pd.DataFrame(M.round(2))
iterations = 20
result = [0]*iterations
```

```python
for ite in range(iterations):
    if(ite == 0):
        newT = np.dot(M_inv, initial)
        result[0]=initial
    else:
        #print('ite: ' + str(ite))
        np.put(newT, len(newT)-1, 0)
        newT = np.dot(M_inv,newT)
        result[ite]= newT
```

```python
plt.plot(x, newT, marker = '2', color = 'red', label = 'Prediction')
plt.title('Solucion dt= ' + str(deltat) + ' en ' + str(iterations) + ' iteraciones')
plt.legend()
plt.show()
```

```python
Data_result = pd.DataFrame(result).T
Data_result.columns = ['It '+ str(col+1) for col in Data_result.columns]
Data_result.index = Data_result.index*deltaX
```

```python
plt.plot(Data_result.iloc[:,:])
plt.title('All iterations (each color)')
plt.xlabel('Length')
plt.ylabel('Temperature')
plt.show()
```

```python
Data_result.round(2)
```

      It 1
      It 2
      It 3
      It 4
      It 5
      It 6
      It 7
      It 8
      It 9
      It 10
      It 11
      It 12
      It 13
      It 14
      It 15
      It 16
      It 17
      It 18
      It 19
      It 20

      0.00
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.05
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.10
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.15
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.20
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.25
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00

      0.30
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.99

      0.35
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.99
      14.99
      14.98
      14.98

      0.40
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.99
      14.98
      14.98
      14.97
      14.96
      14.95

      0.45
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.98
      14.98
      14.97
      14.96
      14.95
      14.94
      14.92
      14.91

      0.50
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.98
      14.97
      14.96
      14.95
      14.93
      14.92
      14.90
      14.87
      14.85
      14.82

      0.55
      15.0
      15.00
      15.00
      15.00
      15.00
      15.00
      14.99
      14.99
      14.98
      14.97
      14.95
      14.93
      14.91
      14.89
      14.86
      14.83
      14.79
      14.75
      14.71
      14.67

      0.60
      15.0
      15.00
      15.00
      15.00
      14.99
      14.99
      14.98
      14.96
      14.94
      14.92
      14.89
      14.85
      14.81
      14.76
      14.71
      14.65
      14.59
      14.53
      14.46
      14.39

      0.65
      15.0
      15.00
      15.00
      14.99
      14.98
      14.96
      14.94
      14.90
      14.86
      14.80
      14.74
      14.67
      14.60
      14.52
      14.43
      14.34
      14.24
      14.14
      14.04
      13.94

      0.70
      15.0
      15.00
      14.99
      14.97
      14.94
      14.89
      14.83
      14.75
      14.66
      14.55
      14.44
      14.32
      14.19
      14.06
      13.92
      13.78
      13.64
      13.50
      13.36
      13.22

      0.75
      15.0
      14.99
      14.96
      14.90
      14.81
      14.70
      14.56
      14.40
      14.22
      14.04
      13.84
      13.65
      13.45
      13.25
      13.06
      12.87
      12.68
      12.50
      12.32
      12.14

      0.80
      15.0
      14.96
      14.85
      14.68
      14.46
      14.21
      13.93
      13.63
      13.34
      13.04
      12.75
      12.47
      12.20
      11.94
      11.70
      11.46
      11.24
      11.02
      10.82
      10.62

      0.85
      15.0
      14.82
      14.47
      14.03
      13.55
      13.05
      12.57
      12.11
      11.68
      11.28
      10.91
      10.57
      10.26
      9.97
      9.70
      9.44
      9.21
      8.99
      8.79
      8.59

      0.90
      15.0
      14.21
      13.21
      12.24
      11.38
      10.62
      9.97
      9.41
      8.92
      8.50
      8.13
      7.80
      7.50
      7.24
      7.00
      6.78
      6.58
      6.40
      6.23
      6.07

      0.95
      15.0
      11.55
      9.39
      7.96
      6.97
      6.24
      5.69
      5.26
      4.91
      4.62
      4.38
      4.17
      3.99
      3.83
      3.68
      3.55
      3.44
      3.33
      3.24
      3.15

      1.00
      15.0
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00
      0.00

```python

```


```python
from sympy.stats import *
from sympy import *
```

```python
l, z = var("lambda z")
pdf = Piecewise((0, Lt(z, 0)), (l*exp(-l*z), True))
pprint(pdf)
X = ContinuousRV(z, pdf)
P(X >= 4000)
```

    ⎧   0     for z < 0
    ⎪
    ⎨   -λ⋅z
    ⎪λ⋅ℯ      otherwise
    ⎩

$\displaystyle \int\limits_{4000}^{\infty} \begin{cases} 0 & \text{for}\: z < 0 \\\lambda e^{- z \lambda} & \text{otherwise} \end{cases}\, dz$

```python
P(Ge(X, 4000))
```

$\displaystyle \int\limits_{4000}^{\infty} \begin{cases} 0 & \text{for}\: z < 0 \\\lambda e^{- z \lambda} & \text{otherwise} \end{cases}\, dz$

```python
P(Ge(X.subs({l: 1/8000}), 4000))
```

$\displaystyle 0.606530659712633$

```python
P(Ge(X, 4000))
```

$\displaystyle \int\limits_{4000}^{\infty} \begin{cases} 0 & \text{for}\: z < 0 \\\lambda e^{- z \lambda} & \text{otherwise} \end{cases}\, dz$

```python
t = X.subs({l: 1/8000})
t
```

$\displaystyle X$


# Logistic Regression

We'll perform a logistic regression to try and classify breast tumors as benign or malignant based on features identified from biopsies.

## Technical setup
First we load packages we'll be using

```
# Import numpy for linear algebra operations
import numpy as np

# Import pandas for data manipulation
import pandas as pd

# Import the functions we'll need from scikit-learn
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report
from sklearn.feature_selection import RFE

import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
```

## Import the Data

```
breast_cancer_dataset = load_breast_cancer()
```

## Investigate the Data

We'll perform some EDA here. First we take a look at the data structure that's been passed in. We'll print the contents of the dataset object.

```
print(dir(breast_cancer_dataset))
```

    ['DESCR', 'data', 'feature_names', 'filename', 'target', 'target_names']

We note that there is a DESCR attribute in the data structure so we can try printing that out. It gives us lots of useful information about our data.

```
print(breast_cancer_dataset.DESCR)
```

    .. _breast_cancer_dataset:

    Breast cancer wisconsin (diagnostic) dataset
    --------------------------------------------

    **Data Set Characteristics:**

        :Number of Instances: 569

        :Number of Attributes: 30 numeric, predictive attributes and the class

        :Attribute Information:
            - radius (mean of distances from center to points on the perimeter)
            - texture (standard deviation of gray-scale values)
            - perimeter
            - area
            - smoothness (local variation in radius lengths)
            - compactness (perimeter^2 / area - 1.0)
            - concavity (severity of concave portions of the contour)
            - concave points (number of concave portions of the contour)
            - symmetry
            - fractal dimension ("coastline approximation" - 1)

            The mean, standard error, and "worst" or largest (mean of the three
            largest values) of these features were computed for each image,
            resulting in 30 features.  For instance, field 3 is Mean Radius, field
            13 is Radius SE, field 23 is Worst Radius.

            - class:
                    - WDBC-Malignant
                    - WDBC-Benign

        :Summary Statistics:

        ===================================== ====== ======
                                               Min    Max
        ===================================== ====== ======
        radius (mean):                        6.981  28.11
        texture (mean):                       9.71   39.28
        perimeter (mean):                     43.79  188.5
        area (mean):                          143.5  2501.0
        smoothness (mean):                    0.053  0.163
        compactness (mean):                   0.019  0.345
        concavity (mean):                     0.0    0.427
        concave points (mean):                0.0    0.201
        symmetry (mean):                      0.106  0.304
        fractal dimension (mean):             0.05   0.097
        radius (standard error):              0.112  2.873
        texture (standard error):             0.36   4.885
        perimeter (standard error):           0.757  21.98
        area (standard error):                6.802  542.2
        smoothness (standard error):          0.002  0.031
        compactness (standard error):         0.002  0.135
        concavity (standard error):           0.0    0.396
        concave points (standard error):      0.0    0.053
        symmetry (standard error):            0.008  0.079
        fractal dimension (standard error):   0.001  0.03
        radius (worst):                       7.93   36.04
        texture (worst):                      12.02  49.54
        perimeter (worst):                    50.41  251.2
        area (worst):                         185.2  4254.0
        smoothness (worst):                   0.071  0.223
        compactness (worst):                  0.027  1.058
        concavity (worst):                    0.0    1.252
        concave points (worst):               0.0    0.291
        symmetry (worst):                     0.156  0.664
        fractal dimension (worst):            0.055  0.208
        ===================================== ====== ======

        :Missing Attribute Values: None

        :Class Distribution: 212 - Malignant, 357 - Benign

        :Creator:  Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian

        :Donor: Nick Street

        :Date: November, 1995

    This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets.
    https://goo.gl/U2Uwz2

    Features are computed from a digitized image of a fine needle
    aspirate (FNA) of a breast mass.  They describe
    characteristics of the cell nuclei present in the image.

    Separating plane described above was obtained using
    Multisurface Method-Tree (MSM-T) [K. P. Bennett, "Decision Tree
    Construction Via Linear Programming." Proceedings of the 4th
    Midwest Artificial Intelligence and Cognitive Science Society,
    pp. 97-101, 1992], a classification method which uses linear
    programming to construct a decision tree.  Relevant features
    were selected using an exhaustive search in the space of 1-4
    features and 1-3 separating planes.

    The actual linear program used to obtain the separating plane
    in the 3-dimensional space is that described in:
    [K. P. Bennett and O. L. Mangasarian: "Robust Linear
    Programming Discrimination of Two Linearly Inseparable Sets",
    Optimization Methods and Software 1, 1992, 23-34].

    This database is also available through the UW CS ftp server:

    ftp ftp.cs.wisc.edu
    cd math-prog/cpo-dataset/machine-learn/WDBC/

    .. topic:: References

       - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction
         for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on
         Electronic Imaging: Science and Technology, volume 1905, pages 861-870,
         San Jose, CA, 1993.
       - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and
         prognosis via linear programming. Operations Research, 43(4), pages 570-577,
         July-August 1995.
       - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques
         to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994)
         163-171.

Now we know a little bit about our data we can copy it over to pandas to check correlations between attributes, including the target variable.

```
breast_cancer_df =  pd.DataFrame(breast_cancer_dataset.data, columns=breast_cancer_dataset.feature_names)
breast_cancer_df['benign'] = breast_cancer_dataset.target

plt.figure(figsize=(22,18))

# Calculate the correlation matrix which will be correlations between each of the attributes
correlation_matrix = breast_cancer_df.corr().round(2)
# Produce a heatmap showing the correlation matrix
sns.heatmap(data=correlation_matrix, annot=True, cmap=sns.color_palette("coolwarm", 7))
```

## Prepare the Data

We'll use information from the correlations between our features and our target, `benign`, to eliminate irrelevant features from our dataset.

```
# correlation_matrix['benign'], get the last row of the correlation matrix
# .iloc[:-1], removing the last column, 'benign'
# .where(lambda x : x < -0.7), for each value, return the value if
#                              it's less than -0.7 otherwise return NaN
# .notnull(), return False if NaN, True otherwise
corr_support = correlation_matrix['benign']\
                  .iloc[:-1]\
                  .where(lambda x : x < -0.73)\
                  .notnull()
# corr_support = [ True, False,  ...]
# Display the feature selection/elimination from RFE
print('Selected features:')
for feature_name, selected in zip(breast_cancer_dataset.feature_names, corr_support):
  print(f"\t\t   {'+' if selected else '-'} {feature_name}")
```

    Selected features:
    		   - mean radius
    		   - mean texture
    		   + mean perimeter
    		   - mean area
    		   - mean smoothness
    		   - mean compactness
    		   - mean concavity
    		   + mean concave points
    		   - mean symmetry
    		   - mean fractal dimension
    		   - radius error
    		   - texture error
    		   - perimeter error
    		   - area error
    		   - smoothness error
    		   - compactness error
    		   - concavity error
    		   - concave points error
    		   - symmetry error
    		   - fractal dimension error
    		   + worst radius
    		   - worst texture
    		   + worst perimeter
    		   - worst area
    		   - worst smoothness
    		   - worst compactness
    		   - worst concavity
    		   + worst concave points
    		   - worst symmetry
    		   - worst fractal dimension

```
g = sns.PairGrid(breast_cancer_df, vars=breast_cancer_dataset.feature_names[corr_support],
                 hue='benign', palette='coolwarm_r')
g.map(plt.scatter, alpha=0.8)
g.add_legend();
for i, j in zip(*np.triu_indices_from(g.axes, 0)):
    g.axes[i, j].set_visible(False)
```

Once we've identified the best features for our model we can create a dataset with the reduced feature list.

```
# Define X, our features
X = pd.DataFrame(breast_cancer_dataset.data[:, corr_support], columns=breast_cancer_dataset.feature_names[corr_support])
# Define y, our lables
y = breast_cancer_dataset.target
```

We split the dataset into a training set and a test set

```
# Split our data into training and testing sets using random sampling with 20% of the data as test data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state=5)  # random_state will cause the same random sequence to be generated every time
```

```
# Create our model: unpenalized logistic regression
logreg = LogisticRegression(solver='lbfgs', penalty='none', max_iter=500, random_state=1)
```

## Train the Data

Fit our logistic regression model to our data

```
# Train the model
logreg.fit(X_train, y_train)
```

    LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
                       intercept_scaling=1, l1_ratio=None, max_iter=500,
                       multi_class='warn', n_jobs=None, penalty='none',
                       random_state=1, solver='lbfgs', tol=0.0001, verbose=0,
                       warm_start=False)

## Evaluate the Performance of the Model

We'll look at the predictive accuracy on our training set and then on the test set. Remember, we're looking to maximize accuracy on our test data.

We'll also look at the classifiation report from scikit-learn.

The classification report gives us
$$Precision = \frac{True\ Positives}{True\ Positives + False\ Positives}$$
and
$$Recall = \frac{True\ Positives}{True\ Positives + False\ Negatives}$$
and
$$F_1\ Score = 2 \times \frac{Precision \times Recall}{Precision + Recall}$$

**Precision** talks about how precise/accurate your model is out of those predicted positive, how many of the labeled positives were actually positive.

**Recall** calculates how many of the actual positives were properly labeled as positive.

*F1 Score* is a measure of accuracy using precision and recall. It takes the occurance of both false positives and false negatives into account.

A **confusion matrix** shows counts of true negatives, true positives, false negatives and false positives.

In the following example the data is almost all negatives. Accuracy is 99% but if the false positive represents something very costly like a highly infectious disease or maybe fraud in a business then accuracy useful on its own.

For this example we calculate F1 as follows:

$$
\begin{align}
Precision & = \frac{1}{1} =  1\\
Recall & = \frac{1}{2} = 0.5\\
F_1 & = 2 \times \frac{1 \times 0.5}{ 1 + 0.5} = 2 \times \frac{0.5}{1.5} = 0.66666'
\end{align}
$$

**Support** provides a count of occurences of the true classes. Continuing our example:

$$
Support = \begin{cases}
998, & \text{if }class \text{ is Negative}\\
2, &  \text{if }class \text{ is Positive}
\end{cases}
$$

```
# Make predictions using training data
y_train_predict = logreg.predict(X_train)
train_accuracy = accuracy_score(y_train, y_train_predict)

# Display the training accuracy
print(f'\n\nTraining accuracy = {train_accuracy*100:.3f}%\n')

# Make predictions using test data
y_test_predict = logreg.predict(X_test)
test_accuracy = accuracy_score(y_test, y_test_predict)

# Display the test accuracy
print(f'Test accuracy = {test_accuracy*100:.3f}%\n\n')

# Show the classification report
print(classification_report(y_test, y_test_predict, target_names=breast_cancer_dataset.target_names))

# Calculate and show the misclassification rate
print(f'This model will mis-classify about 1 in {1/(1 - test_accuracy):.2f}.')
```

    Training accuracy = 95.824%

    Test accuracy = 96.491%

                  precision    recall  f1-score   support

       malignant       0.98      0.94      0.96        48
          benign       0.96      0.98      0.97        66

        accuracy                           0.96       114
       macro avg       0.97      0.96      0.96       114
    weighted avg       0.97      0.96      0.96       114

    This model will mis-classify about 1 in 28.50.

Support is fairly balanced. The f1-score is half way between the precision and recall for both classes and it aligns well with our accuracy so the classification report doesn't give us much additional information in this case but it's very useful to know on cases where our data isn't as well balanced or in cases where the type of incorrect predictions is critical to decisions.

96.49% accuracy might sound pretty good but that means that about 1 in 28.5 of these predictions will be incorrect which is OK but probably not accurate enough for a patient or a provider who has to make a critical, high-risk medical decision based on this information. Every fraction of a percentage point we can gain as we approach 100% provides greater and greater gains.

# Exercise

The correlation matrix heatmap gave us some useful pointers but there's a few issues. It's comparing individual attributes so it doesn't take into account combined effects of features together. There's only two classes in our target so it's difficult to say when a correlation is meaningful.

We'll pivot into a different approach for preparing our data here. It's a technique called l2 regularization. It works by weakening the effect of already weak features. This

OK, let's run that process against our breast cancer data:

```
# This is a logistic regression classifier but it's using an 'l2' penalty
# This is also known as 'ridge regression'. l2 penalty is actually default
# but we're being explicit here for clarity
logreg = LogisticRegression(solver='liblinear', penalty='l2', random_state=1)
```

Fill in the code beneath each of the comment sections below

We're not using corr_support this time so just remove the square brackets and anything contained in them, for instance
```
breast_cancer_dataset.data[:, corr_support]
```
above needs to be replaced with just
```
breast_cancer_dataset.data
```

```
# Define your features, X
# X = ...

```

The following three lines of code will be the same as we used in the example above. Blank lines or comment lines which start with a `#` are not counted as code lines.

```
# Define your labels, y
# y = ...

# Split X and y into training and test datasets
# X_train, X_test, y_train, y_test =

# Train (Fit) your model
# logreg. ...

```

OK, we should have now built our test and training sets and we've trained our model by passing our training features and training labels to the `logreg.fit()` method.

Here's a quick check of the *training data* accuracy (which won't tell us if we're overfitting):

```
# Make predictions using training data
y_train_predict = logreg.predict(X_train)

# Use accuracy_score to get the training score from the test labels and the predictions
train_accuracy = accuracy_score(y_train, y_train_predict)

# Display the training accuracy
print(f'Training accuracy = {train_accuracy*100:.3f}%')
```

In order to check the accuracy under something close to real world conditions we want to predict against our test data, which our model hasn't seen during the training. We can then compare the predictions with the actual test labels to check the accuracy and print it out

```
# Make predictions using test data
# y_test_predict = ...
# test_accuracy = ...

# For the next three lines of code, the comments show how the line starts

# Display the test accuracy
# print(f'Test accuracy = {...

# Display the classification report for test labels vs predictions
# print(classification_report(...

# Calculate and show the mis-classification rate
# print(f'This model will mis-classify about 1 in {...

```


## Linear Regression in Python for Engineers, Data Scientists and Geoscientists
### Michael Pyrcz, Associate Professor, University of Texas at Austin

#### Contacts: [Twitter/@GeostatsGuy](https://twitter.com/geostatsguy) | [GitHub/GeostatsGuy](https://github.com/GeostatsGuy) | [www.michaelpyrcz.com](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446)

This is a tutorial / demonstration of **Linear Regression**.  In $Python$, the $SciPy$ package, specifically the $Stats$ functions (https://docs.scipy.org/doc/scipy/reference/stats.html) provide excellent tools for efficient use of statistics.
I have previously provided this example in R and posted it on GitHub:

1. R https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.R
2. Rmd with docs https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.Rmd
3. knit as an HTML document(https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.html)

In all cases, I use the same dataset available as a comma delimited file (https://git.io/fxMql).

This tutorial includes basic, calculation of a linear regression model (only 1 predictor and 1 response), testing the significance of the parameters, calculation the parameter confidence intervals and the conditional prediction interval.

##### Caveats

I have not included all the details, specifically the test assumptions in this document.  These are included in the accompanying course notes, Lec09_Bivariate_QQ_Regres.pdf.

#### Project Goal

0. Introduction to Python in Jupyter including setting a working directory, loading data into a Pandas DataFrame.
1. Learn the basics for working with linear regresion in Python.
2. Demonstrate the efficiency of using Python and SciPy package for statistical analysis.

#### Load the required libraries

The following code loads the required libraries.

```python
import os                                                   # to set current working directory
import numpy as np                                          # arrays and matrix math
import scipy.stats as st                                    # statistical methods
import pandas as pd                                         # DataFrames
import matplotlib.pyplot as plt                             # for plotting
```

If you get a package import error, you may have to first install some of these packages. This can usually be accomplished by opening up a command window on Windows and then typing 'python -m pip install [package-name]'. More assistance is available with the respective package docs.

#### Set the working directory

I always like to do this so I don't lose files and to simplify subsequent read and writes (avoid including the full address each time).  Also, in this case make sure to place the required (see below) data file in this working directory.

```python
os.chdir("C:\PGE337")                                  # set the working directory
```

#### Loading Data

Let's load the provided dataset. 'Density_Por_data.csv' is available at https://github.com/GeostatsGuy/GeoDataSets. It is a comma delimited file with 20 porosity measures from 2 rock units from the subsurface, porosity (as a fraction). We load it with the pandas 'read_csv' function into a data frame we called 'df' and then preview it by printing a slice and by utilizing the 'head' DataFrame member function (with a nice and clean format, see below).

```python
df = pd.read_csv("Density_Por_data.csv")                # read a .csv file in as a DataFrame
print(df.iloc[0:5,:])                                       # display first 4 samples in the table as a preview
df.head()                                                   # we could also use this command for a table preview
```

        Density   Porosity
    0  1.281391  16.610982
    1  1.404932  13.668073
    2  2.346926   9.590092
    3  1.348847  15.877907
    4  2.331653   4.968240

      Density
      Porosity

      0
      1.281391
      16.610982

      1
      1.404932
      13.668073

      2
      2.346926
      9.590092

      3
      1.348847
      15.877907

      4
      2.331653
      4.968240

It is useful to review the summary statistics of our loaded DataFrame.  That can be accomplished with the 'describe' DataFrame member function.  We transpose to switch the axes for ease of visualization.

```python
df.describe().transpose()
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      Density
      105.0
      1.737917
      0.288278
      0.996736
      1.552713
      1.748788
      1.906634
      2.410560

      Porosity
      105.0
      12.531279
      3.132269
      4.966421
      10.546483
      12.411608
      14.230930
      20.964941

Here we extract the X1 and X2 unit porosity samples from the DataFrame into separate arrays called 'X1' and 'X2' for convenience.

```python
por = df['Porosity']
den = df['Density']
```

#### Liner Regression Model

Let's first calculate the linear regression model

```python
slope, intercept, r_value, p_value, std_err = st.linregress(den,por)

print('The model parameters are, slope (b1) = ' + str(round(slope,2)) + ', and the intercept (b0) = ' + str(round(intercept,2)))

```

    The model parameters are, slope (b1) = -9.1, and the intercept (b0) = 28.35

#### Model Checking

Let's test the slope with the following hypothesis test:

\begin{equation}
H_0: b_{1} = 0.0
\end{equation}

\begin{equation}
H_1: b_{1} \ne 0.0
\end{equation}

and see if we can reject this hypothesis, $H_{0}$ , that the slope parameter is equal to 0.0.  If we reject this null hypothesis, we show that the slope is meaning full and there is a linear relationship between density and porosity that we can use.

Fortunately, the $linregress$ function from the $stats$ package provides us with the two sided p-value for this test.

Let's plot the data and the model.

```python
plt.plot(den, por, 'o', label='sample data')
plt.plot(den, intercept + slope*den, 'r', label='linear regression model')
plt.xaxis
plt.legend()
plt.show()
```

We reject the null hypothesis and adopt the alternative hypothesis, $H_1$, that the slope is not equal to 0.0.

We can also observe correlation coefficient, $r$ value, and the $r^2$ value that indicates the proportion of variance that is described for our model.

```python
print('The correlation coefficient is = ' + str(round(r_value,2)) + ' and the r-squared value = ', str(round(r_value**2,2)))
```

    The correlation coefficient is = -0.84 and the r-squared value =  0.7

Let's calculate the 95% confidence interval for the slope parameter, $b_1$ of our model.

We first need the $t_{critical}$ value, given $alpha = 0.95$ and $df = n-2$.

```python
t_critical = st.t.ppf([0.025,0.975], df=len(den)-2)
print('The t critical lower and upper values are ' + str(t_critical))
print('and the t statistic is ' + str(round(slope/std_err,2)))
```

    The t critical lower and upper values are [-1.98326414  1.98326414]
    and the t statistic is -15.58
    The slope confidence interval is -9.1+/-1.16
    The slope P02.5 and P97.5 are [-10.26291645  -7.94487295]

Since the t statistic is outside the t critical lower and upper interval, we reject the null hypothesis, $h_0$, that the slope, $b_1$ is equal to 0.0.

Next let's calculate the confidence interval of the slope.  We just need our t critical and the standard error in the slope.

```python
print('The slope confidence interval is ' + str(round(slope,2)) + '+/-' + str(round(t_critical[1] * std_err,2)))
print('The slope P02.5 and P97.5 are ' + str( slope + t_critical*std_err))
```

    The slope confidence interval is -9.1+/-1.16
    The slope P02.5 and P97.5 are [-10.26291645  -7.94487295]

Let's use this model to make a prediction at all the data locations.

```python
por_hat = slope * den + intercept
plt.hist(por_hat)
plt.title("Porosity Predictions with Linear Model")
plt.xlabel('Porosity (%)')
```

It is useful to plot the predictions of porosity and porosity data vs. the density data. From this plot we can observe the linear limitation of our model and get a sense of the unexplained variance $\frac{\sum_{i=1}^{n}(y_i - \hat{y}_i)^2} {n-1}$

```python
plt.plot(den, por, 'o', label='sample data')
plt.plot(den, por_hat, 'o', label='model')
plt.title('Sample Data vs. Model')
plt.xlabel('Density (g/cm3)')
plt.ylabel('Porosity (%)')
plt.legend()
plt.show()
```

Next let's calculate the residual and check their distribution.  Residuals are the true values at the data locations minus the estimates at the data locations, $y_i - \hat{y}_i$.  We want to make sure the average is close to 0.0 and to observe the shape and spread of the residual distribution.

```python
residual = por - por_hat
plt.hist(residual)
plt.title("Residual")
plt.xlabel('Porosity True - Estimate (%)')
print('The average of the residuals is ' + str(round(np.mean(residual),2)))
```

Next we will check the residual vs. the fitted value.  We want to see if the errors are consistent over the range of fitted values.  For example, we could use this plot to identify higher error or systematic under- or overestimation over a specific range of fitted values.

```python
plt.plot(por_hat, residual, 'o')
plt.title('Residual vs. Fitted Value')
plt.xlabel('Porosity Estimate (%)')
plt.ylabel('Porosity Residual (%)')
plt.plot([5,20], [0,0], 'r')
plt.show()
```

#### Comments

Linear regression is efficient with the $stats$ functions of the $SciPy$ package in $Python$. With one line of code we can build a model and get the outputs needed to make predictions and check the model.  I hope this was helpful,

*Michael*

Michael Pyrcz, Ph.D., P.Eng. Associate Professor The Hildebrand Department of Petroleum and Geosystems Engineering, Bureau of Economic Geology, The Jackson School of Geosciences, The University of Texas at Austin
On twitter I'm the @GeostatsGuy.

```python

```


```python
%matplotlib inline
```

# Nonlinear Schrödinger equation -- Third-order soliton

This example solves the standard nonlinear Schrödinger equation, given by

\begin{align}\partial_z u = -i \frac{\beta_2}{2}\partial_t^2 u + i\gamma |u|^2 u,\end{align}

wherein $u = u(z, t)$ represents the slowly varying pulse envelope,
$\beta_2=-1$ is the second order dispersion parameter, and
$\gamma=1$ is the nonlinear parameter.  As initial condition, a
higher-order soliton

\begin{align}u_0(t) = N_{\rm{sol}} \sqrt{\frac{|\beta_2|}{\gamma \, t_0^2}} {\rm{sech}}(t/t_0),\end{align}

of duration $t_0=1$, and soliton order $N_{\rm{sol}}=3$ is
considered. The propagation is performed up to $z_{\rm{max}}=\pi/2$, i.e.
for one soliton period.

.. codeauthor:: Oliver Melchert <melchert@iqo.uni-hannover.de>

```python
import fmas
import numpy as np
from fmas.models import ModelBaseClass
from fmas.config import FTFREQ, FT, IFT, C0
from fmas.solver import SySSM
from fmas.grid import Grid
from fmas.tools import plot_evolution

class NSE(ModelBaseClass):
    def __init__(self, w, b2 = -1.0, gamma = 1.0):
        super().__init__(w, 0.5*b2*w*w)
        self.gamma = gamma

    @property
    def Lw(self):
        return 1j*self.beta_w

    def Nw(self, uw):
        ut = IFT(uw)
        return 1j*self.gamma*FT(np.abs(ut)**2*ut)

    def claw(self, i, zi, w, uw):
        return np.sum(np.abs(uw)**2)

def main():
    # -- DEFINE SIMULATION PARAMETERS
    # ... WAVEGUIDE PROPERTIES
    b2 = -1.0
    gamma = 1.
    # ... TEST PULSE PROPERTIES
    t0 = 1.                       # soliton duration
    P0 = np.abs(b2)/t0/t0/gamma   # peak-intensity
    LD = t0*t0/np.abs(b2)         # dispersion length
    N_sol = 3                     # soliton order
    # ... COMPUTATIONAL DOMAIN
    t_max = 30.
    t_num = 2**12
    z_max  = 0.5*np.pi*LD
    z_num  = 1000
    z_skip = 2

    # -- INITIALIZATION STAGE
    # ... COMPUTATIONAL DOMAIN
    grid = Grid( t_max = t_max, t_num = t_num, z_max = z_max, z_num = z_num)
    # ... NSE MODEL
    model = NSE(grid.w, b2, gamma)
    # ... Z-PROPAGATION USING SYMMETRIC SPLIT-STEP FOURIER METHOD
    solver = SySSM(model.Lw, model.Nw)
    # ... INITIAL CONDITION
    u_0t = N_sol*np.sqrt(P0)/np.cosh(grid.t/t0)
    solver.set_initial_condition(grid.w, FT(u_0t))

    # -- RUN SIMULATION
    solver.propagate(z_range = z_max, n_steps = z_num, n_skip = z_skip)

    plot_evolution( solver.z, grid.t, solver.utz,
        t_lim = (-4,4), w_lim = (-50,50), DO_T_LOG=False)

if __name__=='__main__':
    main()
```


# The Explicit Forward Time Centered Space (FTCS) Difference Equation for the Heat Equation

#### John S Butler john.s.butler@tudublin.ie   [Course Notes](https://johnsbutler.netlify.com/files/Teaching/Numerical_Analysis_for_Differential_Equations.pdf)    [Github](https://github.com/john-s-butler-dit/Numerical-Analysis-Python)
## Overview
This notebook will implement the explicit Forward Time Centered Space (FTCS) Difference method for the Heat Equation.

## The Heat Equation
The Heat Equation is the first order in time ($t$) and second order in space ($x$) Partial Differential Equation [1-3]:
$$  \frac{\partial u}{\partial t} = \frac{\partial^2 u}{\partial x^2},$$
The equation describes heat transfer on a domain
$$ \Omega = \{ t \geq 0\leq x \leq 1\}. $$
with an initial condition at time $t=0$ for all $x$ and boundary condition on the left ($x=0$) and right side ($x=1$).

## Forward Time Centered Space (FTCS) Difference method
This notebook will illustrate the Forward Time Centered Space (FTCS) Difference method for the Heat Equation with the __initial conditions__
$$ u(x,0)=2x, \ \ 0 \leq x \leq \frac{1}{2}, $$
$$ u(x,0)=2(1-x), \ \ \frac{1}{2}  \leq x \leq 1, $$
and __boundary condition__
$$ u(0,t)=0,  u(1,t)=0. $$

```python
# LIBRARY
# vector manipulation
import numpy as np
# math functions
import math
import jupytext
# THIS IS FOR PLOTTING
%matplotlib inline
import matplotlib.pyplot as plt # side-stepping mpl backend
import warnings
warnings.filterwarnings("ignore")
```

## Discete Grid
The region $\Omega$ is discretised into a uniform mesh $\Omega_h$. In the space $x$ direction into $N$ steps giving a stepsize of
$$ h=\frac{1-0}{N},$$
resulting in
$$x[i]=0+ih, \ \ \  i=0,1,...,N,$$
and into $N_t$ steps in the time $t$ direction giving a stepsize of
$$ k=\frac{1-0}{N_t}$$
resulting in
$$t[j]=0+jk, \ \ \ j=0,...,15.$$
The Figure below shows the discrete grid points for $N=10$ and $Nt=100$,  the known boundary conditions (green), initial conditions (blue) and the unknown values (red) of the Heat Equation.

```python
N=10
Nt=1000
h=1/N
k=1/Nt
r=k/(h*h)
time_steps=15
time=np.arange(0,(time_steps+.5)*k,k)
x=np.arange(0,1.0001,h)
X, Y = np.meshgrid(x, time)
fig = plt.figure()
plt.plot(X,Y,'ro');
plt.plot(x,0*x,'bo',label='Initial Condition');
plt.plot(np.ones(time_steps+1),time,'go',label='Boundary Condition');
plt.plot(x,0*x,'bo');
plt.plot(0*time,time,'go');
plt.xlim((-0.02,1.02))
plt.xlabel('x')
plt.ylabel('time (ms)')
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.title(r'Discrete Grid $\Omega_h,$ h= %s, k=%s'%(h,k),fontsize=24,y=1.08)
plt.show();
```

## Discrete Initial and Boundary Conditions

The discrete initial conditions are
$$ w[i,0]=2x[i], \ \ 0 \leq x[i] \leq \frac{1}{2} $$
$$ w[i,0]=2(1-x[i]), \ \ \frac{1}{2}  \leq x[i] \leq 1 $$
and the discrete boundary conditions are
$$ w[0,j]=0,  w[10,j]=0, $$
where $w[i,j]$ is the numerical approximation of $U(x[i],t[j])$.

The Figure below plots values of $w[i,0]$ for the inital (blue) and boundary (red) conditions for $t[0]=0.$

```python
w=np.zeros((N+1,time_steps+1))
b=np.zeros(N-1)
# Initial Condition
for i in range (1,N):
    w[i,0]=2*x[i]
    if x[i]>0.5:
        w[i,0]=2*(1-x[i])

# Boundary Condition
for k in range (0,time_steps):
    w[0,k]=0
    w[N,k]=0

fig = plt.figure(figsize=(8,4))
plt.plot(x,w[:,0],'o:',label='Initial Condition')
plt.plot(x[[0,N]],w[[0,N],0],'go',label='Boundary Condition t[0]=0')
#plt.plot(x[N],w[N,0],'go')
plt.xlim([-0.1,1.1])
plt.ylim([-0.1,1.1])
plt.title('Intitial and Boundary Condition',fontsize=24)
plt.xlabel('x')
plt.ylabel('w')
plt.legend(loc='best')
plt.show()
```

## The Explicit Forward Time Centered Space (FTCS) Difference Equation
The explicit Forwards Time Centered Space (FTCS) difference equation of the Heat Equation
is derived by discretising
$$  \frac{\partial u_{ij}}{\partial t} = \frac{\partial^2 u_{ij}}{\partial x^2},$$
around $(x_i,t_{j})$ giving the difference equation
$$
\frac{w_{ij+1}-w_{ij}}{k}=\frac{w_{i+1j}-2w_{ij}+w_{i-1j}}{h^2},
$$
Rearranging the equation we get
$$
w_{ij+1}=rw_{i-1j}+(1-2r)w_{ij}+rw_{i+1j},
$$
for $i=1,...9$ where $r=\frac{k}{h^2}$.

This gives the formula for the unknown term $w_{ij+1}$ at the $(ij+1)$ mesh points
in terms of $x[i]$ along the jth time row.

Hence we can calculate the unknown pivotal values of $w$ along the first row of $j=1$ in terms of the known boundary conditions.

This can be written in matrix form
$$ \mathbf{w}_{j+1}=A\mathbf{w}_{j} +\mathbf{b}_{j} $$
for which $A$ is a $9\times9$ matrix:
$$
\left(\begin{array}{c}
w_{1j+1}\\
w_{2j+1}\\
w_{3j+1}\\
w_{4j+1}\\
w_{5j+1}\\
w_{6j+1}\\
w_{7j+1}\\
w_{8j+1}\\
w_{9j+1}\\
\end{array}\right).
=\left(\begin{array}{cccc cccc}
1-2r&r& 0&0&0 &0&0&0\\
r&1-2r&r&0&0&0 &0&0&0\\
0&r&1-2r &r&0&0& 0&0&0\\
0&0&r&1-2r &r&0&0& 0&0\\
0&0&0&r&1-2r &r&0&0& 0\\
0&0&0&0&r&1-2r &r&0&0\\
0&0&0&0&0&r&1-2r &r&0\\
0&0&0&0&0&0&r&1-2r&r\\
0&0&0&0&0&0&0&r&1-2r\\
\end{array}\right)
\left(\begin{array}{c}
w_{1j}\\
w_{2j}\\
w_{3j}\\
w_{4j}\\
w_{5j}\\
w_{6j}\\
w_{7j}\\
w_{8j}\\
w_{9j}\\
\end{array}\right)+
\left(\begin{array}{c}
rw_{0j}\\
0\\
0\\
0\\
0\\
0\\
0\\
0\\
rw_{10j}\\
\end{array}\right).
$$
It is assumed that the boundary values $w_{0j}$ and $w_{10j}$ are known for $j=1,2,...$, and $w_{i0}$ for $i=0,...,10$ is the initial condition.

The Figure below shows the values of the $9\times 9$  matrix in colour plot form for $r=\frac{k}{h^2}$.

```python
A=np.zeros((N-1,N-1))
for i in range (0,N-1):
    A[i,i]=1-2*r # DIAGONAL

for i in range (0,N-2):
    A[i+1,i]=r # UPPER DIAGONAL
    A[i,i+1]=r # LOWER DIAGONAL

fig = plt.figure(figsize=(6,4));
#plt.matshow(A);
plt.imshow(A,interpolation='none');
plt.xticks(np.arange(N-1), np.arange(1,N-0.9,1));
plt.yticks(np.arange(N-1), np.arange(1,N-0.9,1));
clb=plt.colorbar();
clb.set_label('Matrix elements values');
clb.set_clim((-1,1));
plt.title('Matrix r=%s'%(np.round(r,3)),fontsize=24)
fig.tight_layout()
plt.show();
```

## Results
To numerically approximate the solution at $t[1]$ the matrix equation becomes
$$ \mathbf{w}_{1}=A\mathbf{w}_{0} +\mathbf{b}_{0} $$
where all the right hand side is known.
To approximate solution at time $t[2]$ we use the matrix equation
$$ \mathbf{w}_{2}=A\mathbf{w}_{1} +\mathbf{b}_{1}. $$
Each set of numerical solutions $w[i,j]$ for all $i$ at the previous time step is used to approximate the solution $w[i,j+1]$.

The Figure below shows the numerical approximation $w[i,j]$ of the Heat Equation using the FTCS method at $x[i]$ for $i=0,...,10$ and time steps $t[j]$ for $j=1,...,15$. The left plot shows the numerical approximation $w[i,j]$ as a function of $x[i]$ with each color representing the different time steps $t[j]$. The right plot shows the numerical approximation $w[i,j]$ as colour plot as a function of $x[i]$, on the $x[i]$ axis and time $t[j]$ on the $y$ axis.

For $r>\frac{1}{2}$ the method is unstable resulting a solution that oscillates unnaturally between positive and negative values for each time step.

```python
fig = plt.figure(figsize=(12,6))

plt.subplot(121)
for j in range (1,time_steps+1):
    b[0]=r*w[0,j-1]
    b[N-2]=r*w[N,j-1]
    w[1:(N),j]=np.dot(A,w[1:(N),j-1])
    plt.plot(x,w[:,j],'o:',label='t[%s]=%s'%(j,np.round(time[j],4)))
plt.xlabel('x')
plt.ylabel('w')
#plt.legend(loc='bottom', bbox_to_anchor=(0.5, -0.1))
plt.legend(bbox_to_anchor=(-.4, 1), loc=2, borderaxespad=0.)

plt.subplot(122)
plt.imshow(w.transpose())
plt.xticks(np.arange(len(x)), x)
plt.yticks(np.arange(len(time)), np.round(time,4))
plt.xlabel('x')
plt.ylabel('time')
clb=plt.colorbar()
clb.set_label('Temperature (w)')
plt.suptitle('Numerical Solution of the  Heat Equation r=%s'%(np.round(r,3)),fontsize=24,y=1.08)
fig.tight_layout()
plt.show()
```

## Local Trunction Error
The local truncation error of the classical explicit difference approach to
\begin{equation}
\frac{\partial U}{\partial t} - \frac{\partial^2 U}{\partial x^2}=0,
\end{equation}
with

\begin{equation}
F_{ij}(w)=\frac{w_{ij+1}-w_{ij}}{k}-\frac{w_{i+1j}-2w_{ij}+w_{i-1j}}{h^2}=0,
\end{equation}
is
\begin{equation}
T_{ij}=F_{ij}(U)=\frac{U_{ij+1}-U_{ij}}{k}-\frac{U_{i+1j}-2U_{ij}+U_{i-1j}}{h^2},
\end{equation}
By Taylors expansions we have
\begin{eqnarray*}
U_{i+1j}&=&U((i+1)h,jk)=U(x_i+h,t_j)\\
&=&U_{ij}+h\left(\frac{\partial U}{\partial x} \right)_{ij}+\frac{h^2}{2}\left(\frac{\partial^2 U}{\partial x^2} \right)_{ij}+\frac{h^3}{6}\left(\frac{\partial^3 U}{\partial x^3} \right)_{ij} +...\\
U_{i-1j}&=&U((i-1)h,jk)=U(x_i-h,t_j)\\
&=&U_{ij}-h\left(\frac{\partial U}{\partial x} \right)_{ij}+\frac{h^2}{2}\left(\frac{\partial^2 U}{\partial x^2} \right)_{ij}-\frac{h^3}{6}\left(\frac{\partial^3 U}{\partial x^3} \right)_{ij} +...\\
U_{ij+1}&=&U(ih,(j+1)k)=U(x_i,t_j+k)\\
&=&U_{ij}+k\left(\frac{\partial U}{\partial t} \right)_{ij}+\frac{k^2}{2}\left(\frac{\partial^2 U}{\partial t^2} \right)_{ij}+\frac{k^3}{6}\left(\frac{\partial^3 U}{\partial t^3} \right)_{ij} +...
\end{eqnarray*}

substitution into the expression for $T_{ij}$ then gives

\begin{eqnarray*}
T_{ij}&=&\left(\frac{\partial U}{\partial t} - \frac{\partial^2 U}{\partial x^2} \right)_{ij}+\frac{k}{2}\left(\frac{\partial^2 U}{\partial t^2} \right)_{ij}
-\frac{h^2}{12}\left(\frac{\partial^4 U}{\partial x^4} \right)_{ij}\\
& &	+\frac{k^2}{6}\left(\frac{\partial^3 U}{\partial t^3} \right)_{ij}
-\frac{h^4}{360}\left(\frac{\partial^6 U}{\partial x^6} \right)_{ij}+ ...
\end{eqnarray*}
But $U$ is the solution to the differential equation so
\begin{equation} \left(\frac{\partial U}{\partial t} - \frac{\partial^2 U}{\partial x^2} \right)_{ij}=0,\end{equation}

the principal part of the local truncation error is

\begin{equation}
\frac{k}{2}\left(\frac{\partial^2 U}{\partial t^2} \right)_{ij}-\frac{h^2}{12}\left(\frac{\partial^4 U}{\partial x^4} \right)_{ij}.
\end{equation}

Hence the truncation error is
\begin{equation}
T_{ij}=O(k)+O(h^2).
\end{equation}

## Stability Analysis

To investigating the stability of the fully explicit FTCS difference method of the Heat Equation, we will use the von Neumann method.
The FTCS difference equation is:
$$\frac{1}{k}(w_{pq+1}-w_{pq})=\frac{1}{h_x^2}(w_{p-1q}-2w_{pq}+w_{p+1q}),$$
approximating
$$\frac{\partial U}{\partial t}=\frac{\partial^2 U}{\partial x^2}$$

at $(ph,qk)$. Substituting $w_{pq}=e^{i\beta x}\xi^{q}$ into the difference equation gives:
$$e^{i\beta ph}\xi^{q+1}-e^{i\beta ph}\xi^{q}=r\{e^{i\beta (p-1)h}\xi^{q}-2e^{i\beta ph}\xi^{q}+e^{i\beta (p+1)h}\xi^{q} \}
$$

where $r=\frac{k}{h_x^2}$. Divide across by $e^{i\beta (p)h}\xi^{q}$ leads to

$$ \xi-1=r(e^{i\beta (-1)h} -2+e^{i\beta h}), $$

$$\xi= 1+r (2\cos(\beta h)-2),$$

$$\xi=1-4r(\sin^2(\beta\frac{h}{2})).$$
Hence
$$\left| 1-4r(\sin^2(\beta\frac{h}{2}) )\right|\leq 1$$
for this to hold
$$ 4r(\sin^2(\beta\frac{h}{2}) )\leq 2 $$
which means
$$ r\leq \frac{1}{2}. $$
therefore the equation is conditionally stable as $0 < \xi \leq 1$ for $r<\frac{1}{2}$ and all $\beta$ .

## References
[1] G D Smith Numerical Solution of Partial Differential Equations: Finite Difference Method Oxford 1992

[2] Butler, J. (2019). John S Butler Numerical Methods for Differential Equations. [online] Maths.dit.ie. Available at: http://www.maths.dit.ie/~johnbutler/Teaching_NumericalMethods.html [Accessed 14 Mar. 2019].

[3] Wikipedia contributors. (2019, February 22). Heat equation. In Wikipedia, The Free Encyclopedia. Available at: https://en.wikipedia.org/w/index.php?title=Heat_equation&oldid=884580138 [Accessed 14 Mar. 2019].

```python

```

Chapter 08 - Heat Equations/.ipynb_checkpoints/801_Heat Equation- FTCS-checkpoint.ipynb


# Butcher Tableau

##### Calculates the Butcher Tableau for a given order.

##### Calculates A, B, and C for a given order and method.

##### This class calculates the transformation matrix, T, and its respective inverse. (However small errors may exist in the method used)

##### It can also calculate the interpolator coefficients, P, for a cubic polynomial spline.

# Example

```python
import butchertableau as bt
order = 5
X = butcher(order, 15)
A, B, C = X.radau()
Ainv = X.inv(A)
T, TI = X.Tmat(Ainv)
P = X.P(C)
```

# Results

$\begin{equation} A  = \begin{bmatrix} 0.19681547722366058, \ -0.06553542585019845, \ 0.023770974348220134, \\ 0.39442431473908746, \ 0.29207341166522777, \ -0.04154875212599763, \\ 0.376403062700466, \ 0.512485826188424, \ 0.111111111111110 \end{bmatrix} \end{equation}$

$\begin{equation} B = \begin{bmatrix} 0.376403062700466, \ 0.512485826188424, \ 0.111111111111110  \end{bmatrix} \end{equation}$

$\begin{equation} C = \begin{bmatrix} 0.15505102572168228, \ 0.6449489742783174, \ 1.0 \end{bmatrix} \end{equation}$

$\begin{equation} A^{-1} = \begin{bmatrix} 3.22474487139158, \ 1.16784008469041, \ -0.253197264742181 \\ -3.56784008469039, \ 0.775255128608406, \ 1.05319726474218 \\ 5.53197264742194, \ -7.53197264742187, \ 5.00000000000001 \end{bmatrix} \end{equation}$

$\begin{equation} T = \begin{bmatrix} 0.0944387624889745, \ -0.141255295020953, \ 0.0300291941051473 \\ 0.250213122965334, \ 0.204129352293798, \ -0.38294211275726 \\ 1.0, \ 1.0, \ 0.0 \end{bmatrix} \end{equation}$

$\begin{equation} T^{-1} = \begin{bmatrix} 4.178718591551935, \ 0.32768282076106514, \ 0.5233764454994487 \\ -4.178718591551935, \ -0.3276828207610649, \ 0.476623554500551 \\ 0.5028726349458223, \ -2.571926949855616, \ 0.5960392048282263 \end{bmatrix} \end{equation}$

$\begin{equation} P = \begin{bmatrix} 10.048809399827414, \ -25.62959144707665, \ 15.580782047249254 \\ -1.38214273316075, \ 10.29625811374331, \ -8.914115380582556 \\ 0.3333333333333328, \ -2.6666666666666616, \ 3.333333333333328 \end{bmatrix} \end{equation}$

```python

```


# $H(div, \Omega)$ Elliptic Problems

$\newcommand{\dd}{\,{\rm d}}$
$\newcommand{\uu}{\mathbf{u}}$
$\newcommand{\vv}{\mathbf{v}}$
$\newcommand{\nn}{\mathbf{n}}$
$\newcommand{\ff}{\mathbf{f}}$
$\newcommand{\Hdivzero}{\mathbf{H}_0(\mbox{div}, \Omega)}$
$\newcommand{\Div}{\nabla \cdot}$
$\newcommand{\Grad}{\boldsymbol{\nabla}}$

Let $\Omega \subset \mathbb{R}^d$ be an open Liptschitz bounded set, and we look for the solution of the following problem
\begin{align}
  \left\{
  \begin{array}{rl}
    - \Grad \Div \uu + \mu \uu &= \ff, \quad \Omega
    \\
    \uu \times \nn &= 0, \quad \partial\Omega
  \end{array} \right.
  \label{eq:elliptic_hdiv}
\end{align}
where $\ff \in \mathbf{L}^2(\Omega)$,  $\mu \in L^\infty(\Omega)$ and there exists $\mu_0 > 0$ such that $\mu \geq \mu_0$ almost everywhere.
We take the Hilbert space $V := \Hdivzero$, in which case the variational formulation corresponding to \eqref{eq:elliptic_hdiv} writes

---
Find $\uu \in V$ such that
\begin{align}
  a(\uu,\vv) = l(\vv) \quad \forall \vv \in V
\label{eq:abs_var_elliptic_hdiv}
\end{align}
where
\begin{align}
\left\{
\begin{array}{rll}
a(\uu, \vv) &:= \int_{\Omega} \Div \uu ~ \Div \vv + \int_{\Omega} \mu \uu \cdot \vv, & \forall \uu, \vv \in V  \\
l(\vv) &:= \int_{\Omega} \vv \cdot \ff, & \forall \vv \in V
\end{array} \right.
\end{align}

---

We recall that in $\Hdivzero$, the bilinear form $a$ is equivalent to the inner product and is therefor continuous and coercive. Hence, our abstract theory applies and there exists a unique solution to the problem \eqref{eq:abs_var_elliptic_hdiv}.

**TODO - compute exact solution and associated rhs**

```python
import numpy as np
from sympy import pi, cos, sin, sqrt, Matrix, Tuple, lambdify
from scipy.sparse.linalg import spsolve
from scipy.sparse.linalg import gmres as sp_gmres
from scipy.sparse.linalg import minres as sp_minres
from scipy.sparse.linalg import cg as sp_cg
from scipy.sparse.linalg import bicg as sp_bicg
from scipy.sparse.linalg import bicgstab as sp_bicgstab

from sympde.calculus import grad, dot, inner, div, curl, cross
from sympde.topology import NormalVector
from sympde.topology import ScalarFunctionSpace, VectorFunctionSpace
from sympde.topology import ProductSpace
from sympde.topology import element_of, elements_of
from sympde.topology import Square
from sympde.expr import BilinearForm, LinearForm, integral
from sympde.expr import Norm
from sympde.expr import find, EssentialBC

from psydac.fem.basic          import FemField
from psydac.fem.vector         import ProductFemSpace
from psydac.api.discretization import discretize
from psydac.linalg.utilities   import array_to_stencil
from psydac.linalg.iterative_solvers import pcg, bicg
```

```python
# ... abstract model
domain = Square('A')
B_dirichlet_0 = domain.boundary

x, y   = domain.coordinates

alpha    = 1.
uex      = Tuple(sin(pi*y), sin(pi*x)*cos(pi*y))
f        = Tuple(alpha*sin(pi*y) - pi**2*sin(pi*y)*cos(pi*x) + pi**2*sin(pi*y),
                 alpha*sin(pi*x)*cos(pi*y) + pi**2*sin(pi*x)*cos(pi*y))

V  = VectorFunctionSpace('V', domain, kind='hdiv')

u  = element_of(V, name='u')
v  = element_of(V, name='v')
F  = element_of(V, name='F')

# Bilinear form a: V x V --> R
a   = BilinearForm((u, v), integral(domain, div(u)*div(v) + alpha*dot(u,v)))

nn   = NormalVector('nn')
a_bc = BilinearForm((u, v), integral(domain.boundary, 1e30 * dot(u, nn) * dot(v, nn)))

# Linear form l: V --> R
l = LinearForm(v, integral(domain, dot(f,v)))

# l2 error
error   = Matrix([F[0]-uex[0],F[1]-uex[1]])
l2norm  = Norm(error, domain, kind='l2')
```

```python
ncells = [2**3, 2**3]
degree = [2, 2]
```

```python
# Create computational domain from topological domain
domain_h = discretize(domain, ncells=ncells)

# Discrete spaces
Vh = discretize(V, domain_h, degree=degree)

# Discretize bi-linear and linear form
a_h    = discretize(a, domain_h, [Vh, Vh])
a_bc_h = discretize(a_bc, domain_h, [Vh, Vh])

l_h          = discretize(l, domain_h, Vh)
l2_norm_h    = discretize(l2norm, domain_h, Vh)

M = a_h.assemble() + a_bc_h.assemble()
b = l_h.assemble()
```

```python
# Solve linear system
sol, info  = pcg(M ,b, pc='jacobi', tol=1e-8)

uh       = FemField( Vh, sol )
l2_error = l2_norm_h.assemble(F=uh)
print(l2_error)
```

    3.7564893755638256

```python

```


# Angular velocity in 3D movements

> Renato Naville Watanabe, Marcos Duarte
> [Laboratory of Biomechanics and Motor Control](http://pesquisa.ufabc.edu.br/bmclab)
> Federal University of ABC, Brazil

Contents

1  Axis of rotation

2  Computing the angular velocity

2.1  1 ) 3D pendulum bar

3  Further reading

4  Problems

5  References

An usual problem found in Biomechanics (and Mechanics in general) is to find the angular velocity of an object. We consider that a basis
$\hat{\boldsymbol e_1}$, $\hat{\boldsymbol  e_2}$ and  $\hat{\boldsymbol  e_3}$ is attached to the body and is known. To learn how to find a basis of a frame of reference, see [this notebook](https://nbviewer.jupyter.org/github/BMClab/bmc/blob/master/notebooks/ReferenceFrame.ipynb).

## Axis of rotation

As in the planar movement, the angular velocity is a vector perpendicular to the rotation. The line in the direction of the angular velocity vector is known as the axis of rotation.

The rotation beween two frames of reference is characterized by the rotation matrix $R$ obtained by stacking the versors  $\hat{\boldsymbol e_1}$, $\hat{\boldsymbol e_2}$ and  $\hat{\boldsymbol e_3}$ in each column of the matrix (for a revision on rotation matrices see [this](https://nbviewer.jupyter.org/github/bmclab/bmc/blob/master/notebooks/Transformation2D.ipynb) and [this](https://nbviewer.jupyter.org/github/bmclab/bmc/blob/master/notebooks/Transformation3D.ipynb) notebooks).

A vector in the direction of the axis of rotation is a vector that does not changes the position after the rotation. That is:

\begin{equation}
v = Rv
\end{equation}

This vector is the eigenvector of the rotation matrix  $R$ with eigenvalue equal to one.

Below the yellow arrow indicates the axis of rotation of the rotation between the position of the reference frame   $\hat{\boldsymbol i}$, $\hat{\boldsymbol j}$ and  $\hat{\boldsymbol k}$ and the reference frame of  $\hat{\boldsymbol e_1}$, $\hat{\boldsymbol e_2}$ and  $\hat{\boldsymbol e_3}$.

```python
from IPython.core.display import Math, display
import sympy as sym
import sys
sys.path.insert(1, r'./../functions')  # add to pythonpath
%matplotlib notebook
from CCSbasis import CCSbasis
import numpy as np

a, b, g = sym.symbols('alpha, beta, gamma')

# Elemental rotation matrices of xyz in relation to XYZ:
RX = sym.Matrix([[1, 0, 0],
                 [0, sym.cos(a), -sym.sin(a)],
                 [0, sym.sin(a), sym.cos(a)]])
RY = sym.Matrix([[sym.cos(b), 0, sym.sin(b)],
                 [0, 1, 0],
                 [-sym.sin(b), 0, sym.cos(b)]])
RZ = sym.Matrix([[sym.cos(g), -sym.sin(g), 0],
                 [sym.sin(g), sym.cos(g), 0],
                 [0, 0, 1]])

# Rotation matrix of xyz in relation to XYZ:
R = RY@RX@RZ
R = sym.lambdify((a, b, g), R, 'numpy')

alpha = np.pi/4
beta = np.pi/4
gamma = np.pi/4

R = R(alpha, beta, gamma)

e1 = np.array([[1, 0, 0]])
e2 = np.array([[0, 1, 0]])
e3 = np.array([[0, 0, 1]])

basis = np.vstack((e1, e2, e3))
basisRot = R@basis
lv, v = np.linalg.eig(R)

axisOfRotation = [np.real(np.squeeze(v[:, np.abs(lv-1) < 1e-6]))]

CCSbasis(Oijk=np.array([0, 0, 0]), Oxyz=np.array([0, 0, 0]), ijk=basis.T,
         xyz=basisRot.T, vector=True, point=axisOfRotation);
```

    <IPython.core.display.Javascript object>

## Computing the angular velocity

The angular velocity $\vec{\boldsymbol\omega}$ is in the direction of the axis of rotation (hence it is parallel to the axis of rotation) and can be described in the basis fixed in the body:

\begin{equation}
    \vec{\boldsymbol{\omega}} = \omega_1\hat{\boldsymbol{e_1}} + \omega_2\hat{\boldsymbol{e_2}} + \omega_3\hat{\boldsymbol{e_3}}
\end{equation}

So, we must find  $\omega_1$, $\omega_2$ and  $\omega_3$.

First we will express the angular velocity  $\vec{\boldsymbol{\omega}}$ in  terms of these derivatives.

Remember that the angular velocity is described as a vector in the orthogonal plane of the rotation. ($\vec{\boldsymbol{\omega_1}} = \frac{d\theta_1}{dt}\hat{\boldsymbol{e_1}}$, $\vec{\boldsymbol{\omega_2}} = \frac{d\theta_2}{dt}\hat{\boldsymbol{e_2}}$ and  $\vec{\boldsymbol{\omega_3}} = \frac{d\theta_3}{dt}\hat{\boldsymbol{e_3}}$).

Note also that the derivative of the angle  $\theta_1$ can be described as the projection of the vector  $\frac{d\hat{\boldsymbol{e_2}}}{dt}$ on the vector  $\hat{\boldsymbol{e_3}}$.

This can be written by using the scalar product between these vectors:  $\frac{d\theta_1}{dt} = \frac{d\hat{\boldsymbol{e_2}}}{dt}\cdot \hat{\boldsymbol{e_3}}$.

Similarly, the same is valid for the angles in the other two directions:  $\frac{d\theta_2}{dt} = \frac{d\hat{\boldsymbol{e_3}}}{dt}\cdot \hat{\boldsymbol{e_1}}$ and  $\frac{d\theta_3}{dt} = \frac{d\hat{\boldsymbol{e_1}}}{dt}\cdot \hat{\boldsymbol{e_2}}$.

So, we can write the angular velocity as:

\begin{equation}
    \vec{\boldsymbol{\omega}} =  \left(\frac{d\hat{\boldsymbol{e_2}}}{dt}\cdot \hat{\boldsymbol{e_3}}\right) \hat{\boldsymbol{e_1}} + \left(\frac{d\hat{\boldsymbol{e_3}}}{dt}\cdot \hat{\boldsymbol{e_1}}\right) \hat{\boldsymbol{e_2}} + \left(\frac{d\hat{\boldsymbol{e_1}}}{dt}\cdot \hat{\boldsymbol{e_2}}\right) \hat{\boldsymbol{e_3}}
\end{equation}

Note that the angular velocity $\vec{\boldsymbol\omega}$ is expressed in the reference frame of the object. If you want it described as a linear combination of the versors of the global basis $\hat{\boldsymbol{i}}$, $\hat{\boldsymbol{j}}$ and $\hat{\boldsymbol{k}}$, just multiply the vector $\vec{\boldsymbol\omega}$ by the rotation matrix formed by stacking each versor in a column of the rotation matrix (for a revision on rotation matrices see [this](https://nbviewer.jupyter.org/github/bmclab/bmc/blob/master/notebooks/Transformation2D.ipynb) and [this](https://nbviewer.jupyter.org/github/bmclab/bmc/blob/master/notebooks/Transformation3D.ipynb) notebooks).

### 3D pendulum bar

At the file '../data/3Dpendulum.txt' there are 3 seconds of data of 3 points of a three-dimensional cylindrical pendulum. It can move in every direction and has a motor at the upper part of the cylindrical bar producing torques to move the bar.

The point m1 is at the upper part of the cylinder and is the origin of the system.

The point m2 is at the center of mass of the cylinder.

The point m3 is a point at the surface of the cylinder.

Below we compute its angular velocity.

First we load the file.

```python
import numpy as np
import matplotlib.pyplot as plt
np.set_printoptions(precision=4, suppress=True)

data = np.loadtxt('../data/3dPendulum.txt', skiprows=1, delimiter = ',')
```

And separate each mark in a variable.

```python
t  = data[:, 0]
m1 = data[:, 1:4]
m2 = data[:, 4:7]
m3 = data[:, 7:]
dt = t[1]
```

Now, we form the basis  $\hat{\boldsymbol e_1}$, $\hat{\boldsymbol e_2}$ and  $\hat{\boldsymbol e_3}$.

```python
V1 = m2 - m1
e1 = V1 / np.linalg.norm(V1, axis=1, keepdims=True)

V2 = m3-m2

V3 = np.cross(V2, V1)
e2 = V3 / np.linalg.norm(V3, axis=1, keepdims=True)

e3 = np.cross(e1, e2)
```

Below, we compute the derivative of each of the versors.

```python
de1dt = np.diff(e1, axis=0) / dt
de2dt = np.diff(e2, axis=0) / dt
de3dt = np.diff(e3, axis=0) / dt
```

Here we compute each of the components  $\omega_1$, $\omega_2$ and  $\omega_3$ of the angular velocity  $\vec{\boldsymbol \omega}$ by using the scalar product.

```python
omega1 = np.sum(de2dt*e3[0:-1, :], axis=1).reshape(-1, 1)
omega2 = np.sum(de3dt*e1[0:-1, :], axis=1).reshape(-1, 1)
omega3 = np.sum(de1dt*e2[0:-1, :], axis=1).reshape(-1, 1)
```

Finally, the angular velocity vector  $\vec{\boldsymbol \omega}$ is formed by stacking the three components together.

```python
omega = np.hstack((omega1, omega2, omega3))
```

```python
%matplotlib inline
fig, ax = plt.subplots(1, 1, figsize=(8, 4))
ax.plot(t[:-1], omega)
ax.set_xlabel('Time [s]')
ax.set_ylabel('Angular velocity [rad/s]')
ax.legend(labels=['$ω_1$', '$ω_2$', '$ω_3$'])
plt.tight_layout()
```

## Further reading

 - Read pages 66-92 of the 1st chapter of the [Ruina and Rudra's book](http://ruina.tam.cornell.edu/Book/index.html) for a review of Gram-Schmidt, dot product and cross product.
 - [Scalar and Vector](https://nbviewer.jupyter.org/github/bmclab/bmc/blob/master/notebooks/ScalarVector.ipynb)

## Problems

1) Initially, the lateral malleolus, medial malleolus, fibular head and medial condyle of the leg have the following positions (described in the laboratory coordinate system with coordinates  𝑥,𝑦,𝑧  in cm, the  𝑥  axis points forward and the  𝑦  axes points upward): lateral malleolus (lm = [2.92, 10.10, 18.85]), medial malleolus (mm = [2.71, 10.22, 26.52]), fibular head (fh = [5.05, 41.90, 15.41]), and medial condyle (mc = [8.29, 41.88, 26.52]). After 0.05 seconds the markers have the following positions: lateral malleolus (lm = [2.95, 10.19, 18.41]), medial malleolus (mm = [3.16, 10.04, 26.10]), fibular head (fh = [4.57, 42.13, 15.97]), and medial condyle (mc = [8.42, 41.76, 26.90]).

Find the angular velocity of of the leg.

## References

- Kane T, Levinson D (1985) [Dynamics: Theory and Applications](https://ecommons.cornell.edu/handle/1813/638). McGraw-Hill, Inc


# Assignment 4: Word Embeddings

Welcome to the fourth programming assignment of Course 2. In this assignment we will show you how to compute the word embeddings. In Natural Language Processing (NLP) we can not only rely on counting the number of positive words and negative words, as we did in the last course using logistic regression. Instead we will try to find a way to represent each word by a vector. The vector could then represent syntactic (i.e. parts of speech) and semantic (i.e. meaning) structures. In this assignment you will explore a classic way of learning embeddings, or representations, of words by using a famous model called the continuous bag of words (CBOW) model. By completing this assignment you will:

- Train word vectors from scratch.
- Learn how to create batches of data.
- Understand how backpropagation works.
- Plot and visualize your learned word vectors.

Because it will take a while to train your CBOW model, you will code the model and make sure to get the expected outputs. We will give you some slightly pre-trained vectors and then you will fine tune them on the Shakespeare dataset.

Knowing how to train these models will give you a better understanding of word vectors, which are building blocks to many applications in natural language processing.

# 1.0 The Continuous bag of words model

Let's take a look at the following sentence: **'I am happy because I am learning'**. In continuous bag of words modeling we try to predict the center word given a few context words. For example, if you were to choose a context (say $C = 2$), then you would try to predict the word **happy** given the context: {I, am, because, I}. In other words, you have

$$context = [I,am, because, I]$$
$$target = happy$$

The structure of your model will look like this:

 Figure 1

Where $\bar x$ is the average one hot vector for all the context word encodings.

 Figure 2

Once you have encoded all the context words, you can use $\bar x$ as the input to your model. The architecture you will be implementing is as follows:

\begin{align}
 h &= W_1 \  X + b_1  \tag{1} \\
 a &= ReLU(h)  \tag{2} \\
 z &= W_2 \  a + b_2   \tag{3} \\
 \hat y &= softmax(z)   \tag{4} \\
\end{align}

```python
# Import Python libraries and helper functions (in utils2)
import nltk
from nltk.tokenize import word_tokenize
import numpy as np
from collections import Counter
from utils2 import sigmoid, get_batches, compute_pca, get_dict
```

```python
# Download sentence tokenizer
nltk.data.path.append('.')
```

```python
# Load, tokenize and process the data
import re                                                           #  Load the Regex-modul
data = open('shakespeare.txt').read()                               #  Read in the data
data = re.sub(r'[,!?;-]', '.',data)                                 #  Punktuations are replaced by .
data = nltk.word_tokenize(data)                                     #  Tokenize string to words
data = [ ch.lower() for ch in data if ch.isalpha() or ch == '.']    #  Lower case and drop non-alphabetical tokens
print("Number of tokens:", len(data),'\n', data[:15])               #  print data sample
```

    Number of tokens: 60975
     ['o', 'for', 'a', 'muse', 'of', 'fire', '.', 'that', 'would', 'ascend', 'the', 'brightest', 'heaven', 'of', 'invention']

```python
# Compute the frequency distribution of the words in the dataset (vocabulary)
fdist = nltk.FreqDist(word for word in data)
print("Size of vocabulary: ",len(fdist) )
print("Most frequent tokens: ",fdist.most_common(20) ) # print the 20 most frequent words and their freq.
```

    Size of vocabulary:  5777
    Most frequent tokens:  [('.', 9630), ('the', 1521), ('and', 1394), ('i', 1257), ('to', 1159), ('of', 1093), ('my', 857), ('that', 781), ('in', 770), ('a', 752), ('you', 748), ('is', 630), ('not', 559), ('for', 467), ('it', 460), ('with', 441), ('his', 434), ('but', 417), ('me', 417), ('your', 397)]

#### Mapping words to indices and indices to words
We provide a helper function to create a dictionary that maps words to indices and indices to words.

```python
# get_dict creates two dictionaries, converting words to indices and viceversa.
word2Ind, Ind2word = get_dict(data)
V = len(word2Ind)
print("Size of vocabulary: ", V)
```

    Size of vocabulary:  5777

```python
# example of word to index mapping
print("Index of the word 'king' :  ",word2Ind['king'] )
print("Word which has index 2743:  ",Ind2word[2743] )
```

    Index of the word 'king' :   2744
    Word which has index 2743:   kinds

# 2.0 Training the Model

###  Initializing the model

You will now initialize two matrices and two vectors.
 The first matrix ($W_1$) is of dimension $V \times N$, where $V$ is the number of words in your vocabulary and $N$ is the dimension of your word vector.

The second matrix ($W_2$) is of dimension $N \times V$.
The two vectors, $b_1$ and $b_2$ are of dimension $N\times 1$ and $V\times 1$ respectively (column vectors). $b_1$ and $b_2$ are the bias vectors of the linear layers from matrices $W_1$ and $W_2$.
The overall structure of the model will look as in Figure 1, but at this stage we are just initializing the parameters.

```python
np.random.seed(1)
# UNQ_C1 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# UNIT TEST COMMENT: Candidate for Table Driven Tests
# GRADED FUNCTION: initialize_model
def initialize_model(N,V):
    ### START CODE HERE ###
    W1 = np.random.rand(N,V)
    W2 = np.random.rand(V,N)
    b1 = np.random.rand(N,1)
    b2 = np.random.rand(V,1)
    ### END CODE HERE ###
    return W1, W2, b1, b2
```

```python
# Test your function example.
N = 4
V = 10
W1, W2, b1, b2 = initialize_model(N,V)
assert W1.shape == ((N,V))
assert W2.shape == ((V,N))
print(W1.shape, W2.shape, b1.T)
```

    (4, 10) (10, 4) [[0.88330609 0.62367221 0.75094243 0.34889834]]

* **Expected Output:**  (4, 10) (10, 4) [[0.88330609 0.62367221 0.75094243 0.34889834]]

### Softmax
Before we can start training the model, we need to implement the softmax function as defined in equation 5:

$$ \text{softmax}(z_i) = \frac{e^{z_i} }{\sum_{i} e^{z_i} }  \tag{5} $$

**Instructions**: Implement the softmax function below.

```python
# UNQ_C2 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# UNIT TEST COMMENT: Candidate for Table Driven Tests
# GRADED FUNCTION: softmax
def softmax(x):
    """Compute softmax values for each sets of scores in x."""

    ### START CODE HERE (REPLACE INSTANCES OF 'None' WITH YOUR OWN CODE) ###
    s = (np.exp(x) / np.sum(np.exp(x), axis=0)).T

    ### END CODE HERE ###
    return s
```

```python
# testing the softmax function
print(softmax([0,3,-2]))
```

    [0.04712342 0.94649912 0.00637746]

**Expected Ouput:**  array([0.04712342, 0.94649912, 0.00637746])

### Forward propagation

Implement the forward propagation $z$ according to equations (1) to (3).

For that, you will use as activation the Rectified Linear Unit (ReLU) given by:

$$f(h)=\max (0,h) \tag{6}$$

```python
# UNQ_C3 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# UNIT TEST COMMENT: Candidate for Table Driven Tests
# GRADED FUNCTION: forward_prop
def forward_prop(x, W1, W2, b1, b2):
    '''
    Inputs:
        x:  average one hot vector for the context
        W1, W2, b1, b2:  matrices and biases to be learned
     Outputs:
        z:  output score vector
        h:  output of first hidden layer
    '''
    ### START CODE HERE (REPLACE INSTANCES OF 'None' WITH YOUR OWN CODE) ###
    ### (YOU WILL NEED TO ADD CODE IN ADDITION TO JUST REPLACING THE 'None' VALUES) ###

    # Calculate h and then used it to calculate z
    h = np.dot(W1,x) + b1
    h[h<=0.0] = 0.0                     # this is the ReLU function
    z = np.dot(W2, h) + b2

#  h and z must be column vector for the solution to be right
    h = h.T
    z = z.T

    ### END CODE HERE ###

    return z, h
```

## Cost function

We have implemented the *cross-entropy* cost function for you.

If you want to understand it better, we refer to a [good explanation](https://cs224d.stanford.edu/lecture_notes/notes1.pdf).

```python
try:
    from scipy.misc import logsumexp
except ImportError:
    from scipy.special import logsumexp

# compute_cost: cross-entropy cost function,
def compute_cost(z, C, y, yhat, batch_size):
    z_hat = logsumexp(z, axis=-1, keepdims=True)
    cost = (-np.sum(y*np.log(yhat)) + np.sum(2.0*C*z_hat)) / batch_size
    return cost
```

##  Training the Model

Now that you have understood how the CBOW model works, you will train it.

You created a function for the forward propagation. Now you will implement a function that computes the gradients to backpropagate the errors.

```python
# UNQ_C4 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# UNIT TEST COMMENT: Candidate for Table Driven Tests
# GRADED FUNCTION: back_prop
def back_prop(x, z, y, h, W1, W2, b1, b2, batch_size, m):
    '''
    Inputs:
        x:  average one hot vector for the context
        z:  score vector
        y:  target vector
        h:  hidden vector (see eq. 1)
        W1, W2, b1, b2:  matrices and biases
        batch_size: batch size
        m:  number of context words
     Outputs:
        grad_W1, grad_W2, grad_b1, grad_b2:  gradients of matrices and biases
    '''
    ### START CODE HERE (REPLACE INSTANCES OF 'None' WITH YOUR OWN CODE) ###
    ### (YOU WILL NEED TO ADD CODE IN ADDITION TO JUST REPLACING THE 'None' VALUES) ###
    l1 = np.dot(W2.T, (z - y))
    l1[l1<0] = 0.0

    grad_W2 = (1.0/(batch_size*m)) * np.dot((z - y), h)

    grad_b2 = (1.0/(batch_size*m)) * np.sum((z - y), axis = 1, keepdims=True)

    grad_b1 = (1.0/(batch_size*m)) * np.sum(l1, axis = 1, keepdims=True)

    grad_W1 = (1.0/(batch_size*m)) * np.dot(l1, x.T)
    ### END CODE HERE ###

    return grad_W1, grad_W2, grad_b1, grad_b2
```

Now that you have implemented a function to compute the gradients, you will implement batch gradient descent over your training set.

**Hint:** For that, you will use initialize_model and the back_prop function that you just created (and the compute_cost function). You can also use the provided get_batches helper function:

```for x, y in get_batches(data, word2Ind, V, C, batch_size):```

```...```

Also: print the cost after each batch is processed (use batch size = 128)

```python
# UNQ_C5 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# UNIT TEST COMMENT: Candidate for Table Driven Tests
# GRADED FUNCTION: gradient_descent
def gradient_descent(data, word2Ind, C, N, V, num_iters, alpha=0.03):

    '''
    This is the gradient_descent function

      Inputs:
        data:      text
        word2Ind:  words to Indices
        C:         context window
        N:         dimension of hidden vector
        V:         dimension of vocabulary
        num_iters: number of iterations
     Outputs:
        W1, W2, b1, b2:  updated matrices and biases

    '''
    W1, W2, b1, b2 = initialize_model(N,V) #W1=(N,V) and W2=(V,N)
    m = (2*C)
    batch_size = 128
    iters = 0
    for x, y in get_batches(data, word2Ind, V, C, batch_size):
        z, h = forward_prop(x, W1, W2, b1, b2)
        yhat = softmax(z)
        cost = compute_cost(z, C, y, yhat, batch_size)
        print('iters', iters + 1 , '   cost',cost)
        grad_W1, grad_W2, grad_b1, grad_b2 = back_prop(x, yhat, y, h, W1, W2, b1, b2, batch_size, m)
        W1 = W1 - alpha * grad_W1
        W2 = W2 - alpha * grad_W2
        b1 = b1 - alpha * grad_b1
        b2 = b2 - alpha * grad_b2
        iters += 1
        if iters == num_iters:
            break
        if iters % 100 == 0:
            alpha *= 0.66

        ### END CODE HERE ###

    return W1, W2, b1, b2
```

```python
# test your function
C = 2
N = 50
word2Ind, Ind2word = get_dict(data)
V = len(word2Ind)
num_iters = 15
W1, W2, b1, b2 = gradient_descent(data, word2Ind, C, N, V, num_iters)
```

    iters 1    cost 159.459353013221
    iters 2    cost 135.04677235814708
    iters 3    cost 111.63198667805563
    iters 4    cost 89.75967845163359
    iters 5    cost 71.26761559927665
    iters 6    cost 57.03879565747018
    iters 7    cost 46.90182477839866
    iters 8    cost 42.25843129148009
    iters 9    cost 41.648730386152245
    iters 10    cost 41.64850422580056
    iters 11    cost 41.648278127273265
    iters 12    cost 41.64805209103558
    iters 13    cost 41.64782611755612
    iters 14    cost 41.64760020730713
    iters 15    cost 41.64737436076431

**Expected Output:** iters 15 cost 41.66082959286652

## 3.0 Visualizing the word vectors

In this part you will visualize the word vectors trained using the function you just coded above.

```python
# visualizing the word vectors here
from matplotlib import pyplot
%config InlineBackend.figure_format = 'svg'
words = ['king', 'queen','lord','man', 'woman','dog','horse',
         'rich','happy','sad']

embs = (W1.T + W2)/2.0

# given a list of words and the embeddings, it returns a matrix with all the embeddings
idx = [word2Ind[word] for word in words]
X = embs[idx, :]
print(X.shape, idx)  # X.shape:  Number of words of dimension N each
```

    (10, 50) [2744, 3950, 2960, 3022, 5674, 1452, 2472, 4190, 2316, 4277]

```python
result= compute_pca(X, 2)
pyplot.scatter(result[:, 0], result[:, 1])
for i, word in enumerate(words):
    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))
pyplot.show()
```

You can see that woman and queen are next to each other. However, we have to be carefull with the interpretation of this projected word vectors, since the PCA depends on the projection -- as shown in the following illustration.

```python
result= compute_pca(X, 4)
pyplot.scatter(result[:, 3], result[:, 1])
for i, word in enumerate(words):
    pyplot.annotate(word, xy=(result[i, 3], result[i, 1]))
pyplot.show()
```

2 - Natural Language Processing with Probabilistic Models/Week 4/edits/C2_W4_Assignment _5of5-THISWORKS.ipynb

NLP/Learn_by_deeplearning.ai/Course 2 - Probabilistic Models/Labs/Week 4/edits/C2_W4_Assignment _5of5-THISWORKS.ipynb


```julia
using Rocket
using ReactiveMP
using GraphPPL
using Distributions
using BenchmarkTools
using Random
using Plots
```

```julia
import Pkg; Pkg.status(); versioninfo() # All benchmarks have been peroformed on a regular MacbookPro 2018 with Julia 1.5.4
```

    Project ReactiveMP v0.5.0
    Status `~/.julia/dev/ReactiveMP/Project.toml`
      [864edb3b] DataStructures v0.18.9
      [31c24e10] Distributions v0.24.15
      [cc6bae93] DomainIntegrals v0.2.3
      [442a2c76] FastGaussQuadrature v0.4.7
      [f6369f11] ForwardDiff v0.10.17
      [1914dd2f] MacroTools v0.5.6
      [2774e3e8] NLsolve v4.5.1
      [429524aa] Optim v1.2.4
      [85a6dd25] PositiveFactorizations v0.2.4
      [df971d30] Rocket v1.3.5 `~/.julia/dev/Rocket`
      [276daf66] SpecialFunctions v1.3.0
      [2913bbd2] StatsBase v0.33.3
      [4c63d2b9] StatsFuns v0.9.6
      [9d95972d] TupleTools v1.2.0
      [37e2e46d] LinearAlgebra
      [9a3f8284] Random
    Julia Version 1.5.4
    Commit 69fcb5745b (2021-03-11 19:13 UTC)
    Platform Info:
      OS: macOS (x86_64-apple-darwin18.7.0)
      CPU: Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz
      WORD_SIZE: 64
      LIBM: libopenlibm
      LLVM: libLLVM-9.0.1 (ORCJIT, skylake)

## State Estimation in a Linear Multivariate Gaussian State-space Model

In this demo, the goal is to smooth noisy observations by performing full inference with a state-space model (SSM).

We wil use the following model:

\begin{equation} \label{eq:linear_gaussian_ssm}
  \begin{aligned}
    \mathbf{x}_k & = \, \mathbf{A}\mathbf{x}_{k - 1} + \mathcal{P} \\
    \mathbf{y}_k & = \, \mathbf{B}\mathbf{x}_{k} + \mathcal{Q} \\
  \end{aligned}
\end{equation}

In this model, we denote by $\mathbf{x}_k$ the current state of the system (at time step $k$), by $\mathbf{x}_{k - 1}$ the previous state at time $k-1$, $\mathbf{A}$ and $\mathbf{B}$ are a constant system inputs and $\mathbf{y}_k$ is a noisy observation of $\mathbf{x}_k$. We further assume that the states and the observations are corrupted by i.i.d. Gaussian noise with variances $\mathcal{P}$ and $\mathcal{Q}$ respectively.

The SSM can be represented by the following factor graph, where the pictured section is chained over time:

For smoothing (forward-backward) estimation in ReactiveMP.jl, it is possible to build a full graph of the model.

```julia
# We create model using GraphPPL.jl package interface with @model macro
# For simplicity of the example we consider all matrices to be known and constant
@model function linear_gaussian_ssm_smoothing(n, A, B, P, Q)

    x = randomvar(n) # Create a sequence of random variables
    y = datavar(Vector{Float64}, n) # Create a sequence of observed multivariate data points

    cA = constvar(A)
    cB = constvar(B)
    cP = constvar(P)
    cQ = constvar(Q)

    # Set a prior distribution for x[1]
    x[1] ~ MvGaussianMeanCovariance([ 0.0, 0.0 ], [ 100.0 0.0; 0.0 100.0 ])
    y[1] ~ MvGaussianMeanCovariance(B * x[1], Q)

    for t in 2:n
        x[t] ~ MvGaussianMeanCovariance(A * x[t - 1], P)
        y[t] ~ MvGaussianMeanCovariance(B * x[t], Q)
    end

    return x, y
end

# It is also possible to create a single time step of the graph
# for filtering algorithm
@model function linear_gaussian_ssm_filtering(A, B, P, Q)

    # Priors for the previous x_t-1 step
    x_min_t_mean = datavar(Vector{Float64})
    x_min_t_cov  = datavar(Matrix{Float64})

    x_min_t ~ MvGaussianMeanCovariance(x_min_t_mean, x_min_t_cov)
    x_t     ~ MvGaussianMeanCovariance(A * x_min_t, P)

    y_t = datavar(Vector{Float64})
    y_t ~ MvGaussianMeanCovariance(B * x_t, Q)

    return x_min_t_mean, x_min_t_cov, x_t, y_t
end
```

    linear_gaussian_ssm_filtering (generic function with 1 method)

```julia
function generate_data(n, A, B, P, Q; seed = 1234)
    Random.seed!(seed)

    x_prev = zeros(2)
    x      = Vector{Vector{Float64}}(undef, n)
    y      = Vector{Vector{Float64}}(undef, n)

    for i in 1:n
        x[i]   = rand(MvNormal(A * x_prev, P))
        y[i]   = rand(MvNormal(B * x[i], Q))
        x_prev = x[i]
    end

    return x, y
end
```

    generate_data (generic function with 1 method)

```julia
n = 100
θ = π / 15
A = [ cos(θ) -sin(θ); sin(θ) cos(θ) ]
B = [ 1.3 0.0; 0.0 0.7 ]
P = [ 1.0 0.0; 0.0 1.0 ]
Q = [ 1.0 0.0; 0.0 1.0 ]
θ = π / 15

real_x, real_y = generate_data(n, A, B, P, Q);
```

```julia
# Inference procedure for full graph
function reactivemp_inference_smoothing(observations, A, B, P, Q)
    n = length(observations)

    model, (x, y) = linear_gaussian_ssm_smoothing(n, A, B, P, Q, options = (limit_stack_depth = 500, ))

    xbuffer       = buffer(Marginal, n)
    xsubscription = subscribe!(getmarginals(x), xbuffer)

    update!(y, observations)
    unsubscribe!(xsubscription)

    return getvalues(xbuffer)
end

# Inference procedure for single time step graph and filtering
function reactivemp_inference_filtering(observations, A, B, P, Q)
    n = length(observations)

    model, (x_min_t_mean, x_min_t_cov, x_t, y_t) = linear_gaussian_ssm_filtering(A, B, P, Q)

    xbuffer = keep(Marginal)

    redirect_to_prior_subscription = subscribe!(getmarginal(x_t), (x_t_posterior) -> begin
        update!(x_min_t_mean, mean(x_t_posterior))
        update!(x_min_t_cov, cov(x_t_posterior))
    end)

    xsubscription = subscribe!(getmarginal(x_t), xbuffer)

    update!(x_min_t_mean, [ 0.0, 0.0 ])
    update!(x_min_t_cov, [ 100.0 0.0; 0.0 100.0 ])

    for observation in observations
        update!(y_t, observation)
    end

    unsubscribe!(xsubscription)
    unsubscribe!(redirect_to_prior_subscription)

    return getvalues(xbuffer)
end
```

    reactivemp_inference_filtering (generic function with 1 method)

```julia
x_reactivemp_smoothing_inferred = reactivemp_inference_smoothing(real_y, A, B, P, Q);
```

```julia
x_reactivemp_filtering_inferred = reactivemp_inference_filtering(real_y, A, B, P, Q);
```

```julia
# Results verification

reshape_data(data) = transpose(reduce(hcat, data))

ylimit = (-20, 20)

generated_data = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
generated_data = plot!(1:n, real_y |> reshape_data, seriestype = :scatter, ms = 3, alpha = 0.5, label = [ "observations[:, 1]" "observations[:, 2]" ])
generated_data = plot!(generated_data, legend = :bottomleft, ylimit = ylimit)

rmp_smoothing_results_plot = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
rmp_smoothing_results_plot = plot!(rmp_smoothing_results_plot, 1:n, mean.(x_reactivemp_smoothing_inferred) |> reshape_data, ribbon = var.(x_reactivemp_smoothing_inferred) |> reshape_data, label = [ "inferred[:, 1]" "inferred[:, 2]" ])
rmp_smoothing_results_plot = plot!(rmp_smoothing_results_plot, legend = :bottomleft, ylimit = ylimit)

rmp_filtering_results_plot = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
rmp_filtering_results_plot = plot!(rmp_filtering_results_plot, 1:n, mean.(x_reactivemp_filtering_inferred) |> reshape_data, ribbon = var.(x_reactivemp_filtering_inferred) |> reshape_data, label = [ "inferred[:, 1]" "inferred[:, 2]" ])
rmp_filtering_results_plot = plot!(rmp_filtering_results_plot, legend = :bottomleft, ylimit = ylimit)

plot(generated_data, rmp_smoothing_results_plot, rmp_filtering_results_plot, layout = @layout([ a b c ]), size = (1200, 300))
```

```julia
benchmark_rmp_sizes = [ 50, 100, 250, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 20_000, 25_000, 50_000 ];
```

```julia
reactivemp_benchmark_results = map(benchmark_rmp_sizes) do size
    states, observations = generate_data(size, A, B, P, Q);
    benchmark_smoothing = @benchmark reactivemp_inference_smoothing($observations, $A, $B, $P, $Q)
    benchmark_fitlering = @benchmark reactivemp_inference_filtering($observations, $A, $B, $P, $Q)
    println("Finished $size")
    return (size, benchmark_smoothing, benchmark_fitlering)
end
```

    Finished 50
    Finished 100
    Finished 250
    Finished 500
    Finished 1000
    Finished 2000
    Finished 5000
    Finished 10000
    Finished 15000
    Finished 20000
    Finished 25000
    Finished 50000

    12-element Array{Tuple{Int64,BenchmarkTools.Trial,BenchmarkTools.Trial},1}:
     (50, 4.931 ms, 550.622 μs)
     (100, 10.441 ms, 989.952 μs)
     (250, 29.188 ms, 2.382 ms)
     (500, 60.966 ms, 4.579 ms)
     (1000, 131.906 ms, 9.716 ms)
     (2000, 309.516 ms, 18.748 ms)
     (5000, 907.643 ms, 50.581 ms)
     (10000, 1.817 s, 115.098 ms)
     (15000, 3.213 s, 187.402 ms)
     (20000, 5.196 s, 257.468 ms)
     (25000, 5.389 s, 341.697 ms)
     (50000, 12.490 s, 699.797 ms)

```julia
benchmark_time_ms(trial) = minimum(trial).time / 1_000_000
```

    benchmark_time_ms (generic function with 1 method)

```julia
lgssm_scaling = plot(xscale = :log10, yscale = :log10, xlabel = "Number of observations", ylabel = "Minimum execution time (in ms)", title = "ReactiveMP.jl Linear Gaussian State Space Model Benchmark", legend = :bottomright, size = (650, 400))
lgssm_scaling = plot!(lgssm_scaling, benchmark_rmp_sizes, map(i -> benchmark_time_ms(reactivemp_benchmark_results[i][2]), 1:length(benchmark_rmp_sizes)), markershape = :diamond, label = "Smoothing")
lgssm_scaling = plot!(lgssm_scaling, benchmark_rmp_sizes, map(i -> benchmark_time_ms(reactivemp_benchmark_results[i][3]), 1:length(benchmark_rmp_sizes)), markershape = :utriangle, label = "Filtering")

# display(lgssm_scaling)

savefig(lgssm_scaling, "./plots/lgssm_scaling.svg")
```

```julia
import Turing
```

    ┌ Info: Precompiling Turing [fce5fe82-541a-59a6-adf8-730c64b5f9a0]
    └ @ Base loading.jl:1278
    ┌ Info: Precompiling NamedArrays [86f7a689-2022-50b4-a561-43c23ac3c673]
    └ @ Base loading.jl:1278

```julia
Turing.@model LinearGaussianSSM(y, A, B, P, Q) = begin
    n = length(y)

    # State sequence.
    x = Vector(undef, n)

    # Observe each point of the input.
    x[1] ~ MvNormal([ 0.0, 0.0 ], 1e1)
    y[1] ~ MvNormal(B * x[1], Q)

    for t in 2:n
        x[t] ~ MvNormal(A * x[t - 1], P)
        y[t] ~ MvNormal(B * x[t], Q)
    end
end;
```

```julia
function turing_inference_smoothing(observations, A, B, P, Q; nsamples = 500)
    return Turing.sample(LinearGaussianSSM(observations, A, B, P, Q), Turing.HMC(0.1, 5), nsamples)
end
```

    turing_inference_smoothing (generic function with 1 method)

```julia
x_turing_500_smoothing_inferred  = turing_inference_smoothing(real_y, A, B, P, Q, nsamples = 500);
x_turing_1000_smoothing_inferred = turing_inference_smoothing(real_y, A, B, P, Q, nsamples = 1000);
```

    Sampling: 100%|█████████████████████████████████████████| Time: 0:00:40
    Sampling: 100%|█████████████████████████████████████████| Time: 0:01:20

```julia
samples_500  = get(x_turing_500_smoothing_inferred, :x);
samples_1000 = get(x_turing_1000_smoothing_inferred, :x);

mx_500 = [ mean(samples_500.x[i].data) for i in 1:2n ]
vx_500 = [ std(samples_500.x[i].data)  for i in 1:2n ];

mx_1000 = [ mean(samples_1000.x[i].data) for i in 1:2n ]
vx_1000 = [ std(samples_1000.x[i].data)  for i in 1:2n ];
```

```julia
reshape_turing_data(data) = transpose(reshape(data, (2, Int(length(data) / 2))))
```

    reshape_turing_data (generic function with 1 method)

```julia
generated_data = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
generated_data = plot!(1:n, real_y |> reshape_data, seriestype = :scatter, ms = 3, alpha = 0.5, label = [ "observations[:, 1]" "observations[:, 2]" ])
generated_data = plot!(generated_data, legend = :bottomleft)

turing_500_results_plot = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
turing_500_results_plot = plot!(turing_500_results_plot, 1:n, mx_500 |> reshape_turing_data, ribbon = vx_500 |> reshape_turing_data, label = [ "inferred[:, 1]" "inferred[:, 2]" ])
turing_500_results_plot = plot!(turing_500_results_plot, legend = :bottomleft)

turing_1000_results_plot = plot(1:n, real_x |> reshape_data, label = [ "x[:, 1]" "x[:, 2]" ])
turing_1000_results_plot = plot!(turing_1000_results_plot, 1:n, mx_1000 |> reshape_turing_data, ribbon = vx_1000 |> reshape_turing_data, label = [ "inferred[:, 1]" "inferred[:, 2]" ])
turing_1000_results_plot = plot!(turing_1000_results_plot, legend = :bottomleft)

plot(generated_data, turing_500_results_plot, turing_1000_results_plot, layout = @layout([ a b c ]), size = (1200, 300))
```

```julia
reactivemp_filtering_n_100_benchmark = @benchmark reactivemp_inference_filtering($real_y, $A, $B, $P, $Q);
reactivemp_smoothing_n_100_benchmark = @benchmark reactivemp_inference_smoothing($real_y, $A, $B, $P, $Q);
turing_500_n_100_benchmark  = @benchmark turing_inference_smoothing($real_y, $A, $B, $P, $Q, nsamples = 500);
turing_1000_n_100_benchmark = @benchmark turing_inference_smoothing($real_y, $A, $B, $P, $Q, nsamples = 1000);
```

    Sampling: 100%|█████████████████████████████████████████| Time: 0:00:41
    Sampling: 100%|█████████████████████████████████████████| Time: 0:00:39
    Sampling: 100%|█████████████████████████████████████████| Time: 0:00:38
    Sampling: 100%|█████████████████████████████████████████| Time: 0:00:38
    Sampling: 100%|█████████████████████████████████████████| Time: 0:01:18
    Sampling: 100%|█████████████████████████████████████████| Time: 0:01:18
    Sampling: 100%|█████████████████████████████████████████| Time: 0:01:18
    Sampling: 100%|█████████████████████████████████████████| Time: 0:01:17

```julia
using ColorSchemes, Colors ;

c = palette(:rainbow)

c1 = get(ColorSchemes.rainbow, 0.5) ;
c2 = get(ColorSchemes.rainbow, 0.2) ;
c3 = get(ColorSchemes.rainbow, 0.3) ;
c4 = get(ColorSchemes.rainbow, 0.8) ;
c5 = get(ColorSchemes.rainbow, 0.9) ;

colors = [ c1, c2, c3, c4, c5 ];
```

```julia
bar_names  = [ "ReactiveMP.jl Filtering", "ReactiveMP.jl Smoothing", "Turing.jl HMC(500)", "Turing.jl HMC(1000)" ]
bar_values = [
    benchmark_time_ms(reactivemp_filtering_n_100_benchmark),
    benchmark_time_ms(reactivemp_smoothing_n_100_benchmark),
    benchmark_time_ms(turing_500_n_100_benchmark),
    benchmark_time_ms(turing_1000_n_100_benchmark)
]

lgssm_comparison = bar(bar_names, bar_values, fillcolor = colors, size = (650, 400), yscale = :log10, ylabel = "Inference time (log scale, in ms)", title = "Linear Gaussian State Space Model with 100 observations", legend = false)

# display(lgssm_comparison)

savefig(lgssm_comparison, "./plots/lgssm_comparison.svg");
```

```julia
pyplot()
```

    ┌ Info: Precompiling PyPlot [d330b81b-6aea-500a-939a-2ce795aea3ee]
    └ @ Base loading.jl:1278

    Plots.PyPlotBackend()

```julia
lgssm = plot(lgssm_comparison, lgssm_scaling, size = (1400, 400))
# display(lgssm)
savefig(lgssm, "./plots/lgssm.svg")
```

```julia

```

benchmark/notebooks/Linear Multivariate Gaussian State Space Model Benchmark.ipynb