```python
import sympy as sy
from sympy.utilities.codegen import codegen

import os
from pathlib import Path

```

```python
N = 2  #次元

x = sy.Matrix(sy.MatrixSymbol('x', N, 1))
x_dot = sy.Matrix(sy.MatrixSymbol('x_dot', N, 1))

#x_norm = x.norm(ord=2)
x_norm = 0
for i in range(N):
    x_norm += x[i, 0]**2
x_norm = sy.sqrt(x_norm)# ** (1/2)
x_hat = x / x_norm

x_norm
```

$\displaystyle \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}$

```python
x
```

$\displaystyle \left[\begin{array}{c}x_{0, 0}\\x_{1, 0}\end{array}\right]$

```python
### 慣性行列 ###

sigma_alpha, sigma_gamma, w_u, w_l, alpha, epsilon = sy.symbols('sigma_alpha, sigma_gamma, w_u, w_l, alpha, epsilon')

alpha_x = sy.exp(-((x_norm**2) / (2 * sigma_alpha**2)))
gamma_x = sy.exp(-((x_norm**2) / (2 * sigma_gamma**2)))
w_x = gamma_x * w_u + (1 - gamma_x) * w_l
nabla_x_phi = (1 - sy.exp(-2 * alpha * x_norm)) / (1 + sy.exp(-2 * alpha * x_norm)) * x_hat

M = w_x  * ((1 - alpha_x) * nabla_x_phi * nabla_x_phi.T + (alpha_x + epsilon) * sy.eye(N))

M = sy.simplify(M)
M
```

$\displaystyle \left[\begin{matrix}\frac{\left(w_{l} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) + w_{u} e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) \left(\left(1 - e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}}\right)^{2} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) x_{0, 0}^{2} + \left(\epsilon + e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) \left(e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}} + 1\right)^{2} \left(x_{0, 0}^{2} + x_{1, 0}^{2}\right)\right)}{\left(e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}} + 1\right)^{2} \left(x_{0, 0}^{2} + x_{1, 0}^{2}\right)} & \frac{\left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) \left(w_{l} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) + w_{u} e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) \tanh^{2}{\left(\alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}} \right)} x_{0, 0} x_{1, 0}}{x_{0, 0}^{2} + x_{1, 0}^{2}}\\\frac{\left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) \left(w_{l} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) + w_{u} e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) \tanh^{2}{\left(\alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}} \right)} x_{0, 0} x_{1, 0}}{x_{0, 0}^{2} + x_{1, 0}^{2}} & \frac{\left(w_{l} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) + w_{u} e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\gamma}^{2}}}\right) \left(\left(1 - e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}}\right)^{2} \left(1 - e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) x_{1, 0}^{2} + \left(\epsilon + e^{- \frac{x_{0, 0}^{2} + x_{1, 0}^{2}}{2 \sigma_{\alpha}^{2}}}\right) \left(e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}} + 1\right)^{2} \left(x_{0, 0}^{2} + x_{1, 0}^{2}\right)\right)}{\left(e^{2 \alpha \sqrt{x_{0, 0}^{2} + x_{1, 0}^{2}}} + 1\right)^{2} \left(x_{0, 0}^{2} + x_{1, 0}^{2}\right)}\end{matrix}\right]$

```python
# 曲率校xi_Mの前半を計算
partialx_m_dx = []
for i in range(N):
    print("i = ", i)
    partialx_m_dx.append(M[:, i:i+1].jacobian(x) * x_dot)
# partialx_m1 = M[:, 0:1].jacobian(x)
# partialx_m2 = M[:, 1:2].jacobian(x)
# partialx_m3 = M[:, 2:3].jacobian(x)

# print(sy.simplify(partialx_m1))

xi_M_before_ = partialx_m_dx[0]
for i in range(1, N):
    print("i = ", i)
    xi_M_before_ = xi_M_before_.row_join(partialx_m_dx[i])
# xi_M_before_before = partialx_m1_dx.row_join(partialx_m2_dx)
# xi_M_before_before = xi_M_before_before.row_join(partialx_m3_dx)

#xi_M_before_ = sy.simplify(xi_M_before_)
print(xi_M_before_.shape)

xi_M_before = xi_M_before_ * x_dot

# #print(xi_M_before)

# # 曲率校xi_Mの後半を計算
xi_M_after_ = x_dot.T * M * x_dot
xi_M_after = -1/2* (xi_M_after_.jacobian(x)).T #　sympyにはナブラあるかも？

# # 合体
xi_M = xi_M_before + xi_M_after
xi_M = sy.simplify(xi_M)  # Elisじゃ無理
# print(xi_M)
```

    i =  0
    i =  1
    i =  1
    (2, 2)

```python
def gen(func, name):
    [(c_name, c_code), (h_name, c_header)] = codegen(
        name_expr=(name, func),
        language="C",
        project= name + "project",
        to_files=False
    )

    f = open(c_name, 'w')
    f.write(c_code)
    f.close()

    f = open(h_name, 'w')
    f.write(c_header)
    f.close()

gen(xi_M, "rmp2_attractor_xi")
```


# Regression Analyse

```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import itertools
import warnings

import mysql.connector
from sqlalchemy import create_engine, exc

from time import gmtime, strftime
```

```python
# Settings

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)

# Seaborn
sns.set()
sns.set_style("darkgrid")
plt.matplotlib.style.use('default')

my_colors = ["windows blue", "saffron", "hot pink", "algae green", "dusty purple", "greyish", "petrol", "denim blue", "lime"]
sns.set_palette(sns.xkcd_palette(my_colors))
colors = sns.xkcd_palette(my_colors)

# Warnings
warnings.filterwarnings("ignore")
```

```python
def my_df_summary(data):
    '''function for the summary'''
    try:
        dat = data.copy()
        df = pd.DataFrame([dat.min(), dat.max(), dat.mean(), dat.std(), dat.isna().sum(), dat.nunique(), dat.dtypes],
                     index=['Minimum', 'Maximum', 'Mean', 'Stand. Dev.','#NA', '#Uniques', 'dtypes'])
        print(f'Dataset has {len(data)} rows.')
        return df
    except:
        print('No summary!!')
        return data
```

```python
df = pd.read_csv('data/Werbungseffekte.csv')
df.head()
```

      ID
      TV
      Radio
      Newspaper
      Sales

      0
      1
      230.1
      37.8
      69.2
      22.1

      1
      2
      44.5
      39.3
      45.1
      10.4

      2
      3
      17.2
      45.9
      69.3
      9.3

      3
      4
      151.5
      41.3
      58.5
      18.5

      4
      5
      180.8
      10.8
      58.4
      12.9

```python
my_df_summary(df)
```

    Dataset has 200 rows.

      ID
      TV
      Radio
      Newspaper
      Sales

      Minimum
      1
      0.7
      0
      0.3
      1.6

      Maximum
      200
      296.4
      49.6
      114
      27

      Mean
      100.5
      147.042
      23.264
      30.554
      14.0225

      Stand. Dev.
      57.8792
      85.8542
      14.8468
      21.7786
      5.21746

      #NA
      0
      0
      0
      0
      0

      #Uniques
      200
      190
      167
      172
      121

      dtypes
      int64
      float64
      float64
      float64
      float64

```python
df.info()
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 200 entries, 0 to 199
    Data columns (total 5 columns):
     #   Column     Non-Null Count  Dtype
    ---  ------     --------------  -----
     0   ID         200 non-null    int64
     1   TV         200 non-null    float64
     2   Radio      200 non-null    float64
     3   Newspaper  200 non-null    float64
     4   Sales      200 non-null    float64
    dtypes: float64(4), int64(1)
    memory usage: 7.9 KB

```python
df.describe()
```

      ID
      TV
      Radio
      Newspaper
      Sales

      count
      200.000000
      200.000000
      200.000000
      200.000000
      200.000000

      mean
      100.500000
      147.042500
      23.264000
      30.554000
      14.022500

      std
      57.879185
      85.854236
      14.846809
      21.778621
      5.217457

      min
      1.000000
      0.700000
      0.000000
      0.300000
      1.600000

      25%
      50.750000
      74.375000
      9.975000
      12.750000
      10.375000

      50%
      100.500000
      149.750000
      22.900000
      25.750000
      12.900000

      75%
      150.250000
      218.825000
      36.525000
      45.100000
      17.400000

      max
      200.000000
      296.400000
      49.600000
      114.000000
      27.000000

### Understanding Data

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 16]
fig, axs = plt.subplots(4, 1)
sns.distplot(df.iloc[:,1], color=colors[0], ax=axs[0], bins=20)
sns.distplot(df.iloc[:,2], color=colors[1], ax=axs[1], bins=20)
sns.distplot(df.iloc[:,3], color=colors[2], ax=axs[2], bins=20)
sns.distplot(df.iloc[:,4], color=colors[3], ax=axs[3], bins=20)
plt.show()
```

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 16]
fig, axs = plt.subplots(4, 1)
sns.boxplot(df.iloc[:,1], color=colors[0], ax=axs[0])
sns.boxplot(df.iloc[:,2], color=colors[1], ax=axs[1])
sns.boxplot(df.iloc[:,3], color=colors[2], ax=axs[2])
sns.boxplot(df.iloc[:,4], color=colors[3], ax=axs[3])
plt.show()
```

## Exploring Sales data

### 1. Linear Correlation

```python
df.iloc[:,1:].corr()
```

      TV
      Radio
      Newspaper
      Sales

      TV
      1.000000
      0.054809
      0.056648
      0.782224

      Radio
      0.054809
      1.000000
      0.354104
      0.576223

      Newspaper
      0.056648
      0.354104
      1.000000
      0.228299

      Sales
      0.782224
      0.576223
      0.228299
      1.000000

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [8, 8]
plt.matshow(df.iloc[:,1:].corr())
plt.colorbar()
plt.show()
```

## 2. Regression Analyse

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 8]
fig, axs = plt.subplots(1, 3)
df.plot(kind='scatter', x='TV', y='Sales', ax=axs[0])
df.plot(kind='scatter', x='Radio', y='Sales', ax=axs[1])
df.plot(kind='scatter', x='Newspaper', y='Sales', ax=axs[2])
plt.show()
```

## Einfache lineare Regression (häufig auch einfach OLS-Regression)

Mit einer einfachen linearen Regression misst man den direkten, linearen Zusammenhang (häufig engl. Response) zwischen zwei Werten.

Gleichung: $y = \beta_0 + \beta_1x$

- $y$ ist die zu erklärende Größe (Response-Value)
- $x$ ist die erklärende Größe (häufig engl. Feature)
- $\beta_0$ ist der geschätzte Achsenabschnitt
- $\beta_1$ ist der geschätzte lineare Wirkzusammenhang (Steigung der Geraden)

$\beta_0$ und $\beta_1$ zusammen nennt man **Koeffizienten**.

## Berechnung der Koeffizienten

 Schätzfunktion: $y = \hat{\beta}_0 + \hat{\beta}_{1x} + \epsilon$

Die optimalen Werte für $\beta_0$ und $\beta_1$ werden für eine OLS-Regression so berechnet:

### Root Mean Squared Error - RMSE

Tipp: https://de.wikipedia.org/wiki/Lineare_Regression

\begin{align}
\hat{\beta}_1   \; &=  \frac{\sum_{i=1}^n (X_i - \bar{X})(Y_i - \bar{Y})}{\sum_{i=1}^n (X_i - \bar{X})^2}
\end{align}

\begin{align*}
\hat{\beta}_0   \; &=  \bar{Y} - \hat{\beta}_1 \bar{X}
\end{align*}

D.h., der Steigungs-Koeffizient ($\beta_1$) lässt sich direkt als Cov(X,Y)/Var(X) berechnen.

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 8]
g = sns.lmplot(x="TV", y="Sales", data=df)
plt.show()
```

```python
df.head()
```

      ID
      TV
      Radio
      Newspaper
      Sales

      0
      1
      230.1
      37.8
      69.2
      22.1

      1
      2
      44.5
      39.3
      45.1
      10.4

      2
      3
      17.2
      45.9
      69.3
      9.3

      3
      4
      151.5
      41.3
      58.5
      18.5

      4
      5
      180.8
      10.8
      58.4
      12.9

```python
# Coefficients on Statsmodel
import statsmodels.formula.api as smf
# create a fitted model in one line
lm = smf.ols(formula='Sales ~ TV', data=df).fit()
```

```python
dir(lm)
```

    ['HC0_se',
     'HC1_se',
     'HC2_se',
     'HC3_se',
     '_HCCM',
     '__class__',
     '__delattr__',
     '__dict__',
     '__dir__',
     '__doc__',
     '__eq__',
     '__format__',
     '__ge__',
     '__getattribute__',
     '__gt__',
     '__hash__',
     '__init__',
     '__init_subclass__',
     '__le__',
     '__lt__',
     '__module__',
     '__ne__',
     '__new__',
     '__reduce__',
     '__reduce_ex__',
     '__repr__',
     '__setattr__',
     '__sizeof__',
     '__str__',
     '__subclasshook__',
     '__weakref__',
     '_cache',
     '_data_attr',
     '_get_robustcov_results',
     '_is_nested',
     '_use_t',
     '_wexog_singular_values',
     'aic',
     'bic',
     'bse',
     'centered_tss',
     'compare_f_test',
     'compare_lm_test',
     'compare_lr_test',
     'condition_number',
     'conf_int',
     'conf_int_el',
     'cov_HC0',
     'cov_HC1',
     'cov_HC2',
     'cov_HC3',
     'cov_kwds',
     'cov_params',
     'cov_type',
     'df_model',
     'df_resid',
     'eigenvals',
     'el_test',
     'ess',
     'f_pvalue',
     'f_test',
     'fittedvalues',
     'fvalue',
     'get_influence',
     'get_prediction',
     'get_robustcov_results',
     'initialize',
     'k_constant',
     'llf',
     'load',
     'model',
     'mse_model',
     'mse_resid',
     'mse_total',
     'nobs',
     'normalized_cov_params',
     'outlier_test',
     'params',
     'predict',
     'pvalues',
     'remove_data',
     'resid',
     'resid_pearson',
     'rsquared',
     'rsquared_adj',
     'save',
     'scale',
     'ssr',
     'summary',
     'summary2',
     't_test',
     't_test_pairwise',
     'tvalues',
     'uncentered_tss',
     'use_t',
     'wald_test',
     'wald_test_terms',
     'wresid']

```python
# summary function
lm.summary()
```

OLS Regression Results

  Dep. Variable:	          Sales	        R-squared:         	    0.612

  Model:	                   OLS	         Adj. R-squared:    	    0.610

  Method:	             Least Squares	    F-statistic:       	    312.1

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 1.47e-42

  Time:	                 11:43:17	       Log-Likelihood:    	  -519.05

  No. Observations:	         200	        AIC:               	    1042.

  Df Residuals:	             198	        BIC:               	    1049.

  Df Model:	                   1

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	     7.0326	     0.458	    15.360	  0.000	     6.130	     7.935

  TV	            0.0475	     0.003	    17.668	  0.000	     0.042	     0.053

  Omnibus:	        0.531	   Durbin-Watson:     	    1.935

  Prob(Omnibus):	  0.767	   Jarque-Bera (JB):  	    0.669

  Skew:	          -0.089	   Prob(JB):          	    0.716

  Kurtosis:	       2.779	   Cond. No.          	     338.

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

## Wie lese ich die Ergebnisse einer OLS-Regression?

To check whether a regression has good explanatory power at all, we should look at the following key-features:

1. Determination coefficient adj. R-squared

2. The estimation statistics of all coefficients

3. Jarque-Bera-Test

4. Durbin-Watson-Test

Only when these values have the desired properties we should go further with the actual coefficients

### Analyse the Rest

```python
# The residuals are also directly ejected by the statsmodel

%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 8]
fig, axs = plt.subplots(2, 1)
lm.resid.plot(ax=axs[0])
lm.resid.plot(kind='hist', bins=25, ax=axs[1])
plt.show()
```

### Once we have estimated the model, we can also use it to explain Y with new x values.

```python
# Important: predict expects the values with the same labels as data with which estimates were made.

X_neu = pd.DataFrame({'TV': [100, 200, 300]})
pred = lm.predict(X_neu)
pred
```

    0    11.786258
    1    16.539922
    2    21.293586
    dtype: float64

### The quality of an estimate can also be made again with training and test data. With an OLS model, however, this is only very rarely necessary due to the large number of estimation statistics.

```python
from sklearn.model_selection import train_test_split
train, test = train_test_split(df, test_size=0.2)
```

```python
lm = smf.ols(formula='Sales ~ TV', data=train).fit()
```

```python
lm.summary()
```

OLS Regression Results

  Dep. Variable:	          Sales	        R-squared:         	    0.585

  Model:	                   OLS	         Adj. R-squared:    	    0.583

  Method:	             Least Squares	    F-statistic:       	    222.9

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 5.33e-32

  Time:	                 11:56:55	       Log-Likelihood:    	  -417.26

  No. Observations:	         160	        AIC:               	    838.5

  Df Residuals:	             158	        BIC:               	    844.7

  Df Model:	                   1

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	     7.3339	     0.535	    13.702	  0.000	     6.277	     8.391

  TV	            0.0461	     0.003	    14.930	  0.000	     0.040	     0.052

  Omnibus:	        0.476	   Durbin-Watson:     	    1.920

  Prob(Omnibus):	  0.788	   Jarque-Bera (JB):  	    0.620

  Skew:	          -0.097	   Prob(JB):          	    0.733

  Kurtosis:	       2.765	   Cond. No.          	     355.

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

```python
y_hat = lm.predict(test)
y_hat.head()
```

    192     8.126934
    108     7.937909
    80     10.856274
    23     17.859427
    139    15.858526
    dtype: float64

```python
# Error estimation over sklearn
from sklearn.metrics import mean_squared_error
from math import sqrt
```

```python
rmse = sqrt(mean_squared_error(test.Sales, y_hat))
me = (test.Sales - y_hat).sum() / len(test)
print('ME: {0:.4f}, RMSE: {1:.4f}.'.format(me,rmse))
```

    ME: -0.4533, RMSE: 3.0903.

## Multivariate lineare Regression

Das Schätzverfahren lässt sich (theoretisch) beliebig erweitern. Sobald wir y mit mehr als einer Variablen erklären wollen, schätzen wir eine **Multivariatie Regression**:

$y = \beta_0 + \beta_1x_1 + ... + \beta_nx_n$

 Jedes $x$ ist ein eigenes "Feature" und hat dementsprechend einen eigenen Koeffizienten:

$y = \beta_0 + \beta_1 \times TV + \beta_2 \times Radio + \beta_3 \times Newspaper$

```python
df.head()
```

      ID
      TV
      Radio
      Newspaper
      Sales

      0
      1
      230.1
      37.8
      69.2
      22.1

      1
      2
      44.5
      39.3
      45.1
      10.4

      2
      3
      17.2
      45.9
      69.3
      9.3

      3
      4
      151.5
      41.3
      58.5
      18.5

      4
      5
      180.8
      10.8
      58.4
      12.9

```python
# Estimation
lm = smf.ols(formula='Sales ~ TV + Radio + Newspaper', data=df).fit()

# Coefficients
lm.params
```

    Intercept    2.938889
    TV           0.045765
    Radio        0.188530
    Newspaper   -0.001037
    dtype: float64

```python
lm.summary()
```

OLS Regression Results

  Dep. Variable:	          Sales	        R-squared:         	    0.897

  Model:	                   OLS	         Adj. R-squared:    	    0.896

  Method:	             Least Squares	    F-statistic:       	    570.3

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 1.58e-96

  Time:	                 11:59:21	       Log-Likelihood:    	  -386.18

  No. Observations:	         200	        AIC:               	    780.4

  Df Residuals:	             196	        BIC:               	    793.6

  Df Model:	                   3

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	     2.9389	     0.312	     9.422	  0.000	     2.324	     3.554

  TV	            0.0458	     0.001	    32.809	  0.000	     0.043	     0.049

  Radio	         0.1885	     0.009	    21.893	  0.000	     0.172	     0.206

  Newspaper	    -0.0010	     0.006	    -0.177	  0.860	    -0.013	     0.011

  Omnibus:	       60.414	   Durbin-Watson:     	    2.084

  Prob(Omnibus):	  0.000	   Jarque-Bera (JB):  	  151.241

  Skew:	          -1.327	   Prob(JB):          	 1.44e-33

  Kurtosis:	       6.332	   Cond. No.          	     454.

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 8]
fig, axs = plt.subplots(2, 1)
lm.resid.plot(ax=axs[0])
lm.resid.plot(kind='hist', bins=25, ax=axs[1])
plt.show()
```

### The estimate is already better (has a higher adj. R-squared), but it also has significantly more noticeable (not normally distributed) residuals, so we should remove the insignificant feature.

```python
lm = smf.ols(formula='Sales ~ TV + Radio', data=df).fit()
lm.summary()
```

OLS Regression Results

  Dep. Variable:	          Sales	        R-squared:         	    0.897

  Model:	                   OLS	         Adj. R-squared:    	    0.896

  Method:	             Least Squares	    F-statistic:       	    859.6

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 4.83e-98

  Time:	                 12:03:07	       Log-Likelihood:    	  -386.20

  No. Observations:	         200	        AIC:               	    778.4

  Df Residuals:	             197	        BIC:               	    788.3

  Df Model:	                   2

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	     2.9211	     0.294	     9.919	  0.000	     2.340	     3.502

  TV	            0.0458	     0.001	    32.909	  0.000	     0.043	     0.048

  Radio	         0.1880	     0.008	    23.382	  0.000	     0.172	     0.204

  Omnibus:	       60.022	   Durbin-Watson:     	    2.081

  Prob(Omnibus):	  0.000	   Jarque-Bera (JB):  	  148.679

  Skew:	          -1.323	   Prob(JB):          	 5.19e-33

  Kurtosis:	       6.292	   Cond. No.          	     425.

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

```python
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 8]
fig, axs = plt.subplots(2, 1)
lm.resid.plot(ax=axs[0])
lm.resid.plot(kind='hist', bins=25, ax=axs[1])
plt.show()
```

### Logarithmic relationships can also be measured

```python
lm = smf.ols(formula='np.log(Sales) ~ np.log(TV) + np.log(Radio + 0.00000001)', data=df).fit()
lm.summary()
```

OLS Regression Results

  Dep. Variable:	      np.log(Sales)	    R-squared:         	    0.823

  Model:	                   OLS	         Adj. R-squared:    	    0.821

  Method:	             Least Squares	    F-statistic:       	    458.4

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 7.84e-75

  Time:	                 12:04:07	       Log-Likelihood:    	   66.155

  No. Observations:	         200	        AIC:               	   -126.3

  Df Residuals:	             197	        BIC:               	   -116.4

  Df Model:	                   2

  Covariance Type:	      nonrobust

               	                 coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	                      0.7467	     0.061	    12.177	  0.000	     0.626	     0.868

  np.log(TV)	                     0.3516	     0.012	    28.456	  0.000	     0.327	     0.376

  np.log(Radio + 0.00000001)	     0.0649	     0.007	     9.498	  0.000	     0.051	     0.078

  Omnibus:	       65.501	   Durbin-Watson:     	    1.803

  Prob(Omnibus):	  0.000	   Jarque-Bera (JB):  	  338.943

  Skew:	           1.138	   Prob(JB):          	 2.51e-74

  Kurtosis:	       8.958	   Cond. No.          	     28.4

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

```python
lm.resid
```

    0      0.201211
    1      0.022522
    2      0.234834
    3      0.164581
    4     -0.171073
    5      0.214404
    6      0.070447
    7     -0.043245
    8      0.017286
    9     -0.310161
    10    -0.182454
    11     0.015936
    12     0.127256
    13    -0.216273
    14     0.101235
    15     0.257064
    16     0.063035
    17     0.226443
    18    -0.007461
    19    -0.026788
    20     0.034526
    21    -0.249626
    22    -0.110562
    23    -0.098525
    24    -0.091624
    25    -0.301884
    26    -0.002318
    27    -0.089989
    28     0.038922
    29    -0.071846
    30     0.102973
    31    -0.117171
    32    -0.120246
    33    -0.047025
    34    -0.120773
    35    -0.282951
    36     0.278665
    37     0.171619
    38     0.029575
    39     0.177081
    40    -0.005798
    41     0.044977
    42     0.070267
    43    -0.202168
    44     0.049652
    45    -0.063309
    46    -0.115390
    47     0.229063
    48    -0.138672
    49    -0.111859
    50    -0.248816
    51    -0.143620
    52     0.238791
    53     0.227896
    54     0.082383
    55     0.304965
    56     0.042704
    57    -0.085839
    58     0.288485
    59     0.065037
    60    -0.098907
    61     0.239374
    62    -0.096635
    63     0.044157
    64     0.185710
    65    -0.149937
    66     0.083872
    67    -0.060493
    68     0.054460
    69     0.221442
    70     0.077119
    71    -0.053491
    72     0.045073
    73    -0.171343
    74    -0.006788
    75     0.177515
    76    -0.010813
    77     0.004608
    78     0.107617
    79    -0.152430
    80    -0.016110
    81    -0.255128
    82    -0.036524
    83     0.131595
    84     0.200904
    85    -0.064902
    86    -0.000758
    87     0.130811
    88     0.025113
    89     0.165906
    90    -0.156576
    91     0.035940
    92     0.098218
    93     0.177573
    94    -0.119695
    95     0.065129
    96    -0.226824
    97    -0.038498
    98     0.252110
    99     0.131111
    100   -0.281748
    101    0.188935
    102   -0.183343
    103   -0.084194
    104    0.129910
    105    0.227230
    106   -0.059860
    107   -0.088743
    108    0.076039
    109    0.076719
    110   -0.193307
    111    0.169910
    112   -0.095093
    113   -0.055850
    114    0.152205
    115    0.037955
    116   -0.153158
    117   -0.015856
    118    0.086070
    119   -0.082033
    120    0.040218
    121   -0.031923
    122   -0.255008
    123    0.052571
    124    0.097374
    125   -0.116846
    126    0.180629
    127    1.082072
    128    0.310896
    129   -0.072882
    130   -0.390074
    131   -0.236047
    132    0.031191
    133    0.105100
    134    0.127266
    135    0.091303
    136    0.126877
    137    0.097008
    138   -0.018406
    139    0.202942
    140   -0.052066
    141    0.125339
    142    0.129756
    143   -0.152635
    144   -0.093319
    145   -0.194230
    146   -0.222388
    147    0.304076
    148    0.123359
    149    0.018986
    150   -0.120503
    151   -0.119802
    152   -0.000033
    153    0.150632
    154   -0.037846
    155   -0.238670
    156    0.139461
    157   -0.212244
    158    0.142317
    159   -0.094255
    160   -0.078061
    161    0.044139
    162   -0.074925
    163    0.117871
    164   -0.119368
    165   -0.268183
    166    0.083181
    167   -0.226667
    168   -0.001510
    169   -0.178163
    170   -0.152833
    171   -0.063781
    172    0.040635
    173   -0.216513
    174   -0.283749
    175    0.319674
    176    0.098980
    177   -0.226353
    178   -0.309430
    179   -0.158737
    180   -0.233994
    181   -0.248464
    182   -0.112724
    183    0.284612
    184   -0.023707
    185    0.252730
    186   -0.198715
    187    0.039502
    188   -0.139579
    189   -0.035945
    190    0.099257
    191   -0.128770
    192   -0.063473
    193    0.187426
    194    0.111356
    195   -0.084128
    196   -0.175685
    197   -0.161683
    198    0.263982
    199   -0.206073
    dtype: float64

```python

```

## What can we do if we have text or category information?

```python
np.random.seed(12345)
nums = np.random.rand(len(df))
mask_large = nums > 0.5

df['Size'] = 'small'
df.loc[mask_large, 'Size'] = 'large'
df.head()
```

      ID
      TV
      Radio
      Newspaper
      Sales
      Size

      0
      1
      230.1
      37.8
      69.2
      22.1
      large

      1
      2
      44.5
      39.3
      45.1
      10.4
      small

      2
      3
      17.2
      45.9
      69.3
      9.3
      small

      3
      4
      151.5
      41.3
      58.5
      18.5
      small

      4
      5
      180.8
      10.8
      58.4
      12.9
      large

### The easiest way is to simply convert the categories into dummy variables (variables that can take 0 and 1).

```python
df['IsLarge'] = df.Size.map({'small':0, 'large':1})
df.head()
```

      ID
      TV
      Radio
      Newspaper
      Sales
      Size
      IsLarge

      0
      1
      230.1
      37.8
      69.2
      22.1
      large
      1

      1
      2
      44.5
      39.3
      45.1
      10.4
      small
      0

      2
      3
      17.2
      45.9
      69.3
      9.3
      small
      0

      3
      4
      151.5
      41.3
      58.5
      18.5
      small
      0

      4
      5
      180.8
      10.8
      58.4
      12.9
      large
      1

### Dummy variables can easily be included in the prediction.
#### ATTENTION: In this case, the dummy variable was created randomly and has no meaning. This is for demonstration purposes only.

```python
lm = smf.ols(formula='Sales ~ TV + Radio + IsLarge', data=df).fit()
lm.summary()
```

OLS Regression Results

  Dep. Variable:	          Sales	        R-squared:         	    0.897

  Model:	                   OLS	         Adj. R-squared:    	    0.896

  Method:	             Least Squares	    F-statistic:       	    570.3

  Date:	             Mon, 27 Jul 2020	   Prob (F-statistic):	 1.56e-96

  Time:	                 12:07:46	       Log-Likelihood:    	  -386.17

  No. Observations:	         200	        AIC:               	    780.3

  Df Residuals:	             196	        BIC:               	    793.5

  Df Model:	                   3

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	  [0.025	    0.975]

  Intercept	     2.8938	     0.318	     9.092	  0.000	     2.266	     3.522

  TV	            0.0457	     0.001	    32.493	  0.000	     0.043	     0.048

  Radio	         0.1882	     0.008	    23.258	  0.000	     0.172	     0.204

  IsLarge	       0.0555	     0.242	     0.229	  0.819	    -0.422	     0.533

  Omnibus:	       59.724	   Durbin-Watson:     	    2.082

  Prob(Omnibus):	  0.000	   Jarque-Bera (JB):  	  147.220

  Skew:	          -1.319	   Prob(JB):          	 1.08e-32

  Kurtosis:	       6.272	   Cond. No.          	     490.

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

### If there are many different text values in a column, we can easily create dummy variables with the "OneHotEncoder" from the sklearn module.

```python
from sklearn import preprocessing
```

```python
df_titanic = sns.load_dataset('titanic')
df_titanic.head()
```

      survived
      pclass
      sex
      age
      sibsp
      parch
      fare
      embarked
      class
      who
      adult_male
      deck
      embark_town
      alive
      alone

      0
      0
      3
      male
      22.0
      1
      0
      7.2500
      S
      Third
      man
      True
      NaN
      Southampton
      no
      False

      1
      1
      1
      female
      38.0
      1
      0
      71.2833
      C
      First
      woman
      False
      C
      Cherbourg
      yes
      False

      2
      1
      3
      female
      26.0
      0
      0
      7.9250
      S
      Third
      woman
      False
      NaN
      Southampton
      yes
      True

      3
      1
      1
      female
      35.0
      1
      0
      53.1000
      S
      First
      woman
      False
      C
      Southampton
      yes
      False

      4
      0
      3
      male
      35.0
      0
      0
      8.0500
      S
      Third
      man
      True
      NaN
      Southampton
      no
      True

```python
my_df_summary(df_titanic)
```

    Dataset has 891 rows.

      survived
      pclass
      sex
      age
      sibsp
      parch
      fare
      class
      who
      adult_male
      alive
      alone
      embarked
      deck
      embark_town

      Minimum
      0
      1
      female
      0.42
      0
      0
      0
      First
      child
      False
      no
      False
      NaN
      NaN
      NaN

      Maximum
      1
      3
      male
      80
      8
      6
      512.329
      Third
      woman
      True
      yes
      True
      NaN
      NaN
      NaN

      Mean
      0.383838
      2.30864
      NaN
      29.6991
      0.523008
      0.381594
      32.2042
      NaN
      NaN
      0.602694
      NaN
      0.602694
      NaN
      NaN
      NaN

      Stand. Dev.
      0.486592
      0.836071
      NaN
      14.5265
      1.10274
      0.806057
      49.6934
      NaN
      NaN
      0.489615
      NaN
      0.489615
      NaN
      NaN
      NaN

      #NA
      0
      0
      0
      177
      0
      0
      0
      0
      0
      0
      0
      0
      2
      688
      2

      #Uniques
      2
      3
      2
      88
      7
      7
      248
      3
      3
      2
      2
      2
      3
      7
      3

      dtypes
      int64
      int64
      object
      float64
      int64
      int64
      float64
      category
      object
      bool
      object
      bool
      object
      category
      object

```python
df_text = df_titanic[['class']].dropna()
my_df_summary(df_text)
```

    Dataset has 891 rows.

      class

      Minimum
      First

      Maximum
      Third

      Mean
      NaN

      Stand. Dev.
      NaN

      #NA
      0

      #Uniques
      3

      dtypes
      category

### Text-Encoder

```python
enc = preprocessing.OneHotEncoder()
```

```python
enc.fit(df_text)
```

    OneHotEncoder(categories='auto', drop=None, dtype=<class 'numpy.float64'>,
                  handle_unknown='error', sparse=True)

```python
onehotlabels = enc.transform(df_text).toarray()
df_dummys = pd.DataFrame(onehotlabels)
df_dummys.sum(axis=1).describe()
```

    count    891.0
    mean       1.0
    std        0.0
    min        1.0
    25%        1.0
    50%        1.0
    75%        1.0
    max        1.0
    dtype: float64

```python
dir(enc)
```

    ['__class__',
     '__delattr__',
     '__dict__',
     '__dir__',
     '__doc__',
     '__eq__',
     '__format__',
     '__ge__',
     '__getattribute__',
     '__getstate__',
     '__gt__',
     '__hash__',
     '__init__',
     '__init_subclass__',
     '__le__',
     '__lt__',
     '__module__',
     '__ne__',
     '__new__',
     '__reduce__',
     '__reduce_ex__',
     '__repr__',
     '__setattr__',
     '__setstate__',
     '__sizeof__',
     '__str__',
     '__subclasshook__',
     '__weakref__',
     '_check_X',
     '_compute_drop_idx',
     '_fit',
     '_get_feature',
     '_get_param_names',
     '_get_tags',
     '_more_tags',
     '_transform',
     '_validate_keywords',
     'categories',
     'categories_',
     'drop',
     'drop_idx_',
     'dtype',
     'fit',
     'fit_transform',
     'get_feature_names',
     'get_params',
     'handle_unknown',
     'inverse_transform',
     'set_params',
     'sparse',
     'transform']

### Sometimes (NOT WITH LINEAR REGRESSION) it can also be helpful not to label in dummies, but simply in numerical values. This is very easy with the label encoder.

```python
le = preprocessing.LabelEncoder()
```

```python
df_labels = df_text.apply(le.fit_transform)
df_labels.head()
```

      class

      0
      2

      1
      0

      2
      2

      3
      0

      4
      2


# JAX implementation of temporal parallelisation of Kalman filter and smoother

This notebook implements the algorithms in the paper by S. Särkkä and Á. García-Fernández by leveraging the [JAX](https://github.com/google/jax) framework.

## Author: Adrien Corenflos

## Reference:
Simo Särkkä and Ángel F. García-Fernández (2021). Temporal Parallelization of Bayesian Smoothers.
IEEE Transactions on Automatic Control, Volume: 66, Issue: 1, Pages 299-306.
arXiv: https://arxiv.org/abs/1905.13002
DOI: https://doi.org/10.1109/TAC.2020.2976316

- If you are on Colab don't forget to switch the Runtime (in the tab with the same name) type to GPU.
- If you are using a personal machine, to install JAX with GPU support please read JAX README.md [file](https://github.com/google/jax).

```
!pip install -U jax==0.1.77 jaxlib==0.1.55+cuda101 -f https://storage.googleapis.com/jax-releases/jax_releases.html
```

    Looking in links: https://storage.googleapis.com/jax-releases/jax_releases.html
    Requirement already up-to-date: jax==0.1.77 in /usr/local/lib/python3.6/dist-packages (0.1.77)
    Collecting jaxlib==0.1.55+cuda101
      Using cached https://storage.googleapis.com/jax-releases/cuda101/jaxlib-0.1.55%2Bcuda101-cp36-none-manylinux2010_x86_64.whl
    Requirement already satisfied, skipping upgrade: numpy>=1.12 in /usr/local/lib/python3.6/dist-packages (from jax==0.1.77) (1.19.4)
    Requirement already satisfied, skipping upgrade: opt-einsum in /usr/local/lib/python3.6/dist-packages (from jax==0.1.77) (3.3.0)
    Requirement already satisfied, skipping upgrade: absl-py in /usr/local/lib/python3.6/dist-packages (from jax==0.1.77) (0.10.0)
    Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from jaxlib==0.1.55+cuda101) (1.4.1)
    Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from absl-py->jax==0.1.77) (1.15.0)
    Installing collected packages: jaxlib
      Found existing installation: jaxlib 0.1.55
        Uninstalling jaxlib-0.1.55:
          Successfully uninstalled jaxlib-0.1.55
    Successfully installed jaxlib-0.1.55

The above is necessary at the moment due to a batching (vectorization) bug in LU factorisation that was only fixed on 16 Dec 2020. It will be removed when next release of JAX (>2.7.0) reaches Colab. There should not be any interface change otherwise.

### Imports

```
# Python specific imports that will make our job easier and our code prettier
from collections import namedtuple
from functools import partial
import time
from tqdm.auto import trange, tqdm

# JAX specific imports that we will use to code the logic
from jax import jit, vmap, make_jaxpr, device_put, devices
from jax.config import config
from jax.core import eval_jaxpr  # this will serve as a proxy compiler as JAX doesn't have an AOT function
from jax.lax import associative_scan, psum, scan
import jax.numpy as jnp
import jax.scipy as jsc

# Auxiliary libraries that we will use to report results and create the data
import math
import matplotlib.pyplot as plt
import numpy as np
import scipy as sc
```

```
config.update("jax_enable_x64", True)  # We use this so that we have the same precision as the pure numpy implementation
                                       # This can be useful in particular for large observations (long running series)
```

### Utilities
We define some utilities that will make the code more readable

```
LOG2PI = math.log(2 * math.pi)
```

### Model
In the vein of Example 3.6 in the book Bayesian Filtering and Smoothing (S. Särkkä) we consider a simple car tracking model where the car is characterised by its coordinates and directional speeds $\mathbf{z} = [x, y, \dot{x}, \dot{y}]^\top$ for which only the positional components are observed with some uncertainty. Formally this corresponds to the following state-space model:

\begin{equation}
\begin{split}
  x_k &=  F \, x_{k-1} + q_{k-1},\\
  y_k &=  H \, x_k + r_k,
\end{split}
\end{equation}

where $q_k \sim \mathrm{N}(0,Q)$, $r_k \sim \mathrm{N}(0,R)$, and
 \begin{equation}
  F = \begin{pmatrix}
    1 & 0 & \Delta t & 0 \\
    0 & 1 & 0 & \Delta t \\
    0 & 0 & 1 & 0 \\
    0 & 0 & 0 & 1
  \end{pmatrix}
  ,\quad
  Q = q \, \begin{pmatrix}
     \frac{\Delta t^3}{3} & 0 & \frac{\Delta t^2}{2} & 0 \\
     0 & \frac{\Delta t^3}{3} & 0 & \frac{\Delta t^2}{2} \\
     \frac{\Delta t^2}{2} & 0 & \Delta t & 0     \\
     0 & \frac{\Delta t^2}{2} & 0 & \Delta t
  \end{pmatrix},
\end{equation}

along with
\begin{equation}
  H = \begin{pmatrix}
    1 & 0 & 0 & 0 \\
    0 & 1 & 0 & 0
  \end{pmatrix}, \qquad
  R = \begin{pmatrix}
    \sigma^2 & 0  \\
    0 & \sigma^2
  \end{pmatrix}.
\end{equation}

which translates into the following code:

```
StateSpaceModel = namedtuple("StateSpaceModel", ["F", "H", "Q", "R", "m0", "P0", "xdim", "ydim"])
```

```
def make_car_tracking_model(q: float, dt: float, r: float, m0: np.ndarray, P0: np.ndarray):
    F = np.eye(4) + dt * np.eye(4, k=2)
    H = np.eye(2, 4)
    Q = np.kron(np.array([[dt**3/3, dt**2/2],
                          [dt**2/2, dt]]),
                np.eye(2))
    R = r ** 2 * np.eye(2)
    return StateSpaceModel(F, H, q * Q, R, m0, P0, m0.shape[0], H.shape[0])

```

```
car_tracking_model = make_car_tracking_model(q=1., dt=0.1, r=0.5,
                                             m0=np.array([0., 0., 1., -1.]),
                                             P0=np.eye(4))
```

So that we can now write some generic code to sample from a SSM:

```
def get_data(model: StateSpaceModel, T:float, seed:int=0):
    # We first generate the normals we will be using to simulate the SSM:
    rng = np.random.RandomState(seed)
    normals = rng.randn(1 + T, model.xdim + model.ydim)

    # Then we allocate the arrays where the simulated path and observations will
    # be stored:
    xs = np.empty((T, model.xdim))
    ys = np.empty((T, model.ydim))

    # So that we can now run the sampling routine:
    Q_chol = sc.linalg.cholesky(model.Q, lower=True)
    R_chol = sc.linalg.cholesky(model.R, lower=True)
    P0_chol = sc.linalg.cholesky(model.P0, lower=True)
    x = model.m0 + P0_chol @ normals[0, :model.xdim]
    for i, norm in enumerate(normals[1:]):
        x = model.F @ x + Q_chol @ norm[:model.xdim]
        y = model.H @ x + R_chol @ norm[model.xdim:]
        xs[i] = x
        ys[i] = y
    return xs, ys
```

```
log10T = 4
```

```
true_xs, ys = get_data(car_tracking_model, 10 ** log10T, 0)
```

This corresponds to the following car positions in space (truncated at 100 timesteps):

```
fig, ax = plt.subplots(figsize=(7, 7))
ax.plot(true_xs[:100, 0], true_xs[:100, 1], label="True State", color="b")
ax.scatter(*ys[:100].T, label="Observations", color="r")
_ = plt.legend()
```

### Sequential Kalman filter and smoother
We can now write the Kalman filter and smoother sequential routines in a JAX fashion. For more information on these algorithms, we refer the reader to the book:

Särkkä, S. (2013). Bayesian filtering and smoothing. Cambridge University Press.

For which an online version is also available on his [webpage](https://users.aalto.fi/~ssarkka/).

#### Filter:

```
def mvn_logpdf(x, mean, cov):
    n = mean.shape[0]
    upper = jsc.linalg.cholesky(cov, lower=False)
    log_det = 2 * jnp.sum(jnp.log(jnp.abs(jnp.diag(upper))))
    diff = x - mean
    scaled_diff = jsc.linalg.solve_triangular(upper, diff.T, lower=False)
    distance = jnp.sum(scaled_diff * scaled_diff, 0)
    return -0.5 * (distance + n * LOG2PI + log_det)
```

```
def kf(model, observations):
    def body(carry, y):
        m, P = carry
        m = model.F @ m
        P = model.F @ P @ model.F.T + model.Q

        obs_mean = model.H @ m
        S = model.H @ P @ model.H.T + model.R

        K = jsc.linalg.solve(S, model.H @ P, sym_pos=True).T  # notice the jsc here
        m = m + K @ (y - model.H @ m)
        P = P - K @ S @ K.T
        return (m, P), (m, P)

    _, (fms, fPs) = scan(body, (model.m0, model.P0), observations)
    return fms, fPs

```

```
fms, fPs = kf(car_tracking_model, ys[:100])
```

#### Smoother:

```
def ks(model, ms, Ps):
    def body(carry, inp):
        m, P = inp
        sm, sP = carry

        pm = model.F @ m
        pP = model.F @ P @ model.F.T + model.Q

        C = jsc.linalg.solve(pP, model.F @ P, sym_pos=True).T  # notice the jsc here

        sm = m + C @ (sm - pm)
        sP = P + C @ (sP - pP) @ C.T
        return (sm, sP), (sm, sP)

    _, (sms, sPs) = scan(body, (ms[-1], Ps[-1]), (ms[:-1], Ps[:-1]), reverse=True)
    sms = jnp.append(sms, jnp.expand_dims(ms[-1], 0), 0)
    sPs = jnp.append(sPs, jnp.expand_dims(Ps[-1], 0), 0)
    return sms, sPs

```

```
sms, sPs = ks(car_tracking_model, fms, fPs)
```

So that we can now plot the result agains the true state and the observations:

```
fig, ax = plt.subplots(figsize=(7, 7))
ax.plot(true_xs[:100, 0], true_xs[:100, 1], label="True State", color="b")
ax.plot(fms[:100, 0], fms[:100, 1], label="Filtered", color="g", linestyle="--")
ax.plot(sms[:100, 0], sms[:100, 1], label="Smoothed", color="k", linestyle="--")
ax.scatter(*ys[:100].T, label="Observations", color="r")
_ = plt.legend()
```

#### Filter-Smoother
We can finally combine the filter and the smoother into a single filter-smoother routine:

```
def kfs(model, observations):
    return ks(model, *kf(model, observations, False))
```

### Parallel Kalman filter and smoother
Now that we have a baseline for the sequential model, we can go on and implement the parallel version of the routine, leveraging the `associative_scan` function already offered by JAX. To do this we will need to do a bit more work, but before this let us offer some intuition around what exactly is happening.

#### Prefix-Sum / Associative Scan
Consider the simple problem of computing the cumulative sum of a given array of floats:

```
array_of_floats = np.arange(2 ** 4, dtype=np.float32)
```

A simple way to do this would be to iterate (scan) through the array from left to right and accumulate the result as we go:

```
def cumulative_sum(arr):
    # obviously numpy implements cumulative sums already, but this is for illustrative purposes.
    res = np.empty_like(arr)
    temp = 0.
    for i, val in enumerate(arr):
        temp += val
        res[i] = temp
    return res
```

```
cumulative_sum(array_of_floats)
```

    array([  0.,   1.,   3.,   6.,  10.,  15.,  21.,  28.,  36.,  45.,  55.,
            66.,  78.,  91., 105., 120.], dtype=float32)

However, the addition operation has the fundamental property to be associative: $(a+b) + c = a + (b + c)$, under which accumulative scans have been proven to be parallelizable (Blelloch, 1990, see the paper for a reference) up to a span complexity of $O(log(N))$ in the context where many processors are available. If we were to write the algorithm in pure python, it would look like the below:

```
def prefix_sum(arr):
    # This is for illustration purposes only, and for instance doesn't take into
    # account the case when the array size is not a pure power of 2
    res = np.copy(arr)
    n = res.shape[0]
    log_n  = np.log2(n).astype(int)

    # Up pass
    for d in range(log_n):
        # this loop can't be done in parallel so it defines the span complexity under
        # parallelization
        for i in range(0, n, 2 ** (d + 1)):
            # this should be done in parallel, therefore would not be taken
            # into account in the span complexity provided we have at least
            # n / 2^{d+1} cores on our GPU
            i1 = i + 2 ** d - 1
            i2 = i + 2 ** (d + 1) - 1
            res[i2] += res[i1]

    res[-1] = 0

    # Down pass
    for d in range(log_n-1, -1, -1):
        # this loop can't be done in parallel so it defines the span complexity under
        # parallelization
        for i in range(0, n, 2 ** (d + 1)):
            # this should be done in parallel, therefore would not be taken
            # into account in the span complexity provided we have at least
            # n / 2^{d+1} cores on our GPU
            i1 = i + 2 ** d - 1
            i2 = i + 2 ** (d + 1) - 1

            res[i1], res[i2] = res[i2], res[i1] + res[i2]

    # Extra pass
    res += arr

    return res
```

```
prefix_sum(array_of_floats)
```

    array([  0.,   1.,   3.,   6.,  10.,  15.,  21.,  28.,  36.,  45.,  55.,
            66.,  78.,  91., 105., 120.], dtype=float32)

#### Filter:

In the case of Kalman filters, one can think of the addition as being the bayesian map operator coming from the fact of "adding" one observation to the model:

\begin{equation}
    p(x_k \mid y_{1:k-1})
    = \int p(x_k \mid x_{k-1})
       \, p(x_{k-1} \mid y_{1:k-1}) \, \mathrm{d}x_{k-1},
\end{equation}

\begin{equation}
    p(x_k \mid y_{1:k})
    = \frac{p(y_k \mid x_k)
       \, p(x_k \mid y_{1:k-1})}
       {\int p(y_k \mid x_k)
          \, p(x_k \mid y_{1:k-1}) \, \mathrm{d}x_k}.
\end{equation}

As it is, this operation on the random variables $x_k$'s is not associative as switching around the observations will result in a completely different result. However S. Särkkä and Á. García-Fernández showed that these could be rewritten in terms of the associative operator given by:

\begin{align*}
    \left(f_{i},g_{i}\right)\otimes\left(f_{j},g_{j}\right) & =\left(f_{ij},g_{ij}\right),
\end{align*}
\begin{align*}
    f_{ij}\left(x\mid z\right) & =\frac{\int g_{j}\left(y\right)f_{j}\left(x\mid y\right)f_{i}\left(y\mid z\right)\mathrm{d}y}{\int g_{j}\left(y\right)f_{i}\left(y\mid z\right)\mathrm{d}y}, \\
    g_{ij}\left(z\right) & =g_{i}\left(z\right)\int g_{j}\left(y\right)f_{i}\left(y\mid z\right)\mathrm{d}y.
\end{align*}

This could in turn be rewritten in terms of the linear equations of the state parameters, which leads to the below algorithm:

##### Initialisation of the associative elements:
We write the function that will initialise them (in parallel) based on the data and the observations:

```
def first_filtering_element(model, y):
    S = model.H @ model.Q @ model.H.T + model.R
    CF, low = jsc.linalg.cho_factor(S)  # note the jsc

    m1 = model.F @ model.m0
    P1 = model.F @ model.P0 @ model.F.T + model.Q
    S1 = model.H @ P1 @ model.H.T + model.R
    K1 = jsc.linalg.solve(S1, model.H @ P1, sym_pos=True).T  # note the jsc

    A = jnp.zeros_like(model.F)
    b = m1 + K1 @ (y - model.H @ m1)
    C = P1 - K1 @ S1 @ K1.T

    # note the jsc
    eta = model.F.T @ model.H.T @ jsc.linalg.cho_solve((CF, low), y)
    J = model.F.T @ model.H.T @ jsc.linalg.cho_solve((CF, low), model.H @ model.F)
    return A, b, C, J, eta

def generic_filtering_element(model, y):
    S = model.H @ model.Q @ model.H.T + model.R
    CF, low = jsc.linalg.cho_factor(S)  # note the jsc
    K = jsc.linalg.cho_solve((CF, low), model.H @ model.Q).T  # note the jsc
    A = model.F - K @ model.H @ model.F
    b = K @ y
    C = model.Q - K @ model.H @ model.Q

    # note the jsc
    eta = model.F.T @ model.H.T @ jsc.linalg.cho_solve((CF, low), y)
    J = model.F.T @ model.H.T @ jsc.linalg.cho_solve((CF, low), model.H @ model.F)
    return A, b, C, J, eta
```

And we can combine them properly:

```
def make_associative_filtering_elements(model, observations):
    first_elems = first_filtering_element(model, observations[0])
    generic_elems = vmap(lambda o: generic_filtering_element(model, o))(observations[1:])
    return tuple(jnp.concatenate([jnp.expand_dims(first_e, 0), gen_es])
                 for first_e, gen_es in zip(first_elems, generic_elems))

```

##### Associative filtering operator
We now define the associative filtering operator

```
@vmap
def filtering_operator(elem1, elem2):
    # # note the jsc everywhere
    A1, b1, C1, J1, eta1 = elem1
    A2, b2, C2, J2, eta2 = elem2
    dim = A1.shape[0]
    I = jnp.eye(dim)  # note the jnp

    I_C1J2 = I + C1 @ J2
    temp = jsc.linalg.solve(I_C1J2.T, A2.T, sym_pos=False).T
    A = temp @ A1
    b = temp @ (b1 + C1 @ eta2) + b2
    C = temp @ C1 @ A2.T + C2

    I_J2C1 = I + J2 @ C1
    temp = jsc.linalg.solve(I_J2C1.T, A1, sym_pos=False).T

    eta = temp @ (eta2 - J2 @ b1) + eta1
    J = temp @ J2 @ A1 + J1

    return A, b, C, J, eta
```

##### Parallel routine
And we can now combine all these to compute the filtering result in parallel:

```
def pkf(model, observations):
    initial_elements = make_associative_filtering_elements(model, observations)
    final_elements = associative_scan(filtering_operator, initial_elements)
    return final_elements[1], final_elements[2]
```

That we can compare with the sequential one:

```
pfms, pfPs = pkf(car_tracking_model, ys[:100])
```

and we get a maximum absolute difference between the two methods of

```
np.abs(pfms - fms).max()
```

    2.353672812205332e-14

#### Smoother:

The parallel implementation of the smoother follows similar principles and we refer the reader to the paper for more details about the associative formulation.

##### Initialisation of the associative elements:
We now write the function that will initialise them (in parallel) based on the data and the observations:

```
def last_smoothing_element(m, P):
    return jnp.zeros_like(P), m, P

def generic_smoothing_element(model, m, P):
    Pp = model.F @ P @ model.F.T + model.Q

    E  = jsc.linalg.solve(Pp, model.F @ P, sym_pos=True).T
    g  = m - E @ model.F @ m
    L  = P - E @ Pp @ E.T
    return E, g, L
```

And we can combine them properly:

```
def make_associative_smoothing_elements(model, filtering_means, filtering_covariances):
    last_elems = last_smoothing_element(filtering_means[-1], filtering_covariances[-1])
    generic_elems = vmap(lambda m, P: generic_smoothing_element(model, m, P))(filtering_means[:-1], filtering_covariances[:-1])
    return tuple(jnp.append(gen_es, jnp.expand_dims(last_e, 0), axis=0)
                 for gen_es, last_e in zip(generic_elems, last_elems))

```

##### Associative smoothing operator
We now define the associative smoothing operator

```
@vmap
def smoothing_operator(elem1, elem2):
    E1, g1, L1 = elem1
    E2, g2, L2 = elem2

    E = E2 @ E1
    g = E2 @ g1 + g2
    L = E2 @ L1 @ E2.T + L2

    return E, g, L
```

##### Parallel routine
And we can now combine all these to compute the smoothing result in parallel:

```
def pks(model, filtered_means, filtered_covariances):
    initial_elements = make_associative_smoothing_elements(model, filtered_means, filtered_covariances)
    final_elements = associative_scan(smoothing_operator, initial_elements, reverse=True)  # note the vmap
    return final_elements[1], final_elements[2]
```

That we can compare with the sequential one:

```
psms, psPs = pks(car_tracking_model, pfms, pfPs)
```

#### Filter-Smoother:
Finally we can combine all these functions into one filter-smoother routine.

```
def pkfs(model, observations):
    return pks(model, *pkf(model, observations))
```

### Runtime comparison

To compare the runtime of both methods we will compile the routines on GPU and CPU and modify the size of the observation input fed to the routines. It is noteworthy that the parallel version will appear to be much slower due to a slow compilation in JAX. This could be improved by using a different implementation of the associative scan or by fixing the number of levels the way it is done in TensorFlow Probability.

```
cpu_kfs = jit(kfs, backend="cpu")
cpu_pkfs = jit(pkfs, backend="cpu")
gpu_kfs = jit(kfs, backend="gpu")
gpu_pkfs = jit(pkfs, backend="gpu")
```

```
input_sizes = np.logspace(2, log10T, num=10, base=10).astype(int)
```

```
input_sizes
```

    array([  100,   166,   278,   464,   774,  1291,  2154,  3593,  5994,
           10000])

```
cpu = devices("cpu")[0]
gpu = devices("gpu")[0]
```

```
def get_average_runtimes(func, n_iter, device):
    runtimes = np.empty(input_sizes.shape)
    device_model = StateSpaceModel(*(device_put(jnp.asarray(val), device) for val in car_tracking_model))
    for i, input_size in tqdm(enumerate(input_sizes), total=runtimes.shape[0]):
        observations = device_put(jnp.asarray(ys[:input_size]), device)
        res = func(device_model, observations)  # compilation run
        for e in res:
            _ = e.block_until_ready()
        tic = time.time()
        for _ in trange(n_iter, leave=False):
            res = func(device_model, observations)
            for e in res:
                _ = e.block_until_ready()
        runtimes[i] = (time.time() - tic) / n_iter
    return runtimes
```

```
n_iter = 5
```

```
cpu_sequential_runtimes = get_average_runtimes(cpu_kfs, n_iter, cpu)
cpu_parallel_runtimes = get_average_runtimes(cpu_pkfs, n_iter, cpu)
gpu_sequential_runtimes = get_average_runtimes(gpu_kfs, n_iter, gpu)
gpu_parallel_runtimes = get_average_runtimes(gpu_pkfs, n_iter, gpu)
```

    HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))

And we can plot these:

```
fig, axes = plt.subplots(ncols=2, figsize=(15, 6), sharex=True, sharey=True)
axes[0].loglog(input_sizes, cpu_sequential_runtimes, label="Sequential-CPU", linestyle="-.", linewidth=3)
axes[0].loglog(input_sizes, cpu_parallel_runtimes, label="Parallel-CPU", linewidth=3)
axes[0].legend()

axes[1].loglog(input_sizes, gpu_sequential_runtimes, label="Sequential-GPU", linestyle="-.", linewidth=3)
axes[1].loglog(input_sizes, gpu_parallel_runtimes, label="Parallel-GPU", linewidth=3)
_ = axes[0].set_ylabel("Average run time (seconds)")

for ax in axes:
    _ = ax.set_xlabel("Number of data points")

_ = fig.suptitle("Runtime comparison on CPU and GPU", size=15)
_ = axes[1].legend()
```


# ***Introduction to Radar Using Python and MATLAB***
## Andy Harrison - Copyright (C) 2019 Artech House
<br/>

# Rectangular Plate Radar Cross Section
***

Referring to Section 7.4.1.4, consider the scattering from a three-dimensional object; specifically, a rectangular conducting plate, as shown in Figure 7.9.

For TEx polarization, the bistatic radar cross section can be written as (Equation 7.35)

$$
    \sigma = 4\pi\left( \frac{ab}{\lambda}\right)^2 (\cos^2\theta_o \sin^2\phi_o + \cos^2\phi_o)\left[ \frac{\sin(X)}{X}\right]^2\, \left[ \frac{\sin(Y)}{Y}\right]^2 \hspace{0.0in} \text{(m}^2\text{)},
$$

For the TMx polarization, the bistatic radar cross section can be written as (Equation 7.38)

\begin{align}
    \sigma = 4\pi\left( \frac{ab}{\lambda}\right)^2\big[\cos^2\theta_i (&\cos^2\theta_o \cos^2\phi_o + \sin^2\phi_o)\big]\, \nonumber \\ &\times\left[ \frac{\sin(X)}{X}\right]^2\, \left[ \frac{\sin(Y)}{Y}\right]^2 \hspace{0.5in} \text{(m}^2\text{)},
\end{align}

For the monostatic case, the expressions in (Equation 7.35) and (Equation 7.38) reduce to the same equation, given here as

$$
    \sigma = 4\pi\left( \frac{ab}{\lambda}\right)^2\cos^2\theta_i \left[ \frac{\sin(kb\sin\theta_i)}{kb\sin\theta_i}\right]^2 \hspace{0.5in} \text{(m}^2\text{)},
$$

For the TMx polarization, the maximum of scattered field always occurs at the specular angle.  However, this is not true for the TEx polarization, although it does approach the specular angle as the dimensions of the plate become large compared to the wavelength.  As well, the monostatic radar cross section for the flat plate is the same for both TMx and TEx polarizations.  This is also true for the two-dimensional plate.  Measurements have shown that these two values are slightly different.
***

Begin by getting the library path

```python
import lib_path
```

Set the operating frequency (Hz), the plate width (m), the plate length (m), the incident angle theta (degrees) and the observation angle (phi) (degrees)

```python
frequency = 1e9

width = 2.0

length = 5.0

incident_theta = 15.0

observation_phi = 270.0
```

Set up the observation angles theta (degrees) using the `linspace` routine from `scipy`

```python
from numpy import linspace

observation_theta = linspace(-90, 90, 1801)
```

Calculate the radar cross section (m^2) of the rectangular plate

```python
from Libs.rcs.rectangular_plate import radar_cross_section

rcs_tm, rcs_te = radar_cross_section(frequency, width, length, incident_theta, observation_theta, observation_phi)
```

Display the radar cross section (dBsm) of the rectangular plate

```python
from matplotlib import pyplot as plt

from numpy import log10

# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.plot(observation_theta, 10.0 * log10(rcs_te + 1e-10), '', label='TE$^x$')

plt.plot(observation_theta, 10.0 * log10(rcs_tm + 1e-10), '--', label='TM$^x$')

# Set the plot title and labels

plt.title('RCS vs Bistatic Angle', size=14)

plt.ylabel('RCS (dBsm)', size=12)

plt.xlabel('Observation Angle (deg)', size=12)

plt.ylim(min(10.0 * log10(rcs_te + 1e-4)) - 3, max(10.0 * log10(rcs_te + 1e-4)) + 3)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)

# Set the legend

plt.legend(loc='upper right', prop={'size': 10})
```

```python

```


```python
import pandas as pd
import fredpy as fp
import numpy as np
```

# US Seigniorage Data

Seigniorage is the real value of the change in the monetary base:

\begin{align}
\frac{M_{t}-M_{t-1}}{P_t} & = \Delta m_t + \frac{\pi_t}{1+\pi_t}m_{t-1}
\end{align}

where $\Delta m_t = m_t - m_{t-1}$ and $\pi_t = P_t/ P_{t-1} - 1$. The first term on the right-hand side is the revenue from increasing the real monetary base while the seond term is the inflation tax on the existing monetary base.

This program downloads from FRED the following data for the US economy:

* monetary base (BOGMBASE)
* GDP deflator data (A191RD3A086NBEA)

The program returns a CSV file with columns containing:

* average annual real monetary base (trillions of $)
* average annual GDP deflator inflation
* change in real monetary base from preceding year for 1960 only
* inflation tax from preceding year for 1960 only
* total seigniorage for 1960 only

The purpose behind providing incomplete columns is to give students a starting point for completing the columns using Excel or a comparable tool.

## Download and manage data

```python
# Download monetary base and GDP deflator data
m_base = fp.series('BOGMBASE')
gdp_deflator = fp.series('A191RD3A086NBEA')

# Convert monetary base data to annual frequency
m_base = m_base.as_frequency('A')

# Equalize data ranges for monetary base and GDP deflator data
m_base, gdp_deflator = fp.window_equalize([m_base, gdp_deflator])

# GDP deflator base year
base_year = gdp_deflator.units.split(' ')[-1].split('=')[0]
```

```python
# Construct real monetary base
real_m_base = m_base.data/gdp_deflator.data*100/1000/1000

# Construct inflation data
inflation = (gdp_deflator.data/gdp_deflator.data.shift(1))-1
```

## Save data to csv

```python
real_m_base_col_name = 'real monetary base [billions of '+base_year+' dollars]'
inflation_col_name = 'GDP deflator inflation'
delta_m_base_col_name = 'change in real monetary base [billions of '+base_year+' dollars]'
inflation_tax_col_name = 'inflation tax revenue [billions of '+base_year+' dollars]'
total_col_name = 'total seigniorage [billions of '+base_year+' dollars]'

# Put data into DataFrame
df = pd.DataFrame({real_m_base_col_name:real_m_base,inflation_col_name:inflation})

# Add single value to second row of the change in real monetary base column
df.loc[df.index[1],delta_m_base_col_name] = df.loc[df.index[1],real_m_base_col_name] - df.loc[df.index[0],real_m_base_col_name]

# Add single value to second row of the inflation tax revenue column
df.loc[df.index[1],inflation_tax_col_name] = 1/(1+1/df.loc[df.index[1],inflation_col_name]) * df.loc[df.index[0],real_m_base_col_name]

# Add single value to second row of total seigniorage column
df.loc[df.index[1],total_col_name] = df.loc[df.index[1],delta_m_base_col_name] + df.loc[df.index[1],inflation_tax_col_name]

# Save to csv
df.to_csv('../csv/us_seigniorage_data.csv',index=True)
```