```python
import matplotlib.pyplot as plt
import scipy.stats as st
import seaborn as sns
import pandas as pd
from scipy.stats import norm, uniform, expon
from scipy.integrate import quad
from sympy.solvers import solve
from sympy import Symbol
import numpy as np
from pandas import Series, DataFrame
```

```python
werte = np.array([0,10,11])
ew = np.sum(werte*1/3)
ew
```

    7.0

```python
var_X = np.sum((werte-ew)**2*1/3)
var_X
```

    24.666666666666664

```python
sd_X = np.sqrt(var_X)
```

```python
sim = np.random.choice(werte, size=10, replace = True)
sim
```

    array([ 0, 11, 10, 11, 11, 11, 10,  0, 10, 10])

```python
plt.hist(sim, bins = range(0, 13, 1), edgecolor = "black")
```

```python

```


```python
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import json
import sympy
% matplotlib inline
f = open('./exerc_phyton.txt')

```

```python
V=np.genfromtxt(f,skip_header=6,delimiter='')
```

```python
t=V[:,0]
print(t)
```

    [0.     0.0201 0.0402 0.0603 0.0804 0.1005 0.1206 0.1407 0.1608 0.1809
     0.201  0.2211 0.2412 0.2613 0.2814 0.3015 0.3216 0.3417 0.3618 0.3819
     0.402  0.4221 0.4422 0.4623 0.4824 0.5025 0.5226 0.5427 0.5628 0.5829
     0.603  0.6231 0.6432 0.6633 0.6834 0.7035 0.7236 0.7437 0.7638 0.7839
     0.804  0.8241 0.8442 0.8643 0.8844 0.9045 0.9246 0.9447 0.9648 0.9849
     1.005  1.0251 1.0452 1.0653 1.0854 1.1055 1.1256 1.1457 1.1658 1.1859
     1.206  1.2261 1.2462 1.2663 1.2864 1.3065 1.3266 1.3467 1.3668 1.3869
     1.407  1.4271 1.4472 1.4673 1.4874 1.5075 1.5276 1.5477 1.5678 1.5879
     1.608  1.6281 1.6482 1.6683 1.6884 1.7085 1.7286 1.7487 1.7688 1.7889
     1.809  1.8291 1.8492 1.8693 1.8894 1.9095 1.9296 1.9497 1.9698 1.9899
     2.01   2.0301 2.0502 2.0703 2.0904 2.1105 2.1306 2.1507 2.1708 2.1909
     2.211  2.2311 2.2512 2.2713 2.2914 2.3115 2.3316 2.3517 2.3718 2.3919
     2.412  2.4321 2.4522 2.4723 2.4924 2.5125 2.5326 2.5527 2.5728 2.5929
     2.613  2.6331 2.6532 2.6733 2.6934 2.7135 2.7336 2.7537 2.7738 2.7939
     2.814  2.8341]

```python
Raw=V[:,1]
print(Raw)
```

    [0.1517 0.1492 0.1524 0.1576 0.1635 0.1744 0.1937 0.2126 0.2288 0.2552
     0.2833 0.3125 0.3463 0.3879 0.4324 0.4825 0.5358 0.5928 0.6446 0.6966
     0.753  0.8116 0.8639 0.9251 0.9865 1.0494 1.1175 1.1893 1.261  1.3333
     1.4078 1.4778 1.5505 1.6178 1.6855 1.7481 1.8074 1.8658 1.9245 1.9792
     2.0339 2.0724 2.1126 2.1452 2.1701 2.1844 2.2012 2.2047 2.2021 2.194
     2.1824 2.1634 2.1433 2.1116 2.0795 2.0427 1.9985 1.9519 1.9    1.8442
     1.781  1.7189 1.6515 1.5813 1.5132 1.4401 1.371  1.2987 1.2265 1.1594
     1.0909 1.0255 0.963  0.9033 0.8488 0.7928 0.7359 0.6853 0.637  0.5913
     0.5533 0.5207 0.4856 0.4531 0.4276 0.4055 0.3856 0.3698 0.3665 0.3648
     0.3835 0.4091 0.4622 0.5367 0.6447 0.7663 0.9091 1.068  1.2216 1.3753
     1.5169 1.6317 1.7246 1.7819 1.8011 1.7825 1.7405 1.675  1.5954 1.509
     1.4121 1.3088 1.2    1.0853 0.9715 0.8626 0.7664 0.6798 0.6014 0.5392
     0.4865 0.4359 0.396  0.3587 0.3254 0.3015 0.28   0.2676 0.2499 0.2421
     0.2323 0.2267 0.2226 0.214  0.208  0.1993 0.1953 0.1811 0.1695 0.1565
     0.1438 0.1356]

```python
Noisy=V[:,2]
print(Noisy)
```

    [0.1458 0.1509 0.1575 0.1427 0.1714 0.1731 0.193  0.2097 0.2356 0.2637
     0.2816 0.3121 0.348  0.3904 0.4242 0.4815 0.5402 0.5954 0.637  0.6885
     0.7427 0.805  0.8624 0.9323 0.9806 1.0501 1.1193 1.1841 1.2583 1.3459
     1.4078 1.4811 1.5445 1.6084 1.6825 1.743  1.8056 1.8578 1.9353 1.98
     2.0374 2.0685 2.1042 2.1454 2.1682 2.1889 2.2148 2.2083 2.2075 2.1963
     2.1792 2.1676 2.1447 2.1102 2.0811 2.0507 2.0074 1.9517 1.8948 1.838
     1.7831 1.7264 1.6599 1.5845 1.5161 1.451  1.3744 1.298  1.2351 1.1523
     1.0863 1.0287 0.9668 0.9019 0.8474 0.7965 0.7364 0.6893 0.6387 0.5947
     0.5533 0.5216 0.4897 0.4536 0.425  0.4077 0.3842 0.3629 0.3695 0.3671
     0.3671 0.4053 0.4659 0.546  0.6375 0.7571 0.914  1.0704 1.2246 1.3675
     1.5081 1.6388 1.7404 1.7738 1.8075 1.7801 1.7457 1.6798 1.5943 1.517
     1.4107 1.318  1.2067 1.0783 0.9658 0.8627 0.7659 0.67   0.5805 0.5361
     0.4845 0.4381 0.4025 0.3486 0.3274 0.3067 0.2782 0.2703 0.242  0.2471
     0.2435 0.2299 0.2175 0.2157 0.2061 0.2006 0.1869 0.1873 0.1518 0.1583
     0.1389 0.127 ]

```python
Acell=V[:,3]
```

```python
deltat=t[1]-t[0]
print (deltat)
```

    0.0201

```python
from numpy import diff
velocity=diff(Raw)/deltat
print(velocity)
```

    [-0.12437811  0.15920398  0.25870647  0.29353234  0.54228856  0.960199
      0.94029851  0.80597015  1.31343284  1.39800995  1.45273632  1.68159204
      2.06965174  2.21393035  2.49253731  2.65174129  2.8358209   2.57711443
      2.58706468  2.80597015  2.91542289  2.60199005  3.04477612  3.05472637
      3.12935323  3.3880597   3.5721393   3.56716418  3.59701493  3.70646766
      3.48258706  3.61691542  3.34825871  3.3681592   3.11442786  2.95024876
      2.90547264  2.92039801  2.72139303  2.72139303  1.91542289  2.
      1.62189055  1.23880597  0.71144279  0.8358209   0.17412935 -0.12935323
     -0.40298507 -0.57711443 -0.94527363 -1.         -1.57711443 -1.59701493
     -1.83084577 -2.19900498 -2.31840796 -2.58208955 -2.7761194  -3.14427861
     -3.08955224 -3.35323383 -3.49253731 -3.3880597  -3.63681592 -3.43781095
     -3.59701493 -3.5920398  -3.33830846 -3.4079602  -3.25373134 -3.10945274
     -2.97014925 -2.71144279 -2.78606965 -2.83084577 -2.51741294 -2.40298507
     -2.27363184 -1.89054726 -1.62189055 -1.74626866 -1.61691542 -1.26865672
     -1.09950249 -0.99004975 -0.78606965 -0.1641791  -0.08457711  0.93034826
      1.27363184  2.64179104  3.70646766  5.37313433  6.04975124  7.10447761
      7.90547264  7.64179104  7.64676617  7.04477612  5.71144279  4.62189055
      2.85074627  0.95522388 -0.92537313 -2.08955224 -3.25870647 -3.960199
     -4.29850746 -4.82089552 -5.13930348 -5.41293532 -5.70646766 -5.66169154
     -5.41791045 -4.78606965 -4.30845771 -3.90049751 -3.09452736 -2.62189055
     -2.51741294 -1.98507463 -1.85572139 -1.65671642 -1.18905473 -1.06965174
     -0.61691542 -0.88059701 -0.3880597  -0.48756219 -0.27860697 -0.2039801
     -0.4278607  -0.29850746 -0.43283582 -0.19900498 -0.70646766 -0.57711443
     -0.64676617 -0.6318408  -0.4079602 ]

```python
dacell2=diff(velocity)/deltat
print(dacell2)
```

    [ 14.10856167   4.95037252   1.73263038  12.37593129  20.79156457
      -0.9900745   -6.6830029   25.24689983   4.20781664   2.72270488
      11.38585679  19.30645281   7.17804015  13.86104304   7.92059602
       9.15818915 -12.87096854   0.49503725  10.89081953   5.44540977
     -15.59367342  22.02915769   0.49503725   3.71277939  12.87096854
       9.15818915  -0.24751863   1.48511175   5.44540977 -11.13833816
       6.6830029  -13.36600579   0.9900745  -12.62344991  -8.16811465
      -2.22766763   0.74255588  -9.90074503   0.         -40.09801738
       4.20781664 -18.81141556 -19.05893418 -26.23697433   6.18796564
     -32.91997723 -15.09863617 -13.61352442  -8.6631519  -18.31637831
      -2.72270488 -28.71216059  -0.9900745  -11.63337541 -18.31637831
      -5.94044702 -13.11848717  -9.65322641 -18.31637831   2.72270488
     -13.11848717  -6.93052152   5.19789114 -12.37593129   9.90074503
      -7.92059602   0.24751863  12.62344991  -3.46526076   7.6730774
       7.17804015   6.93052152  12.87096854  -3.71277939  -2.22766763
      15.59367342   5.69292839   6.43548427  19.05893418  13.36600579
      -6.18796564   6.43548427  17.3263038    8.41563328   5.44540977
      10.14826366  30.93982822   3.96029801  50.49379966  17.07878518
      68.06762209  52.96898592  82.91873964  33.66253311  52.47394866
      39.85049875 -13.11848717   0.24751863 -29.94975372 -66.33499171
     -54.20657905 -88.11663078 -94.30459642 -93.56204054 -57.91935843
     -58.16687706 -34.90012623 -16.83126655 -25.98945571 -15.84119205
     -13.61352442 -14.60359892   2.22766763  12.12841266  31.43486547
      23.76178807  20.29652731  40.09801738  23.51426945   5.19789114
      26.48449296   6.43548427   9.90074503  23.26675082   5.94044702
      22.52419495 -13.11848717  24.50434395  -4.95037252  10.39578228
       3.71277939 -11.13833816   6.43548427  -6.6830029   11.63337541
     -25.24689983   6.43548427  -3.46526076   0.74255588  11.13833816]

```python
tamanhodacell2=np.size(dacell2)
```

```python
novo_tempo=t[0:tamanhodacell2]
novo_aceleracao_medida=Acell[0:tamanhodacell2]
```

```python
hfig,hax=plt.subplots(1,1,sharex = True, squeeze=True, figsize=(9,5))
plt.plot(t,Acell, label='Aceleração medida')
plt.plot(novo_tempo,dacell2,label='Aceleração calculada')
hax.legend(frameon=False)
hax.set_ylabel('Amplitude [m/$s^2$]')
hax.set_xlabel('Time[s]')

```

```python
velocidadeNoisy=diff(Noisy)/deltat
Aceleracaonoisy2=diff(velocidadeNoisy)/deltat
```

```python
hfig,hax=plt.subplots(1,1,sharex = True, squeeze=True, figsize=(9,5))
plt.plot(t,Acell, label='Aceleração medida')
plt.plot(novo_tempo,Aceleracaonoisy2,label='Aceleração calculada Noisy')
hax.legend(frameon=False)
hax.set_ylabel('Amplitude [m/$s^2$]')
hax.set_xlabel('Time[s]')

```

```python
hfig,hax=plt.subplots(1,1,sharex = True, squeeze=True, figsize=(9,5))
plt.plot(novo_tempo,dacell2,label='Aceleração calculada')
plt.plot(novo_tempo,Aceleracaonoisy2,label='Aceleração calculada Noisy')
hax.legend(frameon=False)
hax.set_ylabel('Amplitude [m/$s^2$]')
hax.set_xlabel('Time[s]')

```

```python

```


```python
# %load /Users/facai/Study/book_notes/preconfig.py
%matplotlib inline

import matplotlib.pyplot as plt
import seaborn as sns

from IPython.display import SVG
```

逻辑回归在spark中的实现简介
=======================

分析用的代码版本信息：

```bash
~/W/g/spark ❯❯❯ git log -n 1
commit d9ad78908f6189719cec69d34557f1a750d2e6af
Author: Wenchen Fan <wenchen@databricks.com>
Date:   Fri May 26 15:01:28 2017 +0800

    [SPARK-20868][CORE] UnsafeShuffleWriter should verify the position after FileChannel.transferTo
```

### 0. 总纲

下图是ml包中逻辑回归的构成情况：

```python
SVG("./res/spark_ml_lr.svg")
```

可以看到，逻辑回归是比较简单的，在它的`train`函数里，除开左侧的几个辅助类：

+ Instance: 封装数据
+ MetadataUtils: 数据信息
+ Instrumentation: 日志
+ Multi*Summarizer: 统计

主要就是做两件事：

+ 构造损失函数 => costFun: DiffFunction
+ 创建寻优算子 => optimizer: FirstOrderMinizer
  ml里两种算子都是拟牛顿法，理论上比SGD迭代更少，收敛更快。其中QWLQN是LBFGS的变种，可使用L1正则。

接下来，我们就将精力放在这两件事的实现上。这里寻优算子主要是根据正则确定的，而损失函数会由二分类和多分类而有所变化，下面一一叙迖述。

### 1. 寻优算子

jupyter的markdown，无法正确处理`$`取值语法，所以做了点小变动。

```scala
 645         val optimizer = if (elasticNetParam == 0.0 || regParam == 0.0) {
 646 // +--  4 lines: if (lowerBounds != null && upperBounds != null) {--------------------------------------
 650             new BreezeLBFGS[BDV[Double]](maxIter, 10, tol)
 651           }
 652         } else {
 653           val standardizationParam = standardization
 654           def regParamL1Fun = (index: Int) => {
 655 // +--  2 lines: Remove the L1 penalization on the intercept--------------------------------------------
 657             if (isIntercept) {
 658               0.0
 659             } else {
 660               if (standardizationParam) {
 661                 regParamL1
 662               } else {
 663                 val featureIndex = index / numCoefficientSets
 664 // +--  5 lines: If `standardization` is false, we still standardize the data---------------------------
 669                 if (featuresStd(featureIndex) != 0.0) {
 670                   regParamL1 / featuresStd(featureIndex)
 671                 } else {
 672                   0.0
 673                 }
 674               }
 675             }
 676           }
 677           new BreezeOWLQN[Int, BDV[Double]](maxIter, 10, regParamL1Fun, $(tol))
 678         }
```

可以看到，逻辑很简单：如果不用正则，或只用L2，就用LBFGS算子；如果用到L1正则，就用QWLQN算子。其中下半代码均是在折算合适的L1正则值。

因为QWLQN会自己处理L1正则，所以在接下来的损失函数计算中，我们只考虑L2正则，而不管L1。

### 2. 损失函数
#### 2.1 二分类

预测公式：$f(x) = \frac1{1 + e^{w^T x}}$

[损失函数](http://spark.apache.org/docs/latest/mllib-linear-methods.html#logistic-regression)定义是：

\begin{equation}
L(w;x,y) = \log(1+e^{-y w^T x}) + r_2 \cdot \frac{1}{2} w^T w + r_1 \cdot \|w\|
\end{equation}

[导数是](http://spark.apache.org/docs/latest/mllib-linear-methods.html#loss-functions)：

\begin{align}
  \frac{\partial L}{\partial w} &= -y \left(1-\frac1{1+e^{-y w^T x}} \right) \cdot x + r_2 w \pm r_1 \\
  &= \left ( \frac{y}{1+e^{-y w^T x}} - y \right ) \cdot x + r_2 w \pm r_1 \\
  \text{因为$y$只有1和-1两值，可简化为} \\
  &= \left ( \frac{1}{1+e^{-w^T x}} - y \right ) \cdot x + r_2 w \pm r_1 \\
  &= \left ( f(x) - y \right ) \cdot x + r_2 w \pm r_1
\end{align}

好，我们先看没有正则的计算，在LogisticAggregator类里：

```scala
1670   /** Update gradient and loss using binary loss function. */
1671   private def binaryUpdateInPlace(
1672       features: Vector,
1673       weight: Double,
1674       label: Double): Unit = {
1675 +--  4 lines: val localFeaturesStd = bcFeaturesStd.value----------
1679     val margin = - {
1680       var sum = 0.0
1681       features.foreachActive { (index, value) =>
1682         if (localFeaturesStd(index) != 0.0 && value != 0.0) {
1683           sum += localCoefficients(index) * value / localFeaturesStd(index)
1684         }
1685       }
1686       if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
1687       sum
1688     }
1689
1690     val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
1691
1692     features.foreachActive { (index, value) =>
1693       if (localFeaturesStd(index) != 0.0 && value != 0.0) {
1694         localGradientArray(index) += multiplier * value / localFeaturesStd(index)
1695       }
1696     }
1697
1698     if (fitIntercept) {
1699       localGradientArray(numFeaturesPlusIntercept - 1) += multiplier
1700     }
1701
1702     if (label > 0) {
1703       // The following is equivalent to log(1 + exp(margin)) but more numerically stable.
1704       lossSum += weight * MLUtils.log1pExp(margin)
1705     } else {
1706       lossSum += weight * (MLUtils.log1pExp(margin) - margin)
1707     }
1708   }
```

其中，
+ margin = $-w^T x$
  注意：这里用的$x / \operatorname{std}(x)$，相当于归一化，统一量纲。很奇怪，没有同时移动坐标，我不清楚是否合理。
+ multiplier = $\frac1{1 + e^{w^T x}} - y$ = $f(x) - y$
+ localGradientArray = $(f(x) - y) x$
+ lossSum = $\log(1+e^{-y w^T x})$。注意：因为margin计算时是$y=1$，所以1706L，对$y=-1$做了变换。数学技巧比较简单：

\begin{align}
  log(1 + e^x) - x &= log(1 + e^x) - log(e^x) \\
  &= log(\frac{1 + e^x}{e^x}) \\
  &= log(1 + e^{-x})
\end{align}

再在损失函数和偏导里，均加上L2的部份，代码在LogisticCostFun类的calculate方法里：

```scala
1877   override def calculate(coefficients: BDV[Double]): (Double, BDV[Double]) = {
1878 // +--  6 lines: val coeffs = Vectors.fromBreeze(coefficients)------
1884
1885     val logisticAggregator = {
1886 // +--  3 lines: val seqOp = (c: LogisticAggregator, instance: Instance) =>
1889       instances.treeAggregate(
1890         new LogisticAggregator(bcCoeffs, bcFeaturesStd, numClasses, fitIntercept,
1891           multinomial)
1892       )(seqOp, combOp, aggregationDepth)
1893     }
1894
1895     val totalGradientMatrix = logisticAggregator.gradient
1896     val coefMatrix = new DenseMatrix(numCoefficientSets, numFeaturesPlusIntercept, coeffs.toArray)
1897     // regVal is the sum of coefficients squares excluding intercept for L2 regularization.
1898     val regVal = if (regParamL2 == 0.0) {
1899       0.0
1900     } else {
1901       var sum = 0.0
1902       coefMatrix.foreachActive { case (classIndex, featureIndex, value) =>
1903         // We do not apply regularization to the intercepts
1904         val isIntercept = fitIntercept && (featureIndex == numFeatures)
1905         if (!isIntercept) {
1906 // +--  2 lines: The following code will compute the loss of the regularization; also---
1908           sum += {
1909             if (standardization) {
1910               val gradValue = totalGradientMatrix(classIndex, featureIndex)
1911               totalGradientMatrix.update(classIndex, featureIndex, gradValue + regParamL2 * value)
1912               value * value
1913 // +-- 14 lines: } else {------------------
1927             }
1928           }
1929         }
1930       }
1931       0.5 * regParamL2 * sum
1932     }
1933 // +--  2 lines: bcCoeffs.destroy(blocking = false)--------
1935     (logisticAggregator.loss + regVal, new BDV(totalGradientMatrix.toArray))
1936   }
1
```

其中，1912L和1931L是加L2正则$r_2 \cdot \frac{1}{2}w^T w$；1911L是加L2的偏导$r_2 \cdot w$。因为有额外的分支处理归一的情况，分支较多。同时，损失和偏导混在一起算，代码有点混杂。

如此，就有了二分类的损失函数和偏导数。

```scala
 601         val costFun = new LogisticCostFun(instances, numClasses, fitIntercept,
 602           standardization, bcFeaturesStd, regParamL2, multinomial = isMultinomial,
 603           aggregationDepth)
```

#### 2.2 多分类

spark在这里用的是softmax函数来代替logit函数，是比较有意思的解决方案。因为在LogisticAggregator类，已经详细地注释了推导关键过程，所以我就直接搬运过来，稍微作点附注，再把代码和公式对应起来就好。

LogisticAggregator computes the gradient and loss for binary or multinomial logistic (softmax)
loss function, as used in classification for instances in sparse or dense vector in an online
fashion.

Two LogisticAggregators can be merged together to have a summary of loss and gradient of
the corresponding joint dataset.

For improving the convergence rate during the optimization process and also to prevent against
features with very large variances exerting an overly large influence during model training,
packages like R's GLMNET perform the scaling to unit variance and remove the mean in order to
reduce the condition number. The model is then trained in this scaled space, but returns the
coefficients in the original scale. See page 9 in
http://cran.r-project.org/web/packages/glmnet/glmnet.pdf

However, we don't want to apply the [[org.apache.spark.ml.feature.StandardScaler]] on the
training dataset, and then cache the standardized dataset since it will create a lot of overhead.
As a result, we perform the scaling implicitly when we compute the objective function (though
we do not subtract the mean).

Note that there is a difference between multinomial (softmax) and binary loss. The binary case
uses one outcome class as a "pivot" and regresses the other class against the pivot. In the
multinomial case, the softmax loss function is used to model each class probability
independently. Using softmax loss produces `K` sets of coefficients, while using a pivot class
produces `K - 1` sets of coefficients (a single coefficient vector in the binary case). In the
binary case, we can say that the coefficients are shared between the positive and negative
classes. When regularization is applied, multinomial (softmax) loss will produce a result
different from binary loss since the positive and negative don't share the coefficients while the
binary regression shares the coefficients between positive and negative.

The following is a mathematical derivation for the multinomial (softmax) loss.

The probability of the multinomial outcome $y$ taking on any of the K possible outcomes is:

   $$
   P(y_i=0|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1}
      e^{\vec{x}_i^T \vec{\beta}_k}} \\
   P(y_i=1|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_1}}{\sum_{k=0}^{K-1}
      e^{\vec{x}_i^T \vec{\beta}_k}}\\
   P(y_i=K-1|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_{K-1}}\,}{\sum_{k=0}^{K-1}
      e^{\vec{x}_i^T \vec{\beta}_k}}
   $$

The model coefficients $\beta = (\beta_0, \beta_1, \beta_2, ..., \beta_{K-1})$ become a matrix
which has dimension of $K \times (N+1)$ if the intercepts are added. If the intercepts are not
added, the dimension will be $K \times N$.

Note that the coefficients in the model above lack identifiability. That is, any constant scalar
can be added to all of the coefficients and the probabilities remain the same.

   $$
   \begin{align}
   \frac{e^{\vec{x}_i^T \left(\vec{\beta}_0 + \vec{c}\right)}}{\sum_{k=0}^{K-1}
      e^{\vec{x}_i^T \left(\vec{\beta}_k + \vec{c}\right)}}
   = \frac{e^{\vec{x}_i^T \vec{\beta}_0}e^{\vec{x}_i^T \vec{c}}\,}{e^{\vec{x}_i^T \vec{c}}
      \sum_{k=0}^{K-1} e^{\vec{x}_i^T \vec{\beta}_k}}
   = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1} e^{\vec{x}_i^T \vec{\beta}_k}}
   \end{align}
   $$

However, when regularization is added to the loss function, the coefficients are indeed
identifiable because there is only one set of coefficients which minimizes the regularization
term. When no regularization is applied, we choose the coefficients with the minimum L2
penalty for consistency and reproducibility. For further discussion see:

Friedman, et al. "Regularization Paths for Generalized Linear Models via Coordinate Descent"

The loss of objective function for a single instance of data (we do not include the
regularization term here for simplicity) can be written as

   $$
   \begin{align}
   \ell\left(\beta, x_i\right) &= -log{P\left(y_i \middle| \vec{x}_i, \beta\right)} \\
   &= log\left(\sum_{k=0}^{K-1}e^{\vec{x}_i^T \vec{\beta}_k}\right) - \vec{x}_i^T \vec{\beta}_y\\
   &= log\left(\sum_{k=0}^{K-1} e^{margins_k}\right) - margins_y
   \end{align}
   $$

where ${margins}_k = \vec{x}_i^T \vec{\beta}_k$.

For optimization, we have to calculate the first derivative of the loss function, and a simple
calculation shows that

   $$
   \begin{align}
   \frac{\partial \ell(\beta, \vec{x}_i, w_i)}{\partial \beta_{j, k}}
   &= x_{i,j} \cdot w_i \cdot \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k}}{\sum_{k'=0}^{K-1}
     e^{\vec{x}_i \cdot \vec{\beta}_{k'}}\,} - I_{y=k}\right) \\
   &= x_{i, j} \cdot w_i \cdot multiplier_k
   \end{align}
   $$

where $w_i$ is the sample weight, $I_{y=k}$ is an indicator function

   $$
   I_{y=k} = \begin{cases}
         1 & y = k \\
         0 & else
      \end{cases}
   $$

and

   $$
   multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k}}{\sum_{k=0}^{K-1}
      e^{\vec{x}_i \cdot \vec{\beta}_k}} - I_{y=k}\right)
   $$

$\exp(709.78)$超出Double上限。

If any of margins is larger than 709.78, the numerical computation of multiplier and loss
function will suffer from arithmetic overflow. This issue occurs when there are outliers in
data which are far away from the hyperplane, and this will cause the failing of training once
infinity is introduced. Note that this is only a concern when max(margins) > 0.

Fortunately, when max(margins) = maxMargin > 0, the loss function and the multiplier can
easily be rewritten into the following equivalent numerically stable formula.

这里变换非常简单，将括号打开，用指数和对数规则依次套用。

   $$
   \ell\left(\beta, x\right) = log\left(\sum_{k=0}^{K-1} e^{margins_k - maxMargin}\right) -
      margins_{y} + maxMargin
   $$

Note that each term, $(margins_k - maxMargin)$ in the exponential is no greater than zero; as a
result, overflow will not happen with this formula.

For $multiplier$, a similar trick can be applied as the following,

   $$
   multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k - maxMargin}}{\sum_{k'=0}^{K-1}
      e^{\vec{x}_i \cdot \vec{\beta}_{k'} - maxMargin}} - I_{y=k}\right)
   $$

```scala
1711   private def multinomialUpdateInPlace(
1712 // +-- 12 lines: features: Vector,----------
1724     // marginOfLabel is margins(label) in the formula
1725     var marginOfLabel = 0.0
1726     var maxMargin = Double.NegativeInfinity
1727
1728     val margins = new Array[Double](numClasses)
1729     features.foreachActive { (index, value) =>
1730 // +--  2 lines: val stdValue = value / localFeaturesStd(index)-------
1732       while (j < numClasses) {
1733         margins(j) += localCoefficients(index * numClasses + j) * stdValue
1734 // +--  4 lines: j += 1-----------------
1738     while (i < numClasses) {
1739       if (fitIntercept) {
1740         margins(i) += localCoefficients(numClasses * numFeatures + i)
1741       }
1742       if (i == label.toInt) marginOfLabel = margins(i)
1743       if (margins(i) > maxMargin) {
1744         maxMargin = margins(i)
1745       }
1746       i += 1
1747     }
1748 // +--  6 lines: *---------------------
1754     val multipliers = new Array[Double](numClasses)
1755     val sum = {
1756       var temp = 0.0
1757       var i = 0
1758       while (i < numClasses) {
1759         if (maxMargin > 0) margins(i) -= maxMargin
1760         val exp = math.exp(margins(i))
1761         temp += exp
1762         multipliers(i) = exp
1763         i += 1
1764       }
1765       temp
1766     }
1767
1768     margins.indices.foreach { i =>
1769       multipliers(i) = multipliers(i) / sum - (if (label == i) 1.0 else 0.0)
1770     }
1771     features.foreachActive { (index, value) =>
1772       if (localFeaturesStd(index) != 0.0 && value != 0.0) {
1773         val stdValue = value / localFeaturesStd(index)
1774         var j = 0
1775         while (j < numClasses) {
1776           localGradientArray(index * numClasses + j) +=
1777             weight * multipliers(j) * stdValue
1778           j += 1
1779         }
1780       }
1781     }
1782     if (fitIntercept) {
1783       var i = 0
1784       while (i < numClasses) {
1785         localGradientArray(numFeatures * numClasses + i) += weight * multipliers(i)
1786         i += 1
1787       }
1788     }
1789
1790     val loss = if (maxMargin > 0) {
1791       math.log(sum) - marginOfLabel + maxMargin
1792     } else {
1793       math.log(sum) - marginOfLabel
1794     }
1795     lossSum += weight * loss
1796   }
```

+ 1728L-1733L，在计算margins = $x \beta$。1738L的循环是找出maxMargin和标签对应的marginOfLabel，因为后面公式要用到。
+ 1754L-1770L，计算了multipliers。我个人很不喜欢这种一个循环做两件事，且出口不同的风格。
+ 1771L-1788L，计算导数localGradientArray = $x_{i, j} \cdot w_i \cdot \operatorname{multiplier}_k$。
+ 1790L-1795L，根据最大margin是否大于0，计算损失值loss。注意1759L也有针对做修正。

公式较复杂，但代码挺简单的。为了效率，有的地方写得不太好看。

### 3. 小结

spark-ml里逻辑回归支持样本加权，二分类和多分类。寻优算子是相对优秀的拟牛顿算法，多分类是softmax。总体而言，功能完整够用，实现也比较优秀。但有的代码，个人认为像面条，冗余，不够清减。

```python

```


# Kolmogorov forcing

The following equation is solved
\begin{equation}
\partial_t \zeta + J(\psi,\zeta) = A(y) + \nu \nabla^2 \zeta
\end{equation}
where $A(y) = A_1 cos(y) +4 A_4cos(4y)$.

Parameters for the following runs are:
> $\nu = 0.02$

> $A_1 = -1.0$

> $A_4 = -2.0$

```python
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.pylab as pl
import matplotlib.animation as animation
```

```python
def save_vorticity_snapshots(data):

    filenames = []
    S = np.shape(data['Vxy'])[2]

    for i in range (0,S,100):

        filename = dn+'anim/frame_'+str(i)+'.png'
        filenames.append (filename)

        days = round(i/S*10000,2)

        fig,ax = plt.subplots()

        ax.set_title(r'$t = $'+str(days))
        ax.set_xticks([0,M-1,2*M-2])
        ax.set_xticklabels([r'$0$',r'$\pi$',r'$2\pi$'],fontsize=14)

        ax.set_yticks([0,N-1,2*N-2])
        ax.set_yticklabels([r'$0$',r'$\pi$',r'$2\pi$'],fontsize=14)

        im = plt.imshow((data['Vxy'][:,:,i]),cmap="RdBu_r",origin="lower",interpolation="bicubic",vmax=30,vmin=-30)

        fig.colorbar(im)

        plt.savefig(filename,bbox_inches='tight',dpi=128)
        plt.close()

    return filenames

def gen_vorticity_animation(infiles,outfile):

    gif_filename = dn+outfile+'.gif'

    with imageio.get_writer(gif_filename, mode='I') as writer:
        for filename in infiles:  ## here we go through all prepared files
            image = imageio.imread(filename)
            writer.append_data(image)

            os.remove(filename) ## now we remove individual frames

    optimize(gif_filename) ## to optimise the file size
```

```python
def animate_data(data,dn,filename):

    fps = 20
    saveinterval = 5

    fig,ax = plt.subplots()

    im = plt.imshow(data[:,:,0],origin='lower',cmap='RdBu_r',vmax=30,vmin=-30,interpolation='bicubic')
    fig.colorbar(im)

    plt.xticks([0,M-1,2*M-2],[r'$0$',r'$\pi$',r'$2\pi$'],fontsize=14)
    plt.yticks([0,M-1,2*M-2],[r'$0$',r'$\pi$',r'$2\pi$'],fontsize=14)

    def animate(i):

        plt.title(str(i*fps*saveinterval) + ' days')

        im.set_array(data[:,:,i])
        return [im]

    anim = animation.FuncAnimation(fig, animate)
    anim.save(dn+filename+'.mp4', fps=fps, extra_args=['-vcodec', 'libx264'])

    plt.close()
```

# 8x8 grid

```python
dn = "kolmogorov/8x8/"

M,N = 8,8
colors = pl.cm.nipy_spectral(np.linspace(0,1,M))
```

```python
import matplotlib as mpl

plt.rc('font', family='serif') ## setting for figures which I usually use in my publications
mpl.rcParams.update({'font.size': 12})
mpl.rcParams.update({'legend.labelspacing':0.25, 'legend.fontsize': 12})
mpl.rcParams.update({'errorbar.capsize': 4})
```

```python
nl_full = np.load(dn+"nl.npz",allow_pickle=True)
nl_res1 = np.load(dn+"nl_res_1.npz",allow_pickle=True)
nl_res2 = np.load(dn+"nl_res_2.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,3,figsize=(14,5))

ax[0].set_title(f'NL(Full)')
for i,x in enumerate(nl_full['Emtav'].T):
    ax[0].plot(nl_full['t'],x,label=i,c=colors[i])

ax[1].set_title(f'NL(2)')
for i,x in enumerate(nl_res2['Emtav'].T):
    ax[1].plot(nl_res2['t'],x,label=i,c=colors[i])

ax[2].set_title(f'NL(1)')
for i,x in enumerate(nl_res1['Emtav'].T):
    ax[2].plot(nl_res1['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-6,1e4)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[2].legend(bbox_to_anchor=(1.3,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'nl_fullvlowres.png',bbox_inches='tight',dpi=128)
```

```python
fig,ax = plt.subplots(1,3,figsize=(15,6))

im = ax[0].imshow((nl_full['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic",vmax=30,vmin=-30)
im = ax[1].imshow((nl_res1['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic",vmax=30,vmin=-30)
im = ax[2].imshow((nl_res2['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic",vmax=30,vmin=-30)

fig.colorbar(im)

for a in ax:
    a.set_xticks([0,M-1,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M-1,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
animate_data(nl_full['Vxy'],dn,'nl_full')
animate_data(nl_res1['Vxy'],dn,'nl_res1')
animate_data(nl_res2['Vxy'],dn,'nl_res2')
```

```python
nl = np.load(dn+"nl.npz",allow_pickle=True)
gql_m = np.load(dn+"gql_m.npz",allow_pickle=True)
gce2_m = np.load(dn+"gce2_m.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,3,figsize=(14,5))

ax[0].set_title(f'NL')
for i,x in enumerate(nl['Emtav'].T):
    ax[0].plot(nl['t'],x,label=i,c=colors[i])

ax[1].set_title(f'GQL(M)')
for i,x in enumerate(gql_m['Emtav'].T):
    ax[1].plot(gql_m['t'],x,label=i,c=colors[i])

ax[2].set_title(f'GCE2(M)')
for i,x in enumerate(gce2_m['Emtav'].T):
    ax[2].plot(gce2_m['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-3,1e4)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[2].legend(bbox_to_anchor=(1.25,0.5),ncol=1)

plt.show()
```

```python
fig,ax = plt.subplots(1,3,figsize=(15,6))

im = ax[0].imshow((nl['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
im = ax[1].imshow((gql_m['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
im = ax[2].imshow((gce2_m['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")

fig.colorbar(im)
for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
fig,ax = plt.subplots(1,3,figsize=(15,6))

im = ax[0].imshow((nl['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")
im = ax[1].imshow((gql_m['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")
im = ax[2].imshow((gce2_m['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
# fig,ax = plt.subplots(3,1,figsize=(15,12))

# ax[0].set_title('NL')
# ax[0].imshow(nl['Vyt'],interpolation='bicubic')
# ax[0].set_aspect(3)

# ax[1].set_title('GQL(M)')
# ax[1].imshow(gql_m['Vyt'],interpolation='bicubic')
# ax[1].set_aspect(3)

# ax[2].set_title('GCE2(M)')
# ax[2].imshow(gce2_m['Vyt'],interpolation='bicubic')
# ax[2].set_aspect(3)

# for a in ax:
#     a.set_xlabel(r'$t$',fontsize=14)
#     a.set_ylabel(r'$\theta$',fontsize=14)

#     a.set_yticks([0,M,2*M-2])
#     a.set_yticklabels([r'$-90$',0,r'$90$'],fontsize=14)

# plt.show()
```

```python
gql_1 = np.load(dn+"gql_1.npz",allow_pickle=True)
gce2_1 = np.load(dn+"gce2_1.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(14,5))

ax[0].set_title(f'GQL(1)')
for i,x in enumerate(gql_1['Emtav'].T):
    ax[0].plot(gql_1['t'],x,label=i,c=colors[i])

ax[1].set_title(f'GCE2(1)')
for i,x in enumerate(gce2_1['Emtav'].T):
    ax[1].plot(gce2_1['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-4,1e4)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.01,0.5),ncol=1)

plt.show()
```

```python
fig,ax = plt.subplots(1,2,figsize=(12,6))

im = ax[0].imshow((gql_1['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
# fig.colorbar(im)
im = ax[1].imshow((gce2_1['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
# fig.colorbar(im)

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
fig,ax = plt.subplots(2,1,figsize=(12,6))

ax[0].set_title('GQL(1)')
ax[0].imshow(gql_1['Vyt'],cmap="jet",interpolation='bicubic')
ax[0].set_aspect(0.5)

ax[1].set_title('GCE2(1)')
ax[1].imshow(gce2_1['Vyt'],cmap="jet",interpolation='bicubic')
ax[1].set_aspect(0.25)

for a in ax:
    a.set_xlabel(r'$t$',fontsize=14)
    a.set_ylabel(r'$\theta$',fontsize=14)

    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-90$',0,r'$90$'],fontsize=14)

plt.show()
```

```python
gql_3 = np.load(dn+"gql_3.npz",allow_pickle=True)
gce2_3 = np.load(dn+"gce2_3.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(14,5))

ax[0].set_title(f'GQL(3)')
for i,x in enumerate(gql_3['Emtav'].T):
    ax[0].plot(gql_3['t'],x,label=i,c=colors[i])

ax[1].set_title(f'GCE2(3)')
for i,x in enumerate(gce2_3['Emtav'].T):
    ax[1].plot(gce2_3['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-4,1e4)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.01,0.5),ncol=1)

plt.show()
```

```python
fig,ax = plt.subplots(1,2,figsize=(12,6))

ax[0].set_title('GQL(3)')
im = ax[0].imshow((gql_3['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
# fig.colorbar(im)

ax[1].set_title('GCE2(3)')
im = ax[1].imshow((gce2_3['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
# fig.colorbar(im)

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
fig,ax = plt.subplots(2,1,figsize=(12,12))

ax[0].set_title('GQL(3)')
ax[0].imshow(gql_3['Vyt'],cmap="jet",interpolation='bicubic')
ax[0].set_aspect(0.5)

ax[1].set_title('GCE2(3)')
ax[1].imshow(gce2_3['Vyt'],cmap="jet",interpolation='bicubic')
ax[1].set_aspect(0.25)

for a in ax:
    a.set_xlabel(r'$t$',fontsize=14)
    a.set_ylabel(r'$\theta$',fontsize=14)

    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-90$',0,r'$90$'],fontsize=14)

plt.show()
```

```python
ql = np.load(dn+"ql.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2.npz",allow_pickle=True)
gce2_0 = np.load(dn+"gce2_0.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,3,figsize=(14,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emt'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

ax[2].set_title(f'GCE2(0)')
for i,x in enumerate(gce2_0['Emt'].T):
    ax[2].plot(gce2_0['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-36,1e4)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[2].legend(bbox_to_anchor=(1.01,0.5),ncol=1)

# plt.show()
plt.savefig(dn+'figures/kolmogorov_qlce2_dp5.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_bs3.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_ts5.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emt'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

# ax[2].set_title(f'GCE2(0)')
# for i,x in enumerate(gce2_0['Emt'].T):
#     ax[2].plot(gce2_0['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'figures/kolmogorov_qlce2_bs3.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_ts5.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_ts5.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emt'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

# ax[2].set_title(f'GCE2(0)')
# for i,x in enumerate(gce2_0['Emt'].T):
#     ax[2].plot(gce2_0['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'figures/kolmogorov_qlce2_ts5.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_dp5.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_dp5.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emt'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

# ax[2].set_title(f'GCE2(0)')
# for i,x in enumerate(gce2_0['Emt'].T):
#     ax[2].plot(gce2_0['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'figures/kolmogorov_qlce2_dp5.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_250.npz",allow_pickle=True)
ql_2 = np.load(dn+"ql_250_r2.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emtav'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'QL w/ diff seed')
for i,x in enumerate(ql_2['Emtav'].T):
    ax[1].plot(ql_2['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
# plt.savefig(dn+'figures/kolmogorov_qlce2_dp5.png',bbox_inches='tight')
```

```python
fig,ax = plt.subplots(1,2,figsize=(15,6))

im = ax[0].imshow((ql['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")
im = ax[1].imshow((ql_2['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
ql = np.load(dn+"ql_500_r123.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_500_ql_r123.npz",allow_pickle=True)
```

```python
colors = pl.cm.nipy_spectral(np.linspace(0,1,M))
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emtav'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emtav'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'figures/ze_qlce2_dp5.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_500_r123.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_500_r123.npz",allow_pickle=True)
```

```python
colors = pl.cm.nipy_spectral(np.linspace(0,1,M))
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emtav'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emtav'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

# plt.show()
plt.savefig(dn+'figures/ze_qlce2_rand.png',bbox_inches='tight')
```

```python
fig,ax = plt.subplots(1,3,figsize=(15,6))

im = ax[0].imshow((ql['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
im = ax[1].imshow((ce2['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")
im = ax[2].imshow((gce2_0['Emn'][:,:,-1]),cmap="nipy_spectral_r",origin="lower",interpolation="bicubic")

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
fig,ax = plt.subplots(1,3,figsize=(15,6))

im = ax[0].imshow((ql['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")
im = ax[1].imshow((ce2['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")
im = ax[2].imshow((gce2_0['Vxy'][:,:,-1]),cmap="RdBu_r",origin="lower",interpolation="bicubic")

for a in ax:
    a.set_xticks([0,M,2*M-2])
    a.set_xticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)
    a.set_yticks([0,M,2*M-2])
    a.set_yticklabels([r'$-M$',r'$0$',r'$M$'],fontsize=14)

plt.show()
```

```python
ql = np.load(dn+"ql_500_r123_nu01.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_500_r123_nu01.npz",allow_pickle=True)
```

```python
colors = pl.cm.nipy_spectral(np.linspace(0,1,M))
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

colors = pl.cm.nipy_spectral(np.linspace(0,1,M))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emtav'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

plt.show()
# plt.savefig(dn+'figures/ze_qlce2_rand.png',bbox_inches='tight')
```

```python
fig,ax = plt.subplots(figsize=(9,3))

Nt = len(ce2['t'])
t = np.linspace(0,500,Nt)
R = moderanks(ce2['mEVs'])

ax.set_title(f'Ranks for modes in Second Cumulant')
for i in np.arange(M-1):
    ax.plot(t,R[:,i],label=i+1,c=colors[i+1])

ax.set_xlabel(r'$t$',fontsize=14)
# ax.set_yscale('log')
# ax.set_ylim(1e-21,1e3)
ax.set_xlim(0,500)

ax.set_ylabel(r'$Eigenvalues$',fontsize=14)
ax.legend(loc=4,ncol=1)

# plt.show()
# plt.savefig(dn+'figures/ranks_ce2_rand.png',bbox_inches='tight')
```

```python
ql = np.load(dn+"ql_500_r123_nu03.npz",allow_pickle=True)
ce2 = np.load(dn+"ce2_500_r123_nu03.npz",allow_pickle=True)
```

```python
fig,ax = plt.subplots(1,2,figsize=(10,5))

ax[0].set_title(f'QL')
for i,x in enumerate(ql['Emtav'].T):
    ax[0].plot(ql['t'],x,label=i,c=colors[i])

ax[1].set_title(f'CE2')
for i,x in enumerate(ce2['Emt'].T):
    ax[1].plot(ce2['t'],x,label=i,c=colors[i])

for a in ax:

    a.set_xlabel(r'$t$',fontsize=14)
    a.set_yscale('log')
    a.set_ylim(1e-16,1e5)

ax[0].set_ylabel(r'$E(m)$',fontsize=14)
ax[1].legend(bbox_to_anchor=(1.0,0.75),ncol=1)

plt.show()
# plt.savefig(dn+'figures/ze_qlce2_rand.png',bbox_inches='tight')
```

```python

```


```python
import numpy as np
from scipy import linalg
```

# 14.1

## Задані виміри економіки країни

```python
X = np.array([
    [1320, 1170],
    [1060, 965]
])
```

```python
y = np.array([
    [1075],
    [1185]
])
```

```python
s = np.array([0.45, 0.2])
```

# Модель міжгалузевої залежності цін

Нехай $X$ - це матриця в якій $x_{i j}$ елемент позначає витрати продукції $i$-ї галузі на потреби $j$-ї.

y - вектор $y_i$ елемент якого це обсяг $i$-ї продукції, що витрачається у невиробничій сфері.

Тоді загальний випуск $i$-ї галузі (позначимо $x_i$), при загальній кількості галузей $n$, можна записати як:
\begin{equation}
x_i = \sum_{j=1}^n x_{i j} + y_i
\label{eq:sum_prod}
\tag{1}
\end{equation}

У моделі Леонтьєва вводиться припущення про пропорційну залежність між витратами та об'ємом виробництва, тобто вводяться лінійно-однорідні функції виробничих витрат:
\begin{equation}
x_{i j}=a_{i j} x_j => a_{i j} = \frac{x_{i j}}{ x_j}
\label{eq:leon_assum}
\tag{2}
\end{equation}

Модель міжгалузевої залежності цін описується рівнянням:
\begin{equation}
p = A^T p + s => (E - A ^ T) p = s
\label{eq:pr_dep}
\tag{3}
\end{equation}
Тут $p$ - вектор цін на товари, $s$ - вектор доданих вартостей в ціні

```python
x = (np.sum(X, axis=1).reshape(-1, 1) + y)
print(x)
```

    [[3565]
     [3210]]

```python
A = X / x.T
print(A)
```

    [[ 0.37026648  0.36448598]
     [ 0.2973352   0.30062305]]

```python
M = np.eye(A.shape[0]) - A.T
```

```python
p = np.linalg.solve(M, s)
print("Ціни на промислову та сільськогосподарську продукцію: {}".format(p))
```

    Ціни на промислову та сільськогосподарську продукцію: [ 1.12691012  0.87326719]

# 14.2

```python
A = np.array([
    [0.7, 0.05, 0.05],
    [0.05, 0.75, 0.05],
    [0.6, 0.15, 0.05]
])
y = np.array([
    [1900],
    [1500],
    [1100]
])
```

## Аналіз матриці прямих виробничих витрат $A$

```python
w = linalg.eig(A, left=True)
```

```python
eigvalues = w[0]
pol_coef = np.poly(w[0])
max_eig_ind = np.argmax(w[0])
fro_number = w[0][max_eig_ind]
fro_l = w[1][:, max_eig_ind] * (-1 if w[1][0, max_eig_ind] < 0 else 1)
fro_r = w[2][:, max_eig_ind] * (-1 if w[2][0, max_eig_ind] < 0 else 1)
```

```python
print("Характеристичний поліном: {}".format(pol_coef))
print("Власні числа: {}".format(", ".join(map(lambda x: str(x.real), eigvalues))))
print("Число Фробеніуса: {}".format(fro_number.real))
print("Лівий вектор Фробеніуса: ({})".format(", ".join(map(lambda x: str(x.real), fro_l))))
print("Правий вектор Фробеніуса: ({})".format(", ".join(map(lambda x: str(x.real), fro_r))))
```

    Характеристичний поліном: [  1.00000000e+00  -1.50000000e+00   5.57500000e-01  -2.50000000e-04]
    Власні числа: 0.000448972689168, 0.676723625695, 0.822827401615
    Число Фробеніуса: 0.8228274016154079
    Лівий вектор Фробеніуса: (0.723483833587, 0.684306570005, 0.0910805181499)
    Правий вектор Фробеніуса: (0.494997713423, 0.69649905063, 0.519486608276)

## Продуктивність

Теорема
Для продуктивності моделі Леонтьєва необхідно й достататньо, щоб фробеніусове число $\lambda_A$ матриці $A$ задовольняло нерівності $\lambda_A < 1$

```python
print("Матриця A - {}продуктивна".format("" if fro_number.real < 1 else "не "))
```

    Матриця A - продуктивна

## Матриця повних витрат

Матриця повних витрат $B$ обчислюється як сума ряду:
\begin{align}
B &= E + A + A^2 + ... + A^N \\
N &\rightarrow \inf
\end{align}
Для того, щоб дослідити ряд на збіжність та обчислити матрицю B побудуємо послідовність $B_i$, яка прямує до B:
\begin{align}
&B_0 = E \\
&B_{i + 1} = B_i A + E
\end{align}
Тоді критерій збіжності набуває вигляду:
$$max( \bigl| B_i - B_{i + 1} \bigr| ) < \epsilon$$

```python
B = np.eye(A.shape[0])
B_pr = B.copy()
eps = 0.01
for i in range(100):
    B_pr, B = B, B_pr
    B = np.dot(B_pr, A) + np.eye(A.shape[0])
    if np.max(np.fabs(B - B_pr)) < eps:
        break
```

```python
print("Матриця повних витрат")
print(B)
```

    Матриця повних витрат
    [[ 3.98575637  0.93122482  0.25804895]
     [ 1.30988624  4.41209929  0.30012244]
     [ 2.71792601  1.27902876  1.26222235]]

## Вектор кінцевого випуску

З моделі Леонтьєва:
$$x = A x + y => (E-A) x = y$$
Розвязавши систему лінійних рівнянь отримуємо вектор кінцевого випуску $x$

```python
x = np.linalg.solve(np.eye(A.shape[0]) - A, y)
```

```python
x
```

    array([[ 9362.44541485],
           [ 9589.51965066],
           [ 8585.15283843]])

```python

```


```python
from __future__ import division, print_function
%matplotlib inline
```

```python
import sympy
from  sympy import Matrix, eye, symbols, sin, cos, zeros, sqrt, diag
from sympy.physics.mechanics import *
from IPython.display import display
sympy.init_printing(use_latex='mathjax')

import numpy as np

```

# Quaternion Math

```python
def expq(n):
    n *= 0.5
    nNorm = n.norm()
    qn = Matrix([cos(nNorm),n/nNorm*sin(nNorm)])
    return qn

def quat2dcm(q):
    """
    Convert quaternion to DCM
    """

    # Extract components
    w = q[0]
    x = q[1]
    y = q[2]
    z = q[3]

    # Reduce repeated calculations
    ww = w*w
    xx = x*x
    yy = y*y
    zz = z*z
    wx = w*x
    wy = w*y
    wz = w*z
    xy = x*y
    xz = x*z
    yz = y*z

    # Build Direction Cosine Matrix (DCM)
    dcm = Matrix([
        [ww + xx - yy - zz,       2*(xy - wz),       2*(xz + wy)],
        [      2*(xy + wz), ww - xx + yy - zz,       2*(yz - wx)],
        [      2*(xz - wy),       2*(yz + wx), ww - xx - yy + zz]
    ])
    return dcm

def dcm2quat(dcm):
    """
    Determine quaternion corresponding to dcm using
    the stanley method.

    Flips sign to always return shortest path quaterion
    so w >= 0

    Converts the 3x3 DCM into the quaterion where the
    first component is the real part
    """

    tr = Matrix.trace(dcm)

    w = 0.25*(1+tr)
    x = 0.25*(1+2*dcm[0,0]-tr)
    y = 0.25*(1+2*dcm[1,1]-tr)
    z = 0.25*(1+2*dcm[2,2]-tr)

    #kMax = np.argmax([w,x,y,z])

    kMax = 0

    if kMax == 0:
        w = sqrt(w)
        x = 0.25*(dcm[1,2]-dcm[2,1])/w
        y = 0.25*(dcm[2,0]-dcm[0,2])/w
        z = 0.25*(dcm[0,1]-dcm[1,0])/w

    elif kMax == 1:
        x = sqrt(x)
        w = 0.25*(dcm[1,2]-dcm[2,1])/x
        if w<0:
            x = -x
            w = -w
        y = 0.25*(dcm[0,1]+dcm[1,0])/x
        z = 0.25*(dcm[2,0]+dcm[0,2])/x

    elif kMax == 2:
        y = sqrt(y)
        w = 0.25*(dcm[2,0]-dcm[0,2])/y
        if w<0:
            y = -y
            w = -w
        x = 0.25*(dcm[0,1]+dcm[1,0])/y
        z = 0.25*(dcm[1,2]+dcm[2,1])/y

    elif kMax == 3:
        z = sqrt(z)
        w = 0.25*(dcm[0,1]-dcm[1,0])/z
        if w<0:
            z = -z
            w = -w
        x = 0.25*(dcm[2,0]+dcm[0,2])/z
        y = 0.25*(dcm[1,2]+dcm[2,1])/z

    q = Matrix([w,x,y,z])

    return q

def skew3(v):
    vx,vy,vz = v
    out = Matrix([[  0, -vz,   vy],
                  [ vz,   0,  -vx],
                  [-vy,  vx,    0]])
    return out

def skew4Left(v):
    if len(v)==3:
        v = Matrix.vstack(zeros(1),v)
    w,x,y,z = v
    out = Matrix([
            [w, -x, -y, -z],
            [x,  w, -z,  y],
            [y,  z,  w, -x],
            [z, -y,  x,  w],
        ])
    return out

def skew4Right(v):
    if len(v)==3:
        v = Matrix.vstack(zeros(1),v)
    w,x,y,z = v
    out = Matrix([
            [w, -x, -y, -z],
            [x,  w,  z, -y],
            [y, -z,  w,  x],
            [z,  y, -x,  w],
        ])
    return out

def quatConj(q):
    q_out = Matrix(q[:])
    q_out = q_out.T*sympy.diag(1,-1,-1,-1)
    q_out = q_out.T

    return q_out

def qRot(q,v):
    qPrime = quatConj(q)
    v = Matrix.vstack(zeros(1),v)
    vout = skew4Left(q)*skew4Right(qPrime)*v
    return Matrix(vout[1:])

def dRotdq(q,v):
    qw,qx,qy,qz = q
    vx,vy,vz = v

    dRdq = Matrix([
        [2*qw*vx + 2*qy*vz - 2*qz*vy,  2*qx*vx + 2*qy*vy + 2*qz*vz,  2*qw*vz + 2*qx*vy - 2*qy*vx, -2*qw*vy + 2*qx*vz - 2*qz*vx],
        [2*qw*vy - 2*qx*vz + 2*qz*vx, -2*qw*vz - 2*qx*vy + 2*qy*vx,  2*qx*vx + 2*qy*vy + 2*qz*vz,  2*qw*vx + 2*qy*vz - 2*qz*vy],
        [2*qw*vz + 2*qx*vy - 2*qy*vx,  2*qw*vy - 2*qx*vz + 2*qz*vx, -2*qw*vx - 2*qy*vz + 2*qz*vy,  2*qx*vx + 2*qy*vy + 2*qz*vz]
    ])

    return dRdq
```

# Setup

```python
# Time step
t = symbols('T')

# States
qw,qx,qy,qz = symbols('q_w q_x q_y q_z')
rx,ry,rz = symbols('r_x r_y r_z')
vx,vy,vz = symbols('v_x v_y v_z')
abx,aby,abz = symbols('a_bx a_by a_bz')
gbx,gby,gbz = symbols('g_bx g_by g_bz')

r_L = Matrix([rx,ry,rz])
v_L = Matrix([vx,vy,vz])
q_toLfromB = Matrix([qw,qx,qy,qz])
gyroBias = Matrix([gbx,gby,gbz])
accelBias = Matrix([abx,aby,abz])

# Measurements
ax,ay,az = symbols('a_x a_y a_z') # measured
mx,my,mz = symbols('m_x m_y m_z') # measured
wx,wy,wz = symbols('w_x w_y w_z') # measured

accelMeas = Matrix([ax,ay,az])
magMeas = Matrix([mx,my,mz])
wMeas = Matrix([wx,wy,wz])

# Nav vectors
g = symbols('g')
bx,by,bz = symbols('b_x b_y b_z')

g_L = Matrix([0,0,g])
B_L = Matrix([mx,0,0])

# Uncertainties
wax,way,waz = symbols('w_ax w_ay w_az')
wa = symbols('w_a')
ww = symbols('w_w')
waB = symbols('w_aB')
wgB = symbols('w_gB')
```

# Validate quaternion math

```python
print('dcm')
display( quat2dcm(q_toLfromB) )
print('[q]L')
display( skew4Left(q_toLfromB) )
print('[q]R')
display( skew4Right(q_toLfromB) )
print('[wb]R')
display( skew4Right(wMeas) )
r_B = Matrix([rx,ry,rz])
print('d/dq R*r_B')
display(dRotdq(q_toLfromB, r_B))
```

    dcm

$\displaystyle \left[\begin{matrix}q_{w}^{2} + q_{x}^{2} - q_{y}^{2} - q_{z}^{2} & - 2 q_{w} q_{z} + 2 q_{x} q_{y} & 2 q_{w} q_{y} + 2 q_{x} q_{z}\\2 q_{w} q_{z} + 2 q_{x} q_{y} & q_{w}^{2} - q_{x}^{2} + q_{y}^{2} - q_{z}^{2} & - 2 q_{w} q_{x} + 2 q_{y} q_{z}\\- 2 q_{w} q_{y} + 2 q_{x} q_{z} & 2 q_{w} q_{x} + 2 q_{y} q_{z} & q_{w}^{2} - q_{x}^{2} - q_{y}^{2} + q_{z}^{2}\end{matrix}\right]$

    [q]L

$\displaystyle \left[\begin{matrix}q_{w} & - q_{x} & - q_{y} & - q_{z}\\q_{x} & q_{w} & - q_{z} & q_{y}\\q_{y} & q_{z} & q_{w} & - q_{x}\\q_{z} & - q_{y} & q_{x} & q_{w}\end{matrix}\right]$

    [q]R

$\displaystyle \left[\begin{matrix}q_{w} & - q_{x} & - q_{y} & - q_{z}\\q_{x} & q_{w} & q_{z} & - q_{y}\\q_{y} & - q_{z} & q_{w} & q_{x}\\q_{z} & q_{y} & - q_{x} & q_{w}\end{matrix}\right]$

    [wb]R

$\displaystyle \left[\begin{matrix}0 & - w_{x} & - w_{y} & - w_{z}\\w_{x} & 0 & w_{z} & - w_{y}\\w_{y} & - w_{z} & 0 & w_{x}\\w_{z} & w_{y} & - w_{x} & 0\end{matrix}\right]$

    d/dq R*r_B

$\displaystyle \left[\begin{matrix}2 q_{w} r_{x} + 2 q_{y} r_{z} - 2 q_{z} r_{y} & 2 q_{x} r_{x} + 2 q_{y} r_{y} + 2 q_{z} r_{z} & 2 q_{w} r_{z} + 2 q_{x} r_{y} - 2 q_{y} r_{x} & - 2 q_{w} r_{y} + 2 q_{x} r_{z} - 2 q_{z} r_{x}\\2 q_{w} r_{y} - 2 q_{x} r_{z} + 2 q_{z} r_{x} & - 2 q_{w} r_{z} - 2 q_{x} r_{y} + 2 q_{y} r_{x} & 2 q_{x} r_{x} + 2 q_{y} r_{y} + 2 q_{z} r_{z} & 2 q_{w} r_{x} + 2 q_{y} r_{z} - 2 q_{z} r_{y}\\2 q_{w} r_{z} + 2 q_{x} r_{y} - 2 q_{y} r_{x} & 2 q_{w} r_{y} - 2 q_{x} r_{z} + 2 q_{z} r_{x} & - 2 q_{w} r_{x} - 2 q_{y} r_{z} + 2 q_{z} r_{y} & 2 q_{x} r_{x} + 2 q_{y} r_{y} + 2 q_{z} r_{z}\end{matrix}\right]$

# Extended Kalman Filter Equations

## State Equations

For the Linear Kalman filter we have the following for the process and measurment models:

$$\begin{aligned}\dot{\mathbf x} &= \mathbf{Ax} + w_x\\
\mathbf z &= \mathbf{Hx} + w_z
\end{aligned}$$

Where $\mathbf A$ is the systems dynamic matrix. Using the state space methodsthese equations can be transformed into:
$$\begin{aligned}\bar{\mathbf x} &= \mathbf{Fx} \\
\mathbf z &= \mathbf{Hx}
\end{aligned}$$

For the nonlinear model the linear expression $\mathbf{Fx} + \mathbf{Bu}$ is replaced by a nonlinear function $f(\mathbf x, \mathbf u)$, and the linear expression $\mathbf{Hx}$ is replaced by a nonlinear function $h(\mathbf x)$:

$$\begin{aligned}\dot{\mathbf x} &= f(\mathbf x, \mathbf u) + w_x\\
\mathbf z &= h(\mathbf x) + w_z
\end{aligned}$$

The EKF doesn't alter the Kalman Filter's linear equations. Instead, it *linearizes* the nonlinear equations at the point of the current estimate. The system is linearized by using the jacobian.

$$
\begin{aligned}
\mathbf F
&= {\frac{\partial{f(\mathbf x_t, \mathbf u_t)}}{\partial{\mathbf x}}}\biggr|_{{\mathbf x_t},{\mathbf u_t}} \\
\mathbf H &= \frac{\partial{h(\bar{\mathbf x}_t)}}{\partial{\bar{\mathbf x}}}\biggr|_{\bar{\mathbf x}_t}
\end{aligned}
$$

## Summary of Linear and Nonlinear Kalman Filter Equations

$$\begin{array}{l|l}
\text{Linear Kalman filter} & \text{EKF} \\
\hline
& \boxed{\mathbf F = {\frac{\partial{f(\mathbf x_t, \mathbf u_t)}}{\partial{\mathbf x}}}\biggr|_{{\mathbf x_t},{\mathbf u_t}}} \\
\mathbf{\bar x} = \mathbf{Fx} + \mathbf{Bu} & \boxed{\mathbf{\bar x} = f(\mathbf x, \mathbf u)}  \\
\mathbf{\bar P} = \mathbf{FPF}^\mathsf{T}+\mathbf Q  & \mathbf{\bar P} = \mathbf{FPF}^\mathsf{T}+\mathbf Q \\
\hline
& \boxed{\mathbf H = \frac{\partial{h(\bar{\mathbf x}_t)}}{\partial{\bar{\mathbf x}}}\biggr|_{\bar{\mathbf x}_t}} \\
\textbf{y} = \mathbf z - \mathbf{H \bar{x}} & \textbf{y} = \mathbf z - \boxed{h(\bar{x})}\\
\mathbf{K} = \mathbf{\bar{P}H}^\mathsf{T} (\mathbf{H\bar{P}H}^\mathsf{T} + \mathbf R)^{-1} & \mathbf{K} = \mathbf{\bar{P}H}^\mathsf{T} (\mathbf{H\bar{P}H}^\mathsf{T} + \mathbf R)^{-1} \\
\mathbf x=\mathbf{\bar{x}} +\mathbf{K\textbf{y}} & \mathbf x=\mathbf{\bar{x}} +\mathbf{K\textbf{y}} \\
\mathbf P= (\mathbf{I}-\mathbf{KH})\mathbf{\bar{P}} & \mathbf P= (\mathbf{I}-\mathbf{KH})\mathbf{\bar{P}}
\end{array}$$

## Measurment Model

The measurment model jacobian should be (N_measurments, N_states)

$$
\mathbf H = \frac{\partial{h(\bar{\mathbf x})}}{\partial{\bar{\mathbf x}}}\biggr|_{\bar{\mathbf x}_t} =
\begin{bmatrix}
\frac{\partial h_1}{\partial x_1} & \frac{\partial h_1}{\partial x_2} &\dots \\
\frac{\partial h_2}{\partial x_1} & \frac{\partial h_2}{\partial x_2} &\dots \\
\vdots & \vdots
\end{bmatrix}
$$

# Constant Acceleration Model

Lets start by looking at the constant acceleration model. This will let us estimate the position and veloctiy from accleration measurments from an IMU. Here we assume the IMU axes are fixed in orientation and align with the world coordinate frame.

## State and Measurmen Models

Constant acceleration model with acceleration as input and position sensor (GPS) measurements

```python
# Time step
t = symbols('T')
rx,ry,rz,vx,vy,vz,ax,ay,az = symbols('r_x r_y r_z v_x v_y v_z a_x a_y a_z')
erx,ery,erz,evx,evy,evz = symbols('e_r_x e_r_y e_r_z e_v_x e_v_y e_v_z')

eVec = Matrix([
    er,
    ev
])

r_L = Matrix([rx,ry,rz])
v_L = Matrix([vx,vy,vz])
a_L = Matrix([ax,ay,az])
er  = Matrix([erx,ery,erz])
ev  = Matrix([evx,evy,evz])

X = Matrix([
    r_L,
    v_L,
])

# State transition model
fx_r = r_L + v_L*t + 1/2*a_L*t**2 + er
fx_v = v_L + a_L*t + ev

fx = Matrix([
    fx_r,
    fx_v,
])

# Measurment Model
hx_r = r_L
hx_v = Matrix.zeros(3,1)

hx = Matrix([
    hx_r,
    hx_v,
])

# Display
print('State vector (X)')
display(X,X.shape)
print('State transition model (fx)')
display(fx,fx.shape)
print('Measurment function (hx)')
display(hx,hx.shape)
```

    State vector (X)

$\displaystyle \left[\begin{matrix}r_{x}\\r_{y}\\r_{z}\\v_{x}\\v_{y}\\v_{z}\end{matrix}\right]$

$\displaystyle \left( 6, \  1\right)$

    State transition model (fx)

$\displaystyle \left[\begin{matrix}0.5 T^{2} a_{x} + T v_{x} + e_{r x} + r_{x}\\0.5 T^{2} a_{y} + T v_{y} + e_{r y} + r_{y}\\0.5 T^{2} a_{z} + T v_{z} + e_{r z} + r_{z}\\T a_{x} + e_{v x} + v_{x}\\T a_{y} + e_{v y} + v_{y}\\T a_{z} + e_{v z} + v_{z}\end{matrix}\right]$

$\displaystyle \left( 6, \  1\right)$

    Measurment function (hx)

$\displaystyle \left[\begin{matrix}r_{x}\\r_{y}\\r_{z}\\0\\0\\0\end{matrix}\right]$

$\displaystyle \left( 6, \  1\right)$

## State and Measurment Jacobians

Determine the jacobian of rhte state and measurment models to implement the EKF for nonlinear systems.

```python
# State model jacobian
dfdx = fx.jacobian(X)

dfde = fx.jacobian(eVec)

# Measurment model jacobian
dhdx = hx.jacobian(X)

# Display
print('State model jacobian (dfdx)')
display(dfdx,dfdx.shape)
print('Measurment model jacobian (dhdx)')
display(dhdx,dhdx.shape)
print('G (dfde)')
display(dfde,dfde.shape)
```

    State model jacobian (dfdx)

$\displaystyle \left[\begin{matrix}1 & 0 & 0 & T & 0 & 0\\0 & 1 & 0 & 0 & T & 0\\0 & 0 & 1 & 0 & 0 & T\\0 & 0 & 0 & 1 & 0 & 0\\0 & 0 & 0 & 0 & 1 & 0\\0 & 0 & 0 & 0 & 0 & 1\end{matrix}\right]$

$\displaystyle \left( 6, \  6\right)$

    Measurment model jacobian (dhdx)

$\displaystyle \left[\begin{matrix}1 & 0 & 0 & 0 & 0 & 0\\0 & 1 & 0 & 0 & 0 & 0\\0 & 0 & 1 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\end{matrix}\right]$

$\displaystyle \left( 6, \  6\right)$

    G (dfde)

$\displaystyle \left[\begin{matrix}1 & 0 & 0 & 0 & 0 & 0\\0 & 1 & 0 & 0 & 0 & 0\\0 & 0 & 1 & 0 & 0 & 0\\0 & 0 & 0 & 1 & 0 & 0\\0 & 0 & 0 & 0 & 1 & 0\\0 & 0 & 0 & 0 & 0 & 1\end{matrix}\right]$

$\displaystyle \left( 6, \  6\right)$

# Predict
## State Transition

```python
X = Matrix([
    r_L,
    v_L,
    q_toLfromB,
    #gyroBias,
    #accelBias
])

N_state = len(X)

a_B = accelMeas-accelBias
a_L  = qRot(q_toLfromB, a_B) - g_L

fx_r = r_L + v_L*t + 1/2*a_L**2
fx_v = v_L + a_L*t
fx_q = (eye(4) + t/2*skew4Right(wMeas-gyroBias))*q_toLfromB
#fx_gB = eye(3) * gyroBias
#fx_aB = eye(3) * accelBias

fx = Matrix([
    fx_r,
    fx_v,
    fx_q,
    #fx_gB,
    #fx_aB
       ])

print('X')
display(X)
print(X.shape)
print('fx')
display(fx)
print(fx.shape)

```

## State Transition Jacobian

```python
dfdx = fx.jacobian(X)

print('dfdx')
display(dfdx)
print(dfdx.shape)
```

```python
qv = Matrix([q_toLfromB[1:]]).T
QF = Matrix([
    [qv, qw*eye(3)+skew3(qv)]
])
dvdq = 2*QF*Matrix([
                [zeros(1),a_B.T],
                [a_B, -skew3(a_B)]
])

C_toLfromB = quat2dcm(q_toLfromB)
box = Matrix([
    [-qv.T],
    [qw*eye(3) + skew3(qv)]
])

dfdx = Matrix([
    [zeros(3), eye(3), zeros(3,4),zeros(3),zeros(3)],
    [zeros(3), zeros(3), dvdq, zeros(3), -C_toLfromB ],
    [zeros(4,3), zeros(4,3), 1/2*skew4Right(wMeas-gyroBias), -1/2*box, zeros(4,3)],
    #[zeros(3),zeros(3),zeros(3,4),zeros(3),zeros(3)],
    #[zeros(3),zeros(3),zeros(3,4),zeros(3),zeros(3)]
])
dfdx = dfdx * t + eye(N_state)

print('dfdx')
display(dfdx)
mprint(dfdx)
print(dfdx.shape)

```

## Process Noise

```python
Q_r  = (wa*t**2)**2 * eye(3)
Q_v  = (wa*t)**2 * eye(3)
Q_q  = (ww*t/2)**2 *Matrix([
    [1-qw**2,  -qx*qw,  -qy*qw,  -qz*qw],
    [ -qw*qx, 1-qx**2,  -qy*qx,  -qz*qx],
    [ -qw*qy,  -qx*qy, 1-qy**2,  -qz*qy],
    [ -qw*qz,  -qx*qz,  -qy*qz, 1-qz**2]
])
Q_gB = (wgB*t)**2 * eye(3)
Q_aB = (waB*t)**2 * eye(3)

Q = Matrix([
    [Q_r, zeros(3), zeros(3,4), zeros(3), zeros(3)],
    [zeros(3), Q_v, zeros(3,4), zeros(3), zeros(3)],
    [zeros(4,3), zeros(4,3), Q_q, zeros(4,3), zeros(4,3)],
    [zeros(3), zeros(3), zeros(3,4), Q_gB, zeros(3)],
    [zeros(3), zeros(3), zeros(3,4), zeros(3), Q_aB],

])

print('Q')
display(Q)
print(Q.shape)

#print('Q_r')
#display(Q_r)
#print('Q_v')
#display(Q_v)
#print('Q_q')
#display(Q_q)
#print('Q_gB')
#display(Q_gB)
#print('Q_aB')
#display(Q_aB)
```

# Update

## Measurment Function

```python
hx_accel = -qRot(quatConj(q_toLfromB), g_L)
hx_mag   =  qRot(quatConj(q_toLfromB), B_L)
hx = Matrix([
    hx_accel,
    hx_mag,
])

print('hx')
display(hx)
display(B_L)
```

    hx

$\displaystyle \left[\begin{matrix}- g \left(- 2 q_{w} q_{y} + 2 q_{x} q_{z}\right)\\- g \left(2 q_{w} q_{x} + 2 q_{y} q_{z}\right)\\- g \left(q_{w}^{2} - q_{x}^{2} - q_{y}^{2} + q_{z}^{2}\right)\\m_{x} \left(q_{w}^{2} + q_{x}^{2} - q_{y}^{2} - q_{z}^{2}\right)\\m_{x} \left(- 2 q_{w} q_{z} + 2 q_{x} q_{y}\right)\\m_{x} \left(2 q_{w} q_{y} + 2 q_{x} q_{z}\right)\end{matrix}\right]$

$\displaystyle \left[\begin{matrix}m_{x}\\0\\0\end{matrix}\right]$

## Measurment Function Jacobian

```python
dhdx = hx.jacobian(X)

print('dhdx')
display(dhdx)
mprint(dhdx)

```

    dhdx

$\displaystyle \left[\begin{matrix}0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\\0 & 0 & 0 & 0 & 0 & 0\end{matrix}\right]$

    Matrix([
    [0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0]])

```python

```


## **Viscoelastic wave equation implementation on a staggered grid**

This is a first attempt at implementing the viscoelastic wave equation as described in [1]. See also the FDELMODC implementation by Jan Thorbecke [2].

In the following example, a three dimensional toy problem will be introduced consisting of a single Ricker source located at (100, 50, 35) in a 200 m $\times$ 100 m $\times$ 100 *m* domain.

```python
# Required imports:
import numpy as np
import sympy as sp

from devito import *
from examples.seismic.source import RickerSource, TimeAxis
from examples.seismic import ModelViscoelastic, plot_image
```

The model domain is now constructed. It consists of an upper layer of water, 50 m in depth, and a lower rock layer separated by a 4 m thick sediment layer.

```python
# Domain size:
extent = (200., 100., 100.) # 200 x 100 x 100 m domain
h = 1.0 # Desired grid spacing
shape = (int(extent[0]/h+1), int(extent[1]/h+1), int(extent[2]/h+1))

# Model physical parameters:
vp = np.zeros(shape)
qp = np.zeros(shape)
vs = np.zeros(shape)
qs = np.zeros(shape)
rho = np.zeros(shape)

# Set up three horizontally separated layers:
vp[:,:,:int(0.5*shape[2])+1] = 1.52
qp[:,:,:int(0.5*shape[2])+1] = 10000.
vs[:,:,:int(0.5*shape[2])+1] = 0.
qs[:,:,:int(0.5*shape[2])+1] = 0.
rho[:,:,:int(0.5*shape[2])+1] = 1.05

vp[:,:,int(0.5*shape[2])+1:int(0.5*shape[2])+1+int(4/h)] = 1.6
qp[:,:,int(0.5*shape[2])+1:int(0.5*shape[2])+1+int(4/h)] = 40.
vs[:,:,int(0.5*shape[2])+1:int(0.5*shape[2])+1+int(4/h)] = 0.4
qs[:,:,int(0.5*shape[2])+1:int(0.5*shape[2])+1+int(4/h)] = 30.
rho[:,:,int(0.5*shape[2])+1:int(0.5*shape[2])+1+int(4/h)] = 1.3

vp[:,:,int(0.5*shape[2])+1+int(4/h):] = 2.2
qp[:,:,int(0.5*shape[2])+1+int(4/h):] = 100.
vs[:,:,int(0.5*shape[2])+1+int(4/h):] = 1.2
qs[:,:,int(0.5*shape[2])+1+int(4/h):] = 70.
rho[:,:,int(0.5*shape[2])+1+int(4/h):] = 2.
```

Now create a Devito vsicoelastic model generating an appropriate computational grid along with absorbing boundary layers:

```python
# Create model
origin = (0, 0, 0)
spacing = (h, h, h)
so = 4 # FD space order (Note that the time order is by default 1).
nbl = 20 # Number of absorbing boundary layers cells
model = ModelViscoelastic(space_order=so, vp=vp, qp=qp, vs=vs, qs=qs,
                          b=1/rho, origin=origin, shape=shape, spacing=spacing,
                          nbl=nbl)
```

    Operator `initdamp` run in 0.04 s
    Operator `padfunc` run in 0.06 s
    Operator `padfunc` run in 0.35 s
    Operator `padfunc` run in 0.02 s
    Operator `padfunc` run in 0.03 s
    Operator `padfunc` run in 0.01 s

The source frequency is now set along with the required model parameters:

```python
# Source freq. in MHz (note that the source is defined below):
f0 = 0.12

# Thorbecke's parameter notation
l = model.lam
mu = model.mu
ro = model.b

k = 1.0/(l + 2*mu)
pi = l + 2*mu

t_s = (sp.sqrt(1.+1./model.qp**2)-1./model.qp)/f0
t_ep = 1./(f0**2*t_s)
t_es = (1.+f0*model.qs*t_s)/(f0*model.qs-f0**2*t_s)
```

```python
# Time step in ms and time range:
t0, tn = 0., 30.
dt = model.critical_dt
time_range = TimeAxis(start=t0, stop=tn, step=dt)
```

Generate Devito time functions for the velocity, stress and memory variables appearing in the viscoelastic model equations. By default, the initial data of each field will be set to zero.

```python
# PDE fn's:
x, y, z = model.grid.dimensions
damp = model.damp

# Staggered grid setup:

# Velocity:
v = VectorTimeFunction(name="v", grid=model.grid, time_order=1, space_order=so)

# Stress:
tau = TensorTimeFunction(name='t', grid=model.grid, space_order=so, time_order=1)

# Memory variable:
r = TensorTimeFunction(name='r', grid=model.grid, space_order=so, time_order=1)

s = model.grid.stepping_dim.spacing # Symbolic representation of the model grid spacing
```

And now the source and PDE's are constructed:

```python
# Source
src = RickerSource(name='src', grid=model.grid, f0=f0, time_range=time_range)
src.coordinates.data[:] = np.array([100., 50., 35.])

# The source injection term
src_xx = src.inject(field=tau[0, 0].forward, expr=src*s)
src_yy = src.inject(field=tau[1, 1].forward, expr=src*s)
src_zz = src.inject(field=tau[2, 2].forward, expr=src*s)

# Particle velocity
u_v = Eq(v.forward, model.damp * (v + s*ro*div(tau)))

# Stress equations:
u_t = Eq(tau.forward,  model.damp *  (s*r.forward + tau +
                                      s * (l * t_ep / t_s * diag(div(v.forward)) +
                                           mu * t_es / t_s * (grad(v.forward) + grad(v.forward).T))))

# Memory variable equations:
u_r = Eq(r.forward, damp * (r - s / t_s * (r + l * (t_ep/t_s-1) * diag(div(v.forward)) +
                                           mu * (t_es/t_s-1) * (grad(v.forward) + grad(v.forward).T) )))
```

We now create and then run the operator:

```python
# Create the operator:
op = Operator([u_v, u_r, u_t] + src_xx + src_yy + src_zz,
               subs=model.spacing_map)
```

```python
#NBVAL_IGNORE_OUTPUT

# Execute the operator:
op(dt=dt)
```

    Operator `Kernel` run in 21.27 s

    PerformanceSummary([(PerfKey(name='section0', rank=None),
                         PerfEntry(time=0.014527, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[])),
                        (PerfKey(name='section1', rank=None),
                         PerfEntry(time=21.237021000000002, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[])),
                        (PerfKey(name='section2', rank=None),
                         PerfEntry(time=0.005033999999999999, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[]))])

Before plotting some results, let us first look at the shape of the data stored in one of our time functions:

```python
v[0].data.shape
```

    (2, 241, 141, 141)

Since our functions are first order in time, the time dimension is of length 2. The spatial extent of the data includes the absorbing boundary layers in each dimension (i.e. each spatial dimension is padded by 20 grid points to the left and to the right).

The total number of instances in time considered is obtained from:

```python
time_range.num
```

    136

Hence 223 time steps were executed. Thus the final time step will be stored in index given by:

```python
np.mod(time_range.num,2)
```

    0

Now, let us plot some 2D slices of the fields `vx` and `szz` at the final time step:

```python
#NBVAL_SKIP

# Mid-points:
mid_x = int(0.5*(v[0].data.shape[1]-1))+1
mid_y = int(0.5*(v[0].data.shape[2]-1))+1

# Plot some selected results:

plot_image(v[0].data[1, :, mid_y, :], cmap="seismic")
plot_image(v[0].data[1, mid_x, :, :], cmap="seismic")

plot_image(tau[2, 2].data[1, :, mid_y, :], cmap="seismic")
plot_image(tau[2, 2].data[1, mid_x, :, :], cmap="seismic")
```

```python
#NBVAL_IGNORE_OUTPUT

assert np.isclose(norm(v[0]), 0.102959, atol=1e-4, rtol=0)
```

# References

[1] Johan O. A. Roberston, *et.al.* (1994). "Viscoelatic finite-difference modeling" GEOPHYSICS, 59(9), 1444-1456.

[2] https://janth.home.xs4all.nl/Software/fdelmodcManual.pdf


```python
# good python code examples:
https://www.inovex.de/de/blog/support-vector-machines-guide/
```

# Support-Vector-Machines = Maximum Margin Classification

```python
import numpy as np
from IPython.display import Image
import matplotlib.pyplot as plt
```

```python
# seperating line
b = 10
w1 = -1.5
w2 = -1
# as vector and the normal to the separating line
w = np.array([w1, w2])
w_norm = w/np.sqrt(np.dot(w, w))
this_is_the_lenth = np.dot(w_norm, w_norm)
# class 1
cl1 = np.array([np.random.normal(loc=1.5, scale=1.1, size=5),
          np.random.normal(loc=0, scale = 1.2, size = 5)]).T
# support vectors class 1:
cl1_s = np.array([np.array([5, 7]), (b+4) + w1 * np.array([5, 7])]).T

# class 2
cl2 = np.array([np.random.normal(loc=11, scale=2., size=7),
                np.random.normal(loc=4, scale=0.6, size=7)]).T
# support vectors class 2:
cl2_s = np.array([np.array([3, 5]), (b-4) + w1 * np.array([3, 5])]).T

fig = plt.figure()
ax = plt.axes()
plt.xlim(-3, 15)
plt.ylim(-5, 10)
plt.gca().set_aspect('equal', adjustable='box')
x = np.linspace(-5, 15, 100)
y = b + w1 * x
ax.plot(x, b + w1 * x)
ax.plot(x, (b-4) + w1 * x, color='red')
ax.plot(x, (b+4) + w1 * x, color='red')
ax.scatter(cl1[:, 0], cl1[:, 1], color='blue', alpha=0.5)
ax.scatter(cl2[:, 0], cl2[:, 1], color='red', alpha=0.5)
ax.scatter(cl1_s[:, 0], cl1_s[:, 1], color='black', alpha=0.7)
ax.scatter(cl2_s[:, 0], cl2_s[:, 1], color='blue', alpha=0.6)
#ax.scatter(7, b + w[0] * 7, color='black')
ax.scatter(41/13, b + w1 *(41/13), color = 'black')
ax.plot([41/13, cl1_s[0,0]],[b + w1 * (41/13), cl1_s[0,1]], color='black')
plt.annotate(text='d',xy=(45/13, 6.5))

plt.arrow(-2.5, -2.5, w[0] * -2, w[1] * -2, head_width=0.5)
plt.annotate(text='w', xy=(-1,-0.5))
# plt.scatter(x, y, color='red', alpha = 0.45)
# print(f'{b + w1 * x + w2 * y}')
```

Im 2-dimensionalen Raum haben wir die Variablen x, y. Alle Punkte auf einer Geraden (separating plane) bekommt man:
\begin{equation*}
a + b\cdot x = c\cdot y
\end{equation*}

Trivialerweise ist $c=1$ meist gültig.

Umgekehrt kann man die Gerade auch als die Menge aller Punkte definieren, die folgender Bedingung genügen:

\begin{equation*}
a + b\cdot x + c\cdot y = 0
\end{equation*}

Im trivialen Fall wäre $c=-1$

Für alle Punkte unterhalb der Gerade gilt:
\begin{equation*}
a + b\cdot x + c\cdot y < 0
\end{equation*}
und für alle Punkte oberhalb der Geraden gilt:
\begin{equation*}
a + b\cdot x + c\cdot y > 0
\end{equation*}

Für die Klassification gilt, dass die Klassen immer mit $-1$ und $1$ benannt werden. Ein neuer Punkt $(x, y)$ wird klassifiziert:
\begin{equation*}
\hat{\text{pred}} = \text{sign}(a + b\cdot x + c\cdot y)
\end{equation*}

### Aenderung der Schreibweise:
Weil wir kein Regressionsproblem haben, sondern ein Klassifikationsproblem, benennen wir im obigen Beispiel die Koeffizienten und die Variablen um:
\begin{equation*}
\hat{\text{pred}} = \text{sign}(b + \omega_1\cdot x_i + \omega_2\cdot x_2)
\end{equation*}
und in Vektorschreibweise:
\begin{equation*}
\hat{\text{pred}} = \text{sign}(b + \mathbf{\omega}\mathbf{x})
\end{equation*}

Es kann gezeigt werden, dass der Vektor $\mathbf{\omega}$ senkrecht auf die Trennungs-Ebene (in der Graphik oben die mittlere Gerade) steht. Dies habe ich als Pfeil eingezeichnet.

Nun soll eine Ebene gefunden werden, deren __margin maximal__ ist; als margin wird der Abstand zu den nähesten Daten-Punkten bezeichnet. Die Datenpunkte, die auf den margins zu liegen kommen heissen __support vectors__. Die Enfernung dieser Punkte von der Trennlinie kann angegeben werden:

\begin{equation*}
d= y_i(\mathbf{\omega}\mathbf{x}_i + b);\quad \quad y_i \in \{-1, 1\},
\end{equation*}

wobei $y_i$ ändert das Vorzeichen, so dass die Distanz immer positiv ist.

Für alle Punkte $\mathbf{x_i}$, die genau die Distanz 1 haben gilt:

\begin{eqnarray*}
1 =& y_i(\mathbf{\omega}\mathbf{x}_i + b)\\
\frac{1}{\| \mathbf{\omega}\| } =& \left|\frac{\mathbf{\omega}\mathbf{x}_i}{\|\mathbf{\omega}\|} + \frac{b}{\|\mathbf{\omega}\|}\right|
\end{eqnarray*}

$\|\mathbf{\omega}\|$ ist die Länge des Vektors $\mathbf{\omega}$, die sogenannte Vecktor-Norm. Teile ich einen Vektor durch seine Vektor-Norm, dann hat er genau die Länge $1$. Weil alle Punkte auf dem margin dieselbe Distanz zur Trennlinie haben und weil der margin symmetrisch ist muss

\begin{equation*}
p = \frac{2}{\|\mathbf{\omega}\|}
\end{equation*}
maximiert werden, um den margin möglichst gross zu machen.

Der Abstand der beide support-vectors ist also indirekt proportional zur Länge des Vectors $\omega$. Als Minimierungs-Problem ergibt sich:
\begin{equation*}
(\omega^*, b^*) = \text{argmin} \frac{1}{2}\|\mathbf{\omega}\|
\end{equation*}
Damit haben wir die formulierung unseres maximum-margin classifiers:

Finde $\mathbf{\omega}$ und $b$ so, dass:
* $\frac{1}{2}\omega^{T}\omega$ minimiert wird, für alle
* $(\mathbf{x}_i, y_i), y_i(\mathbf{\omega}\mathbf{x}_i + b) \geq 1$

Will man das lösen, ergibt sich ein quadratische Optimierungsproblem, welches mit Lagrange Multiplikatoren gelöst werden kann.

Die characteristische Gleichung ergibt sich zu:
1)  \begin{equation*}\text{max}_{\alpha}\sum_i\alpha_i - \frac{1}{2}\alpha_i\alpha_j y_i y_j \mathbf{x}_i^{T} \cdot \mathbf{x}_j\end{equation*}
2) mit: \begin{equation*}\sum_i\alpha_i y_i = 0\end{equation*}

Für das Verständnis der nachfolgenden Erklärungen zu kernels, ist der Ausdruck $\mathbf{x}_i^{T} \cdot \mathbf{x}_j$ entscheidend. Hierbei wird das Skalarprodukt eines jeden Daten-Punktes mit jedem anderen Daten-Punkt berechnet. Für normierte Vektoren $\mathbf{x}_i$ ist das Skalarprodukt ein Ähnlichkeitsmass (cosine-similarity, correlation-coefficient).

### Problem:
Der Ausdruck

\begin{equation*}
\mathbf{x}_i^{T} \cdot \mathbf{x}_j
\end{equation*}

zeigt, dass Support-Vecktor-Machines quadratisch mit der Anzahl der Trainings-Punkte skalieren. SVMs sind also für BigData nicht zu gebrauchen.

# Kernel-Trick

```python
from sklearn.datasets import make_circles
import matplotlib.pyplot as plt
import numpy as np
```

```python
x, y = make_circles(n_samples = (70, 30), random_state=42, factor=0.8)
```

```python
plt.figure(figsize = (8,6))
plt.scatter(x[y==1, 0], x[y==1, 1], color='red', alpha = 0.45)
plt.scatter(x[y==0, 0], x[y==0, 1], color='blue', alpha = 0.45)
plt.title('inseparable datasets')
plt.xlabel('1st dimension')
plt.ylabel('2nd dimension')
```

### project the data into 3 dimensions

\begin{equation*}
\Phi(\mathbf{x})\rightarrow \mathbf{x}_1^{2}, \mathbf{x}_2^{2},\sqrt{2}\mathbf{x}_1\mathbf{x}_2
\end{equation*}

```python
x3 = np.sqrt(2) * x[:, 0] * x[:, 1]
x3 = x3.reshape((-1, 1))
mapped_x = np.concatenate((x ** 2, x3), axis=1)
mapped_x.shape
```

    (100, 3)

```python
fig = plt.figure(figsize = (8,6))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(mapped_x[y==1, 0], mapped_x[y==1, 1], mapped_x[y==1, 2], color='red', alpha = 0.45)
ax.scatter(mapped_x[y==0, 0], mapped_x[y==0, 1], mapped_x[y==0, 2], color='blue', alpha = 0.45)
#plt.title('inseparable datasets')
ax.set_xlabel('1st dimension')
ax.set_ylabel('2nd dimension')
ax.set_zlabel('3rd dimension')
```

### just another projection into 3D

\begin{equation*}
\Phi(\mathbf{x})\rightarrow \mathbf{x}_i, \mathbf{x}_2, \sum_i^n \exp\left(-\frac{\|\mathbf{x}- \mathbf{x}_i\|^2}{2\sigma^2}\right)
\end{equation*}

__ABER__ wie wir später sehen werden, gibt es für diese Projektion keine einfachere Lösung, d.h. hier kann der kernel-Trick nicht angewendet werden.

```python
# not 100 % correct though
# https://stats.stackexchange.com/questions/63881/use-gaussian-rbf-kernel-for-mapping-of-2d-data-to-3d

from scipy.spatial.distance import pdist, squareform
from scipy import exp
pairwise_d = squareform(pdist(x, 'euclidean'))
K = exp(-pairwise_d **2 / (2 ** 2))
new_dim = np.sum(K, axis=1)
```

    /home/martin/miniconda3/lib/python3.6/site-packages/ipykernel_launcher.py:7: DeprecationWarning: scipy.exp is deprecated and will be removed in SciPy 2.0.0, use numpy.exp instead
      import sys

```python
fig = plt.figure(figsize = (8,6))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x[y==1, 0], x[y==1, 1], new_dim[y==1], color='red', alpha = 0.45)
ax.scatter(x[y==0, 0], x[y==0, 1], new_dim[y==0], color='blue', alpha = 0.45)
#plt.title('inseparable datasets')
ax.set_xlabel('1st dimension')
ax.set_ylabel('2nd dimension')
ax.set_zlabel('3rd dimension')
```

### Der kernel-Trick besteht genau darin, dass die Projektion in einen höher-dimensionalen Raum nicht vorgenommen werden muss

Wir erinnern die charakteristische Formel des maximal-margin classifiers:

1) \begin{equation*}\text{max}_{\alpha}\sum_i\alpha_i - \frac{1}{2}\alpha_i\alpha_j y_i y_j \mathbf{x}_i^{T} \cdot \mathbf{x}_j\end{equation*}
2) mit: \begin{equation*}\sum_i\alpha_i y_i = 0\end{equation*}

Anstatt mit den urspr. Daten-Punkten $\mathbf{x}_i$ und $\mathbf{x}_j$ muss diese Gleichung jetzt für die Projektionen der Daten vorgenommen werden:

### unsere erste Projektion:

$\Phi(\mathbf{x})\rightarrow \mathbf{x}_1^{2}, \mathbf{x}_2^{2},\sqrt{2}\mathbf{x}_1\mathbf{x}_2$

einsetzen in die characteristische Gleichung für 2 Daten-Punkte $\mathbf{x}_i$ und $\mathbf{x}_j$:
\begin{eqnarray*}
\langle\Phi(\mathbf{x}_i),\Phi(\mathbf{x}_j)\rangle =&\\
\langle[x_{i1}^{2},x_{i2}^{2}, \sqrt{2}x_{i1}x_{i2}],[x_{j1}^{2},x_{j2}^{2}, \sqrt{2}x_{j1}x_{j2}]\rangle =& \\
x_{i1}^{2}x_{j1}^{2} + x_{i2}^{2}x_{j2}^{2} +2 x_{i1}x_{i2}x_{j1}x_{j2}
\end{eqnarray*}

### Trick: Es gibt eine einfachere Rechenvorschrift, die zum selben Ergebnis kommt. Das ist der Kernel-Trick

$$\langle\Phi(\mathbf{x}_i),\Phi(\mathbf{x}_j)\rangle = \langle\mathbf{x}_i,\mathbf{x}_j\rangle^{2} =$$
$$\langle[x_{i1}, x_{i2}],[x_{j1}, x_{j2}]\rangle^{2} = $$
$$(x_{i1}x_{j1} + x_{i2}x_{j2})^{2} = $$
$$x_{i1}^{2}x_{j1}^{2} + x_{i2}^{2}x_{j2}^{2} + 2 x_{i1}x_{i2}x_{j1}x_{j2}$$

Man kann jetzt also schreiben:

$$ k(\mathbf{x_i}, \mathbf{x_j}) = \langle\mathbf{x}_i,\mathbf{x}_j\rangle^{2}$$

Für unsere charackteristische Gleichung könnten wir nun schreiben:

$$\text{max}_{\alpha}\sum_i\alpha_i - \frac{1}{2}\alpha_i\alpha_j y_i y_j k(\mathbf{x_i}, \mathbf{x_j})$$

Unser maximal-margin Classifier könnte als das Problem der 2 Ringe lösen, ohne explizit die Projektione in 3D vornehmen zu müssen.

Unterschied:

1.Methode:
* Projektion in höher-dimensionalen Raum
* Multiplikation in höher-dimensionalen Raum

2.Method:
* Multiplikation im Ausgangs-Raum
* Quadratur des Ergebnisses

# What are kernels exactly?

Basically, kernels are similarity measures outside the usual Euclidean space.

For example, take the __radial basis function kernel__:

$$K(X_1, X_2) = \exp\left(-\frac{\|X_1 - X_2\|^2}{2\sigma^2}\right)$$

$\|X_1 - X_2\|$ is the Euclidean distance between the points $X_1$ and $X_2$ or between two vectors $\mathbf{x}_1$ and $\mathbf{x}_2$.

If this distance is $0$, i.e. the two cases are actually identical, then the similarity -- as measured by the rbf-kernel -- is $1$:

```python
Image('../images/rbf_kernel.png')
```

```python
import warnings
warnings.filterwarnings('ignore')
from sklearn.neighbors import KernelDensity
import numpy as np
import matplotlib.pyplot as plt
def format_func(x, loc):
    if x == 0:
        return '0'
    elif x == 1:
        return '1'
    elif x == -1:
        return '-1'
    else:
        return '%i' % x
X_plot = np.linspace(-6, 6, 1000)[:, None]
X_src = np.zeros((1, 1))

fig, ax = plt.subplots(1, 1, sharex=True, sharey=True)
fig.subplots_adjust(left=0.05, right=0.95, hspace=0.05, wspace=0.05)
log_dens = KernelDensity(kernel='gaussian').fit(X_src).score_samples(X_plot)
ax.plot(X_plot[:, 0], np.exp(log_dens)/np.max(np.exp(log_dens)), '-k')#, fc='#AAAAFF')
ax.text(-2.6, 0.95, 'gaussian')

ax.xaxis.set_major_formatter(plt.FuncFormatter(format_func))
ax.xaxis.set_major_locator(plt.MultipleLocator(1))
#ax.yaxis.set_major_locator()

ax.set_ylim(0, 1.05)
ax.set_xlim(-2.9, 2.9)
plt.ylabel('similarity of a distance of 0')
density_at_1_5 = np.exp(KernelDensity(kernel='gaussian').fit(X_src).score_samples(np.array([1.5]).reshape((-1,1))))/np.max(np.exp(log_dens))
ax.plot([1.5] *2, [0, density_at_1_5],
        'b--', lw=2, label='')
ax.plot([1.5, -3], [density_at_1_5] * 2, 'b--', lw=2, label='')
ax.plot([0] * 2, [0, 1], 'r--', lw=2, label='')
ax.plot([0, -3], [1] * 2, 'r--', lw=2, label='')
```

# Simple Kernel-Regression
Here, Kernels just define a neighborhood by similarity in kernel-space. For a certain point the values of its neighbors are weighted by similarity and then averaged.

The Euclidean distance on the x-axis is transformed into a similarity-measure on the y-axis.

```python
# f = plt.figure(figsize=(5, 5), dpi=100)
y = np.load('/home/martin/python/fhnw_lecture/scripts/regression_y.pickle.npy')
X = np.load('/home/martin/python/fhnw_lecture/scripts/regression_X.pickle.npy')

fig = plt.figure(figsize=(8,8), constrained_layout=True)
plt.title(label='kernel regression', fontdict={'fontsize':20})
gs = fig.add_gridspec(nrows=3, ncols=3, figure=fig)
fig.subplots_adjust( left=0.05, right=0.95, wspace=0.05)
ax1 = fig.add_subplot(gs[:-1, :])

ax1.plot(X[:,0], y, 'ro')

for i,j in zip(X[2:7], y[2:7]):
    ax1.plot([i] * 2, [-1, j], 'r--', lw=2, label='')
ax1.set_ylim([np.min(y)-5, np.max(y) +5])
ax1.get_yaxis().set_visible(True)
ax1.set_xlim([np.min(X)-2, np.max(X) +2])
#axes = plt.gca()

ax2 = fig.add_subplot(gs[-1:, :])
ax2.set_xlim([np.min(X)-2, np.max(X) +2])
X_plot = np.linspace(1, 9, 1000)[:, None]
X_src = np.array([[4.8]])
log_dens = KernelDensity(kernel='gaussian').fit(X_src).score_samples(X_plot)
ax2.plot(X_plot[:, 0], np.exp(log_dens)/np.max(np.exp(log_dens)), '-k')#, fc='#AAAAFF')

densities = np.exp(KernelDensity(kernel='gaussian').fit(X_src).score_samples(X[2:7, ]))
for i, d in zip(X[2:7,], densities):
    ax2.plot([i] * 2, [d/np.max(np.exp(log_dens)), 1.5], 'r--', lw=2, label='')
    ax2.plot([-5, i], [d/np.max(np.exp(log_dens))] * 2, 'b--', lw=2, label='')

# make vertikal line at 5.0
ax2.plot([4.8] *2, [0, 1.5], 'k--', lw=1)
ax2.set_ylabel('densities (or similarities)')
#plt.tight_layout()

```

The prediction takes part at the center $x$ of the kernel. The similiarity to this center-point $x$ are evaluated for all points $x_i$ in the vicinity and the respective values $y_i$ are weighted by this similarity:

\begin{equation}
\hat{m}(x) = \frac{\sum_i^n K_h(x - x_i)y_i}{\sum_i^n K_h(x - x_i)}
\end{equation}

Here, $h$ is the bandwidth or the scale of the kernel. In the context of a gaussian kernel one could also say the __variance__.

```python
fig = plt.figure(figsize=(8,8), constrained_layout=True)
plt.title(label='kernel regression', fontdict={'fontsize':20})
gs = fig.add_gridspec(nrows=3, ncols=3, figure=fig)
fig.subplots_adjust( left=0.05, right=0.95, wspace=0.05)
ax1 = fig.add_subplot(gs[:-1, :])

ax1.plot(X[:,0], y, 'ro')

for i,j in zip(X[2:7], y[2:7]):
    ax1.plot([i] * 2, [0, j], 'r--', lw=2, label='')
ax1.set_ylim([np.min(y)-5, np.max(y) +5])
ax1.get_yaxis().set_visible(True)
#axes = plt.gca()
x = np.arange(1, 12, 0.05).reshape((-1, 1))
y_kr = []
for x_i in x:
    densities = np.exp(KernelDensity(kernel='gaussian').fit(np.array([[x_i[0]]])).score_samples(X))
    y_kr.append(np.sum(densities.reshape((-1, 1)) * y)/sum(densities))
ax1.plot(X[:,0], y, 'ro', x[:,0], np.array(y_kr).reshape((-1,)))
ax1.set_xlim([np.min(X)-2, np.max(X) +2])

# make vertikal line at 8.3
densities = np.exp(KernelDensity(kernel='gaussian').fit(np.array([[8.3]])).score_samples(X))
pred_kr = np.sum(densities.reshape((-1, 1)) * y)/sum(densities)
ax1.plot([8.3] * 2, [-1, pred_kr], 'k--', lw=1)
ax1.plot(8.3, pred_kr, 'bo')

# make vertikal line at 4.8
densities = np.exp(KernelDensity(kernel='gaussian').fit(np.array([[4.8]])).score_samples(X))
pred_kr = np.sum(densities.reshape((-1, 1)) * y)/sum(densities)
ax1.plot([4.8] * 2, [-1, pred_kr], 'k--', lw=1)
ax1.plot(4.8, pred_kr, 'bo')

ax2 = fig.add_subplot(gs[-1:, :])
ax2.set_xlim([np.min(X)-2, np.max(X) +2])

X_plot = np.linspace(4.3, 12.3, 1000)[:, None]
X_src = np.array([[8.3]])
log_dens = KernelDensity(kernel='gaussian').fit(X_src).score_samples(X_plot)
ax2.plot(X_plot[:, 0], np.exp(log_dens)/np.max(np.exp(log_dens)), 'k--', lw=1)#, fc='#AAAAFF')

X_plot = np.linspace(0.8, 8.8, 1000)[:, None]
X_src = np.array([[4.8]])
log_dens = KernelDensity(kernel='gaussian').fit(X_src).score_samples(X_plot)
ax2.plot(X_plot[:, 0], np.exp(log_dens)/np.max(np.exp(log_dens)), '-k')#, fc='#AAAAFF')

densities = np.exp(KernelDensity(kernel='gaussian').fit(X_src).score_samples(X[2:7, ]))
for i, d in zip(X[2:7,], densities):
    ax2.plot([i] * 2, [d/np.max(np.exp(log_dens)), 1.5], 'r--', lw=2, label='')
    ax2.plot([-5, i], [d/np.max(np.exp(log_dens))] * 2, 'b--', lw=2, label='')

# make vertikal line at 5.0
ax2.plot([4.8] *2, [0, 1.5], 'k--', lw=1)
# make vertikal line at 8.0
ax2.plot([8.3] *2, [0, 1.5], 'k--', lw=1)

ax2.set_ylabel('densities (or similarities)')
plt.tight_layout()

```

```python
x = np.arange(1, 12, 0.05).reshape((-1, 1))
y_kr = []
for x_i in x:
    densities = np.exp(KernelDensity(kernel='gaussian').fit(np.array([[x_i[0]]])).score_samples(X))
    y_kr.append(np.sum(densities.reshape((-1, 1)) * y)/sum(densities))

f = plt.figure(figsize=(5, 5), dpi=100)
plt.title(label='kernel regression', fontdict={'fontsize':20})
axes = f.add_subplot(111)
axes.plot(X[:,0], y, 'ro', x[:,0], np.array(y_kr).reshape((-1,)))
#axes = plt.gca()
axes.set_ylim([np.min(y)-5, np.max(y) +5])
```

## How is the similarity-measure related to the support-vector-machine?

Remember that in the characteristic equation we had:

$$\mathbf{x}_i^{T} \cdot \mathbf{x}_j$$

This is the dot-product between two vectors. The dot-product for standardized vectors happens to coincide with the cosine-similarity that is identical to the correlation-coefficient $r$.

With the kernel trick, we can now apply many different similarity-measures that hopefully help to distinguish our classes better than euclidean distance can do.

Or, put in another way: we can either project our data into higher spaces and compute our conventional similarity measures in those spaces __or__: we use other similarity-measures that approximate the former approach.

## One-Class SVM
Finde die kleinst mögliche Kugel (hyper sphere), die alle Datenpunkte umschliesst.
Angewendet auf neue Datenpunkte:
    - finde die Distanz zum Mittelpunkt: ist diese Grösser als der Radius der hyper sphere, dann ist der Datenpunkt nicht in der Klasse
    - finde die Distanz zur Begrenuzungsebene: pos Distanz -> in-class; neg Distanz -> out-class