# カーネル法

### 重回帰分析の式
$$
{\hat{y} = f({\bf x}) = w_0 + w_1 x_1 + w_2 x_2 + \ldots + w_n x_n
}
$$
マルチレイヤーパーセプトロンに似ている式
w_0はバイアスと等価
### 正規方程式
$$
J(\theta) = \frac{1}{2m}\sum_{i=1}^{m}(h_{\theta}(x^{(i)})-y^{(i)})^2
$$
微分した結果、２乗が展開されたような式になる

$$
\begin{eqnarray}
\frac{\partial}{\partial \theta_j}J(\theta) &=& \frac{1}{m}\sum_{i=1}^{m}(h_{\theta}(x^{(i)})-y^{(i)})x_j^{(i)} \\
&=& \frac{1}{m}\{\sum_{k}\theta_k\sum_{i}x_j^{(i)}x_k^{(i)} - \sum_{i}x_j^{(i)}y^{(i)}\}
\end{eqnarray}
$$

hθ(x)は、xという入力にθという重みをかけたものになる。Σkがポイント。

$$
m\left[\begin{array}{cc}
\frac{\partial}{\partial \theta_0}J(\theta)\\
\frac{\partial}{\partial \theta_1}J(\theta)\\
\vdots\\
\frac{\partial}{\partial \theta_n}J(\theta)
\end{array}\right] =
X^TX \theta - X^Ty
$$
正規方程式の証明終了
$$
\begin{eqnarray}
X^TX \theta &=& X^Ty\\
\theta &=& (X^TX)^{-1} X^Ty
\end{eqnarray}
$$
両辺に、左辺のΘの係数の逆行列を掛け算する

$$
{f({\bf x}) = \sum_{i=0}^{n} w_i x_i
}
$$

$$
{f({\bf x}) = \sum_{i=1}^{N} \alpha_i k({\bf x}^{(i)}, {\bf x})
}
$$

## ガウスカーネル
$$
{k({\bf x}, {\bf x}') = exp(-\beta \|{\bf x} - {\bf x}'\|^2)
}
$$
expはeのx乗　つまり、2.71...のx乗 xは(  )の中身
βはβ>0のハイパーパラメータであり、任意の変数

$$
\|{\bf x} - {\bf x}'\|=\sqrt{\sum_{j=1}^n (x_j - x'_j)^2}
$$
$$\|{\bf x} - {\bf x}'\|はl2ノルムを表している。$$

$$
k(x,x')=exp(-\beta(x-x')^2)
$$

```python
!pip --proxy=wwwproxy.kanazawa-it.ac.jp:8080 install seaborn
```

    Requirement already satisfied: seaborn in /usr/local/lib/python2.7/site-packages (0.9.0)
    Requirement already satisfied: pandas>=0.15.2 in /usr/local/lib/python2.7/site-packages (from seaborn) (0.23.4)
    Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python2.7/site-packages (from seaborn) (1.1.0)
    Requirement already satisfied: matplotlib>=1.4.3 in /usr/local/lib/python2.7/site-packages (from seaborn) (2.0.2)
    Requirement already satisfied: numpy>=1.9.3 in /usr/local/lib/python2.7/site-packages (from seaborn) (1.13.1)
    Requirement already satisfied: pytz>=2011k in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from pandas>=0.15.2->seaborn) (2016.10)
    Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from pandas>=0.15.2->seaborn) (2.6.0)
    Requirement already satisfied: six>=1.10 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from matplotlib>=1.4.3->seaborn) (1.10.0)
    Requirement already satisfied: functools32 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from matplotlib>=1.4.3->seaborn) (3.2.3.post2)
    Requirement already satisfied: subprocess32 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from matplotlib>=1.4.3->seaborn) (3.2.7)
    Requirement already satisfied: cycler>=0.10 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from matplotlib>=1.4.3->seaborn) (0.10.0)
    Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=1.5.6 in /usr/local/Cellar/matplotlib/2.0.2/libexec/lib/python2.7/site-packages (from matplotlib>=1.4.3->seaborn) (2.1.10)
    You are using pip version 10.0.1, however version 18.1 is available.
    You should consider upgrading via the 'pip install --upgrade pip' command.

```python
%matplotlib inline
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns

# 線形回帰モデル用データセット
# a : slope of a line
# b : intercept of a line
# size : number of data
# xlim : domain of variable x
# scale : standard deviation
def linear_dataset(a, b, size, xlim=[0, 1], scale=None):
    x = np.random.uniform(xlim[0], xlim[1], size)
    y = a * x + b
    if scale is not None:
        noize = np.random.normal(0, scale, size)
        y = y + noize
    df = pd.DataFrame()
    df['x'] = x
    df['y'] = y
    return df

# データを生成
data = linear_dataset(0.5, 0.2, 20, scale=0.03)

# データを描画
plt.figure(figsize=(6.472, 4))
plt.scatter(data['x'].as_matrix(), data['y'].as_matrix())
plt.xlim((0.1, 1.05))
plt.ylim((0.2, 0.8))
plt.show()
```

```python
def kernel(xi, xj, beta=1):
    return np.exp(- beta * np.sum((xi - xj)**2))

X = np.arange(-1.5, 1.5, 0.01)
Y = np.zeros(len(X))
for i in range(len(X)):
    Y[i] = kernel(X[i], 0, 10)

plt.figure(figsize=(6.472, 4))
plt.plot(X, Y)
plt.show()
```

```python
def kernel(xi, xj, beta=1):
    return np.exp(- beta * np.sum((xi - xj)**2))#np.sumを外しても動作します。

def kernel_test(xi, xj, beta=1):
    print((xi - xj))
    return np.exp(- beta * np.sum((xi - xj)**2))

X = np.arange(-1.5, 1.5, 0.01)
Y = np.zeros(len(X))

centers = [-1, 0, 0.5, 0.6, 1]
weights = [0.2, 1, 0.3, -1, 0.5]
#########################################
#kernel関数で、np.sumがどのような挙動を示しているかテスト
print(kernel_test(X[0],centers[1],10))
#########################################

#X:-1.5~1.5
for i in range(len(X)):
    for weight, center in zip(weights, centers):
        Y[i] += weight * kernel(X[i], center, 10)

plt.figure(figsize=(6.472, 4))
plt.plot(X, Y)
plt.show()
```

```python
# カーネル回帰モデル用データセット
# size : number of data
# xlim : domain of variable x
# scale : standard deviation
def wave_dataset(size, xlim=[0, 1], scale=None):
    x = np.random.uniform(xlim[0], xlim[1], size)
    y = np.sin(x)
    if scale is not None:
        noize = np.random.normal(0, scale, size)
        y = y + noize
    df = pd.DataFrame()
    df['x'] = x
    df['y'] = y
    return df

# データを生成
data = wave_dataset(20, xlim=[-3.14, 3.14], scale=0.05)

# データを描画
plt.figure(figsize=(6.472, 4))
plt.scatter(data['x'].as_matrix(), data['y'].as_matrix())
plt.ylim((-1.5, 1.5))
plt.show()
```

```python
from itertools import combinations_with_replacement

#もう一度kernel関数を紹介
#def kernel(xi, xj, beta=1):
#    return np.exp(- beta * np.sum((xi - xj)**2))

X = data['x'].as_matrix()
Y = data['y'].as_matrix()

#TEST Xの可視化
print(X)

# ハイパーパラメータ
beta = 1

# データ数
N = X.shape[0]

#TEST ２つの数字の組み合わせを求める　九九の表のようなもの
for i, j in combinations_with_replacement(range(N), 2):
    print("i:"+str(i)+"| j:"+str(j))
# グラム行列の計算
K = np.zeros((N, N))
for i, j in combinations_with_replacement(range(N), 2):
    K[i][j] = kernel(X[i], X[j])
    K[j][i] = K[i][j]

```

    [ 2.97545323  0.55205916  0.39339059  1.21833953  0.63898912  2.83212807
      2.20957687  0.70218819 -0.0318965  -0.50598084  2.28338585  1.09007326
     -1.9939968  -2.32735348  0.19697489 -2.43727143  1.28229727  2.02736905
     -0.60695192 -0.43282333]
    i:0| j:0
    i:0| j:1
    i:0| j:2
    i:0| j:3
    i:0| j:4
    i:0| j:5
    i:0| j:6
    i:0| j:7
    i:0| j:8
    i:0| j:9
    i:0| j:10
    i:0| j:11
    i:0| j:12
    i:0| j:13
    i:0| j:14
    i:0| j:15
    i:0| j:16
    i:0| j:17
    i:0| j:18
    i:0| j:19
    i:1| j:1
    i:1| j:2
    i:1| j:3
    i:1| j:4
    i:1| j:5
    i:1| j:6
    i:1| j:7
    i:1| j:8
    i:1| j:9
    i:1| j:10
    i:1| j:11
    i:1| j:12
    i:1| j:13
    i:1| j:14
    i:1| j:15
    i:1| j:16
    i:1| j:17
    i:1| j:18
    i:1| j:19
    i:2| j:2
    i:2| j:3
    i:2| j:4
    i:2| j:5
    i:2| j:6
    i:2| j:7
    i:2| j:8
    i:2| j:9
    i:2| j:10
    i:2| j:11
    i:2| j:12
    i:2| j:13
    i:2| j:14
    i:2| j:15
    i:2| j:16
    i:2| j:17
    i:2| j:18
    i:2| j:19
    i:3| j:3
    i:3| j:4
    i:3| j:5
    i:3| j:6
    i:3| j:7
    i:3| j:8
    i:3| j:9
    i:3| j:10
    i:3| j:11
    i:3| j:12
    i:3| j:13
    i:3| j:14
    i:3| j:15
    i:3| j:16
    i:3| j:17
    i:3| j:18
    i:3| j:19
    i:4| j:4
    i:4| j:5
    i:4| j:6
    i:4| j:7
    i:4| j:8
    i:4| j:9
    i:4| j:10
    i:4| j:11
    i:4| j:12
    i:4| j:13
    i:4| j:14
    i:4| j:15
    i:4| j:16
    i:4| j:17
    i:4| j:18
    i:4| j:19
    i:5| j:5
    i:5| j:6
    i:5| j:7
    i:5| j:8
    i:5| j:9
    i:5| j:10
    i:5| j:11
    i:5| j:12
    i:5| j:13
    i:5| j:14
    i:5| j:15
    i:5| j:16
    i:5| j:17
    i:5| j:18
    i:5| j:19
    i:6| j:6
    i:6| j:7
    i:6| j:8
    i:6| j:9
    i:6| j:10
    i:6| j:11
    i:6| j:12
    i:6| j:13
    i:6| j:14
    i:6| j:15
    i:6| j:16
    i:6| j:17
    i:6| j:18
    i:6| j:19
    i:7| j:7
    i:7| j:8
    i:7| j:9
    i:7| j:10
    i:7| j:11
    i:7| j:12
    i:7| j:13
    i:7| j:14
    i:7| j:15
    i:7| j:16
    i:7| j:17
    i:7| j:18
    i:7| j:19
    i:8| j:8
    i:8| j:9
    i:8| j:10
    i:8| j:11
    i:8| j:12
    i:8| j:13
    i:8| j:14
    i:8| j:15
    i:8| j:16
    i:8| j:17
    i:8| j:18
    i:8| j:19
    i:9| j:9
    i:9| j:10
    i:9| j:11
    i:9| j:12
    i:9| j:13
    i:9| j:14
    i:9| j:15
    i:9| j:16
    i:9| j:17
    i:9| j:18
    i:9| j:19
    i:10| j:10
    i:10| j:11
    i:10| j:12
    i:10| j:13
    i:10| j:14
    i:10| j:15
    i:10| j:16
    i:10| j:17
    i:10| j:18
    i:10| j:19
    i:11| j:11
    i:11| j:12
    i:11| j:13
    i:11| j:14
    i:11| j:15
    i:11| j:16
    i:11| j:17
    i:11| j:18
    i:11| j:19
    i:12| j:12
    i:12| j:13
    i:12| j:14
    i:12| j:15
    i:12| j:16
    i:12| j:17
    i:12| j:18
    i:12| j:19
    i:13| j:13
    i:13| j:14
    i:13| j:15
    i:13| j:16
    i:13| j:17
    i:13| j:18
    i:13| j:19
    i:14| j:14
    i:14| j:15
    i:14| j:16
    i:14| j:17
    i:14| j:18
    i:14| j:19
    i:15| j:15
    i:15| j:16
    i:15| j:17
    i:15| j:18
    i:15| j:19
    i:16| j:16
    i:16| j:17
    i:16| j:18
    i:16| j:19
    i:17| j:17
    i:17| j:18
    i:17| j:19
    i:18| j:18
    i:18| j:19
    i:19| j:19

- Kはグラム行列と呼ばれる。
$$
{\hat{y} =
\left(\begin{matrix}
k({\bf x}^{(1)}, {\bf x}) & k({\bf x}^{(2)}, {\bf x}) & \cdots & k({\bf x}^{(N)}, {\bf x})
\end{matrix}\right)
\left(\begin{matrix}
\alpha_0 \\
\alpha_1 \\
\vdots \\
\alpha_N
\end{matrix}\right)
}
$$

$$
{{\bf y} =
\left(\begin{matrix}
y^{(1)} \\
y^{(2)} \\
\vdots \\
y^{(N)}
\end{matrix}\right)
,\;
K=\left(\begin{matrix}
k({\bf x}^{(1)}, {\bf x}^{(1)}) & k({\bf x}^{(2)}, {\bf x}^{(1)}) & \cdots & k({\bf x}^{(N)}, {\bf x}^{(1)}) \\
k({\bf x}^{(1)}, {\bf x}^{(2)}) & k({\bf x}^{(2)}, {\bf x}^{(2)}) & & \vdots \\
\vdots & & \ddots & \vdots \\
k({\bf x}^{(1)}, {\bf x}^{(N)}) & \cdots & \cdots & k({\bf x}^{(N)}, {\bf x}^{(N)})
\end{matrix}\right)
,\;
{\bf \alpha}
=
\left(\begin{matrix}
\alpha^{(1)} \\
\alpha^{(2)} \\
\vdots \\
\alpha^{(N)}
\end{matrix}\right)
}
$$

$${\begin{align}
R({\bf \alpha}) &= r_{square}({\bf \alpha})= \sum_{i=1}^{N}\left\{y^{(i)}- f({\bf x}^{(i)})\right\}^2
=\sum_{i=1}^{N}\left\{y^{(i)}- \sum_{i=1}^{N} \alpha_i k({\bf x}^{(i)}, {\bf x})\right\}^2 \\
&=({\bf y}- K {\bf \alpha})^{\mathrm{T}}({\bf y}- K {\bf \alpha})
\end{align}
}$$

$$
{{\bf \alpha} = (K^{\mathrm{T}}K)^{-1}K^{\mathrm{T}}{\bf y} = K^{-1}{\bf y}
}
$$

```python
alpha = np.linalg.inv(K).dot(Y)
```

```python
# カーネル回帰
def kernel_predict(X, x, alpha, beta):
    Y = 0
    for i in range(len(X)):
        Y += alpha[i] * kernel(X[i], x, beta)
    return Y

# 回帰によって結果を予測
X_axis = np.arange(-3.14, 3.14, 0.01)#0.01刻みで-3.14から3.14までの値を格納
Y_predict = np.zeros(len(X_axis))#ゼロで　Y_predictを初期化
for i in range(len(X_axis)):
    Y_predict[i] = kernel_predict(X, X_axis[i], alpha, beta)

# 結果を描画
plt.figure(figsize=(6.472, 4))

## 観測データ
plt.scatter(data['x'].as_matrix(), data['y'].as_matrix())
## 真の関数
plt.plot(X_axis, np.sin(X_axis), c='red')
## 予測した関数
plt.plot(X_axis, Y_predict)

plt.ylim((-1.5, 1.5))
plt.show()
```

## 正則化して、重みαがあまりにも大きな値にならないようにする

$${\begin{align}
R({\bf \alpha})
&=({\bf y}- K {\bf \alpha})^{\mathrm{T}}({\bf y}- K {\bf \alpha}) + \lambda {\bf \alpha}^{\mathrm{T}} K {\bf \alpha}
\end{align}
}$$

$${\bf \alpha}^{\mathrm{T}} K {\bf \alpha}は、学習器の予測値に、重み\alphaを掛けたもの$$
λの値は、任意に決められるが、λの値が大きいほどαの値が小さくなるように学習しなくてはならない

$${{\bf \alpha} = (K+\lambda I_N)^{-1}{\bf y}
}$$

```python
# 正則化項の係数
lam = 0.5
alpha_r = np.linalg.inv(K + lam * np.eye(K.shape[0])).dot(Y)

# 回帰によって結果を予測
Y_predict_r = np.zeros(len(X_axis))
for i in range(len(X_axis)):
    Y_predict_r[i] = kernel_predict(X, X_axis[i], alpha_r, beta)

# 結果を描画
plt.figure(figsize=(6.472, 4))

## 観測データ
plt.scatter(data['x'].as_matrix(), data['y'].as_matrix())
## 真の関数
plt.plot(X_axis, np.sin(X_axis), c='red')
## 予測した関数
plt.plot(X_axis, Y_predict_r)

plt.ylim((-1.5, 1.5))
plt.show()
```

## 参考文献
- 線形な手法とカーネル法（1. 回帰分析）(Qiita) https://qiita.com/wsuzume/items/09a59036c8944fd563ff　参照日：2018/9/23
- 線形回帰の Normal Equation（正規方程式）について(Qiita) https://qiita.com/antimon2/items/ac1ebaed75ad58406b94　参照日：2018/9/25
- 様々な種類の最小二乗問題の最適解 (Qiita)https://qiita.com/Bikkle/items/df94c641c9e13a094849　参照日：2018/10/13
- 正則化最小二乗法 (regularized least squares method) http://www.sanko-shoko.net/note.php?id=z7xb 参照日：2018/10/13

```python

```


# BP网络/全连接神经网络

BP网络为前向映射网络，网络由输入层、隐藏层以及输出层组成。

## 基本单元

BP网络的基本单元为含有激活函数的单个神经元

$$o = f(x) = f(\omega \cdot \textbf{x} + b) \\ \omega = [\omega_1, \omega_2, \cdots, \omega_n]^T \\ x = [x_1, x_2, \cdots, x_n]^T$$

其中，$o$为输出，$\omega$为权重向量， $\textbf{x}$ 为输入向量，$b$ 为偏执, $f$为激活函数

根据不同的任务类型，可以选用不同的激活函数

## 基本架构

通过多个基本单元组成一层，然后通过多层组成整个BP网络

## BP网络的学习算法

BP网络学习算法由两部分组成
* 前向计算
* 误差反向传播

其中，前向传播过程是网络的应用过程；误差反向传播过程是BP网络权值学习和训练过程。

**核心思想**：将输出误差以**某种形式**从输出层传播到隐藏层最后到输入层。因此，对于BP网络学习算法，最为重要的就是决定误差传播方式

### 符号运算实现误差反向传播

符号运算实现的误差反向传播方式要求首先对梯度进行符号运算并表示成已知量的运算，即**显式**的对每一个神经元的梯度运算进行推导。然后通过带入相应的量求得梯度，实现误差的传播和参数更新。

### 自动微分实现误差反向传播

自动微分实现的误差反向传播本质上是链式法则以及递归思想的综合。其不要求给出**显式**的每一个神经元的梯度计算符号表达式，而是根据链式法则以及递归，以深度优先搜索的方式对整个计算图中的待优化参数进行梯度计算。

自动微分实现误差反向传播要求事先给出每一个运算的微分计算方式，即每一个运算不仅要给出其运算结果还需要给出其对应的微分计算方式。**对于不可导的位置，需要手动给出对应的微分结果**，例如在ReLU激活函数中，0位置不可导，因此需要手动赋予该处的导数为0.

## 三层BP网络误差传递的符号推导

### 定义
定义一个三层BP网络，即输入层、一层隐藏层以及输出层

参数定义如下：

* 输入向量
$$\begin{equation} \textbf{x} = (x_1, x_2, \cdots, x_n)^T, \textbf{x}\in\mathcal{R}^n \end{equation}$$
* 输入层到隐藏层权重
$$\begin{equation} \textbf{V}=\left[\begin{array}{cc} V_{11}&V_{12}& \cdots&V_{1n} \\ \vdots&\vdots&\ddots&\vdots \\ V_{m1}&V_{m2}&\cdots&V_{mn} \end{array}\right], \textbf{V}\in\mathcal{R}^{m\times n} \end{equation}$$
* 隐藏层激活函数
$$\begin{equation} f_1(\textbf{V}\textbf{x}) \end{equation}$$
* 隐藏层输出
$$\begin{equation} \textbf{h} = (h_1, h_2, \cdots, h_m)^T, \textbf{h}\in\mathcal{R}^m \end{equation}$$
* 隐藏层到输出权重
$$\begin{equation} \textbf{W} = \left[\begin{array}{cc} W_{11}&W_{12}& \cdots&W_{1m} \\ \vdots&\vdots&\ddots&\vdots \\ W_{k1}&W_{k2}&\cdots&W_{km} \end{array}\right], \textbf{W}\in\mathcal{R}^{k\times m} \end{equation}$$
* 输出层激活函数
$$\begin{equation} f_2(\textbf{\textbf{W}\textbf{h}}) \end{equation}$$
* 输出层输出
$$\begin{equation} \textbf{o} = (o_1, o_2, \cdots, o_k)^T, \textbf{o}\in\mathcal{R}^k \end{equation}$$
* 期望输出
$$\begin{equation} \textbf{y} = (y_1, y_2, \cdots, y_k)^T, \textbf{o}\in\mathcal{R}^k \end{equation}$$

### 正向运算过程

* 给定输入向量
$$\begin{equation} x_i = (x_{i1}, x_{i2}, \cdots, x_{in})^T \end{equation}$$
* 计算隐藏层输出
$$\begin{equation} \begin{split} \textbf{h}_i &= (h_{i1}, h_{i2}, \cdots, h_{im})^T \\ &= f_1(\sum_{i=0}^nv_{1i}x_i, \sum_{i=0}^nv_{2i}x_i, \cdots, \sum_{i=0}^nv_{mi}x_i) \\ &= f_1(\textbf{v}\textbf{x}) \end{split} \end{equation}$$
* 计算预测输出
$$\begin{equation} \begin{split} \textbf{o}_i &= (o_{i1}, o_{i2}, \cdots, o_{ik})^T \\ &= f_2(\sum_{i=0}^mW_{1i}h_i, \sum_{i=0}^mW_{2i}h_i, \cdots, \sum_{i=0}^mW_{ki}h_i) \\ &= f_2(\textbf{W}\textbf{h}) \end{split} \end{equation}$$
* 计算损失
    * 单个输入输出的损失
$$\begin{equation} \mathcal{L}_i(\textbf{V}, \textbf{W}) = ||\textbf{o}_i-\textbf{y}_i||_2^2\end{equation}$$
    * 总损失
$$\begin{equation} \mathcal{L}(\textbf{V}, \textbf{W}) = \sum_{i=0}^t||\textbf{o}_i-\textbf{y}_i||_2^2\end{equation}$$
* 优化问题
$$\begin{equation} \min\limits_{\textbf{V}, \textbf{W}}\mathcal{L}(\textbf{V}, \textbf{W}) = \sum_{i=0}^t||\textbf{o}_i-\textbf{y}_i||_2^2 \end{equation}$$

### 误差反向传播过程（单样本输入）

* 误差传递到输出层
$$\begin{equation}\begin{split}
\frac{\partial\mathcal{L}(\textbf{V}, \textbf{W})}{\partial{\textbf{o}}}
&= \frac{\partial ||\textbf{o}-\textbf{y}||_2^2}{\partial{\textbf{o}}} \\
&= \frac{\partial{\sum_{j=1}^k(o_j-y_j)^2}}{\partial{\textbf{o}}}\\
&= [\frac{\partial{\sum_{j=1}^k(o_j-y_j)^2}}{\partial{o_1}}, \frac{\partial{\sum_{j=1}^k(o_j-y_j)^2}}{\partial{o_2}}, \cdots, \frac{\partial{\sum_{j=1}^k(o_j-y_j)^2}}{\partial{o_k}}]\\
&= [2(o_1-y_1), 2(o_2-y_2), \cdots, 2(o_k-y_k)] \\
&= 2(\textbf{o}-\textbf{y})^T\\
\end{split}\end{equation}$$
* 误差传递到隐藏层
    * 对于隐藏层到输出层的权重有
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}}
    &= \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{o}_i} \frac{\partial{\textbf{o}_i}}{\partial{\textbf{W}\textbf{h}}} \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{W}}}
\end{split}
\end{equation}
$$
$$
\begin{equation}
    \frac{\partial{\textbf{o}}}{\partial{\textbf{W}\textbf{h}}} = \frac{\partial{f_2(\textbf{W}\textbf{h})}}{\partial{\textbf{W}\textbf{h}}} = diag(f_2^{\prime}(\textbf{W}\textbf{h}))
\end{equation}
$$
由式-15、式-16以及式-17可得
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}\textbf{h}}
    &= 2(\textbf{o}-\textbf{y})^Tdiag(f_2^{\prime}(\textbf{W}\textbf{h}))\\
    &= \boldsymbol{\delta_1} \\
    &= [\delta_{11}, \delta_{12}, \cdots, \delta_{1k}]
    \end{split}
\end{equation}
$$
考虑到$\textbf{W}\textbf{h}$为向量，$\textbf{W}$为矩阵，其直接微分会得到一个$k\times k \times m$的张量，这样不便于操作。考虑使用分量表示，即将$\textbf{W}\textbf{h}$表示为k个分量，分别操作。
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}}
    &= \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}\textbf{h}} \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{W}}} \\
    &= \boldsymbol\delta_1 \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{W}}}\\
    &= \sum_{j=1}^k\delta_{1j}\frac{\partial{\sum_{i=1}^mW_{ji}h_i}}{\partial{\textbf{W}}}\\
    &= \boldsymbol{\delta_1}^T\textbf{h}^T \\
    \end{split}
\end{equation}
$$
    * 对于隐藏层的输入有
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V}, \textbf{W})}}{\partial{\textbf{h}}}
    &= \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}\textbf{h}} \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{h}}}\\
    &= \boldsymbol\delta_1 \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{h}}}\\
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{h}}}
    &= \frac{\partial{((\sum_{i=1}^mW_{1i}h_i, \sum_{i=1}^mW_{2i}h_i, \cdots, \sum_{i=1}^mW_{ki}h_i)^T)}}{\partial{((h_1, h_2, \cdots, h_m)^T)}}\\
    &=
        \left[\begin{array}{cc}
        W_{11}&W_{12}&\cdots&W_{1m}\\
        \vdots&\ddots&\ddots&\vdots\\
        W_{k1}&W_{k2}&\cdots&W_{km}
        \end{array}\right]\\
    &= \textbf{W}
    \end{split}
\end{equation}
$$
由式-20和式-21可得
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V}, \textbf{W})}}{\partial{\textbf{h}}}
    &= \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{W}\textbf{h}} \frac{\partial{\textbf{W}\textbf{h}}}{\partial{\textbf{h}}}\\
    &= \boldsymbol\delta_1 \textbf{W}\\
    \end{split}
\end{equation}
$$
* 误差传递到输入层
    * 对于输入层到隐藏层的权重有
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V}, \textbf{W})}}{\partial{\textbf{V}}}
    &= \frac{\partial{\mathcal{L}(\textbf{V}, \textbf{W})}}{\partial{\textbf{h}}} \frac{\partial{\textbf{h}}}{\partial\textbf{V}\textbf{x}} \frac{\partial{\textbf{V}\textbf{x}}}{\partial\textbf{V}}
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\textbf{h}}}{\partial\textbf{V}\textbf{x}}
    &= \frac{\partial{f_1(\textbf{V}\textbf{x})}}{\partial{\textbf{V}\textbf{x}}} = diag(f_1^{\prime}(\textbf{V}\textbf{x}))
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V}, \textbf{W})}}{\partial{\textbf{h}}} \frac{\partial{\textbf{h}}}{\partial{\textbf{V}\textbf{x}}}
    &= \boldsymbol{\delta_1}\textbf{W}diag(f_1^{\prime}(\textbf{V}\textbf{x}))\\
    &= \boldsymbol{\delta_2}\\
    &= [\delta_{21}, \delta_{22}, \cdots, \delta_{2m}]
    \end{split}
\end{equation}
$$
同上分析，考虑到$\textbf{V}\textbf{x}$为向量，$\textbf{V}$为矩阵，其直接微分会得到一个$m\times m \times n$的张量，这样不便于操作。考虑使用分量表示，即将$\textbf{V}\textbf{x}$表示为k个分量，分别操作。
$$
\begin{equation}
    \begin{split}
    \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{V}}
    &= \frac{\partial{\mathcal{L}(\textbf{V},\textbf{W})}}{\partial\textbf{V}\textbf{x}} \frac{\partial{\textbf{V}\textbf{x}}}{\partial{\textbf{V}}} \\
    &= \boldsymbol\delta_2 \frac{\partial{\textbf{V}\textbf{x}}}{\partial{\textbf{V}}}\\
    &= \sum_{j=1}^m\delta_{2j}\frac{\partial{\sum_{i=1}^nV_{ji}x_i}}{\partial{\textbf{V}}}\\
    &= \boldsymbol{\delta_2}^T\textbf{x}^T \\
    \end{split}
\end{equation}
$$
* 综上可得
待优化参数为$\textbf{W}$和$\textbf{V}$
其中：
$$
\begin{equation}
    \begin{split}
    \frac{\partial\mathcal{L}(\textbf{V}, \textbf{W})}{\partial{\textbf{W}}}
    &= \boldsymbol{\delta_1}^T\textbf{h}^T\\
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \frac{\partial\mathcal{L}(\textbf{V}, \textbf{W})}{\partial{\textbf{V}}}
    &= \boldsymbol{\delta_2}^T\textbf{x}^T\\
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \boldsymbol{\delta_1} = 2(\textbf{o}-\textbf{y})^Tdiag(f_2^{\prime}(\textbf{W}\textbf{h}))
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \boldsymbol{\delta_2} = \boldsymbol{\delta_1}\textbf{W}diag(f_1^{\prime}(\textbf{V}\textbf{x}))
    \end{split}
\end{equation}
$$

对于不同的激活函数有不同的导数值
* Sigmoid
$$
\begin{equation}
S(x)=\frac{1}{1+e^{-x}}
\end{equation}
$$
$$
\begin{equation}
S^{\prime}(x)=\frac{e^{-x}}{(1+e^{-x})^2}=S(x)(1-S(x))
\end{equation}
$$
* ReLU
$$
\begin{equation}
R(x) = \left\{ \begin{array}{cc} x,&x>0\\0,&x\leq0 \end{array} \right.
\end{equation}
$$
$$
\begin{equation}
R^{\prime}(x) = \left\{ \begin{array}{cc} 1,&x>0\\0,&x\leq0 \end{array} \right.
\end{equation}
$$
* Tanh
$$
\begin{equation}
T(x) = \tanh{x} = \frac{e^x-e^{-x}}{e^x+e^{-x}}
\end{equation}
$$
$$
\begin{equation}
T^{\prime}(x) = (\tanh{x})^{\prime} = 1 - \tanh^2x
\end{equation}
$$

若激活函数$f_1$和$f_2$均为Sigmoid则有
$$
\begin{equation}
    \begin{split}
    \boldsymbol{\delta_1} = 2(\textbf{o}-\textbf{y})^Tdiag(S(\textbf{W}\textbf{h})\cdot(1-S(\textbf{W}\textbf{h})))
    \end{split}
\end{equation}
$$
$$
\begin{equation}
    \begin{split}
    \boldsymbol{\delta_2} = \boldsymbol{\delta_1}\textbf{W}diag(S(\textbf{V}\textbf{x})\cdot(1-S(\textbf{V}\textbf{x})))
    \end{split}
\end{equation}
$$

基于符号运算的微分计算方法存在推导复杂的问题，尤其是对于层数较多的神经网络。

```python
import numpy as np
import matplotlib.pyplot as plt
from IPython import display
import seaborn as sns
import imageio
import os
import random
from sklearn.datasets import make_blobs, make_moons

# seed
random.seed(1024)
np.random.seed(1024)
%matplotlib inline
```

```python
# 定义三层神经网络

class MyBPNetwork(object):

    def __init__(self, input_dims, hidden_dims, output_dims, lr) -> None:
        super().__init__()

        # 模型权重
        random.seed(1024)
        np.random.seed(1024)
        self.weight_input2hidden = np.random.randn(hidden_dims, input_dims+1)
        self.weight_hidden2output = np.random.randn(output_dims, hidden_dims)

        # 激活函数
        self.activate = self.activate_func
        self.activate_dif = self.activate_dif_func

        # 中间结果
        self.input_data = None
        self.hidden_value = None
        self.hidden_activate_value = None

        # 梯度结果
        self.grad_hidden2output = None
        self.grad_input2hidden = None

        # 学习率
        self.lr = lr

    def forward(self, input_data):
        """
        正向计算
        """

        # 扩展一个维度
        one_array = np.ones(len(input_data)).reshape(-1, 1)
        self.input_data = np.concatenate((input_data, one_array), axis=1)
        # input data : (batch, input_dims+1)

        # 计算隐藏层
        self.hidden_value = np.matmul(self.weight_input2hidden, self.input_data.T)
        self.hidden_activate_value = self.activate_func(self.hidden_value)
        # hidden_value: (hidden_dim, batch)

        # 计算输出
        output_value = np.matmul(self.weight_hidden2output, self.hidden_activate_value)
        # output_value: (output_dim, batch)

        return output_value.T

    def backward(self, label, pred):
        """
        梯度反向传播
        """

        # label: (batch, output_dim)
        # pred: (batch, output_dim)

        theta_1 = 2 * np.matmul(pred - label, np.eye(pred.shape[1]))
        # theta_1: (batch, output_dim)
        theta_2 = np.zeros((theta_1.shape[0], self.weight_hidden2output.shape[1]))
        for i in range(theta_1.shape[0]):
            theta_2[i, :] = np.matmul(np.matmul(theta_1[i:i+1, :], self.weight_hidden2output),
                                      np.diag(self.activate_dif_func(self.hidden_activate_value[:, i]))).reshape(-1)

        # theta_2: (batch, hidden_dim)

        # 计算隐藏层到输出层权重的梯度
        self.grad_hidden2output = np.matmul(theta_1.T, self.hidden_value.T)
        # 计算输入层到隐藏层权重的梯度
        self.grad_input2hidden = np.matmul(theta_2.T, self.input_data)

    def optimize(self):
        """
        优化
        """
        self.weight_input2hidden -= self.grad_input2hidden * self.lr
        self.weight_hidden2output -= self.grad_hidden2output * self.lr

    @staticmethod
    def activate_func(x):
        """
        sigmoid
        """
        return 1/(1+np.e ** (-x))

    @staticmethod
    def activate_dif_func(activate_result):
        return activate_result*(1-activate_result)
```

```python
# 测试数据

batch_size = 5
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 1e-3

test_input_array = np.ones((batch_size, input_dims))
test_output_array = np.random.randint(0, 2, (batch_size, output_dims))
test_model = MyBPNetwork(input_dims=input_dims,
                         hidden_dims=hidden_num,
                         output_dims=output_dims,
                         lr=lr)

# 正向运算
test_pred = test_model.forward(test_input_array)
# 反向传播
test_model.backward(test_output_array, test_pred)
print("before optimize")
print("input to hidden weight")
print(test_model.weight_input2hidden)
print("hidden to output weight")
print(test_model.weight_hidden2output)
# 优化更新
test_model.optimize()
print("after optimize")
print("input to hidden weight")
print(test_model.weight_input2hidden)
print("hidden to output weight")
print(test_model.weight_hidden2output)
```

    before optimize
    input to hidden weight
    [[ 2.12444863  0.25264613  1.45417876]
     [ 0.56923979  0.45822365 -0.80933344]
     [ 0.86407349  0.20170137 -1.87529904]
     [-0.56850693 -0.06510141  0.80681666]
     [-0.5778176   0.57306064 -0.33667496]]
    hidden to output weight
    [[ 0.29700734 -0.37480416  0.15510474  0.70485719  0.8452178 ]]
    after optimize
    input to hidden weight
    [[ 2.12444463  0.25264213  1.45417477]
     [ 0.56929974  0.4582836  -0.80927349]
     [ 0.86405209  0.20167997 -1.87532044]
     [-0.56862017 -0.06521465  0.80670342]
     [-0.5779505   0.57292774 -0.33680786]]
    hidden to output weight
    [[ 0.29452679 -0.37494538  0.15562887  0.70474505  0.84543886]]

```python
# 数据
class DataCreator(object):

    def __init__(self) -> None:
        super().__init__()

    @staticmethod
    def get_and_data():
        x_list = [[0, 0], [0, 1], [1, 0], [1, 1]]
        y_list = [-1, -1, -1, 1]
        return x_list, y_list

    @staticmethod
    def get_or_data():
        x_list = [[0, 0], [0, 1], [1, 0], [1, 1]]
        y_list = [-1, 1, 1, 1]
        return x_list, y_list

    @staticmethod
    def get_nor_data(nor_index):
        x_list = [[0, 0], [0, 1], [1, 0], [1, 1]]
        y_list = list()
        for i in range(len(x_list)):
            if x_list[i][nor_index]:
                y_list.append(-1)
            else:
                y_list.append(1)

        return x_list, y_list

    @staticmethod
    def get_xor_data():
        x_list = [[0, 0], [0, 1], [1, 0], [1, 1]]
        y_list = [-1, 1, 1, -1]
        return x_list, y_list

    @staticmethod
    def get_moon_data(sample_num, noise=0.05):
        x_list, y_list = make_moons(sample_num, noise=noise, random_state=1024)
        y_list = y_list * 2 - 1
        return x_list, y_list

    @staticmethod
    def get_blob_data(sample_num, std=0.3):
        x_list, y_list = make_blobs(n_samples=sample_num, centers=2,
                                    cluster_std=std, random_state=1024, center_box=(-2, 2))
        y_list = y_list * 2 - 1
        return x_list, y_list

# 数据集
class DataSet(object):

    def __init__(self, input_x_list, output_y_list) -> None:
        super().__init__()

        self.input_x = input_x_list
        self.output_y = output_y_list

    def __len__(self):
        return len(self.input_x)

    def __getitem__(self, item):
        input_x = self.input_x[item]
        output_y = np.array(self.output_y[item]).reshape((1, -1))

        return {"input": input_x, "output": output_y}

    @staticmethod
    def collate_fn(batch):
        input_x = np.array([s["input"] for s in batch])
        output_y = np.array([s["output"][0] for s in batch])
        return {"inputs": input_x, "label": output_y}

# 数据加载器迭代器
class DataLoaderIter(object):

    def __init__(self, data_set, iter_index) -> None:
        super().__init__()
        self.data_set = data_set
        self.iter_index = iter_index
        self.counter = 0
        self.max_iter = len(self.iter_index)

    def __iter__(self):
        return self

    def __next__(self):
        if self.counter == self.max_iter:
            raise StopIteration
        else:
            index_list = self.iter_index[self.counter]
            batch_data = list()

            # 获取数据
            for ind in index_list:
                batch_data.append(self.data_set.__getitem__(ind))
            # 整合数据
            return_data = self.data_set.collate_fn(batch_data)
            self.counter += 1
            return return_data

# 数据加载器
class DataLoader(object):

    def __init__(self, dataset, batch_size, shuffle) -> None:
        super().__init__()

        self.dataset = dataset
        self.batch_size = batch_size
        self.shuffle = shuffle

        # 迭代器
        # 索引分段
        split_index = np.arange(0, len(self.dataset), self.batch_size)[1:]
        # 迭代列表
        if self.shuffle:
            iter_num = np.random.permutation(len(self.dataset))
        else:
            iter_num = np.arange(0, len(self.dataset), 1)

        # 获取迭代索引列表
        self.iter_index_list = np.hsplit(iter_num, split_index)

    def __iter__(self):
        # 定义迭代器
        self.DataIter = DataLoaderIter(self.dataset, self.iter_index_list)
        return self.DataIter

    def __len__(self):
        # 最大迭代次数
        return len(self.iter_index_list)
```

```python
def get_split_face_point(input_range, model, thred=0.01):
    assert np.shape(input_range) == (2, 2)

    x_1_list = np.linspace(input_range[0][0], input_range[0][1], 500)
    x_2_list = np.linspace(input_range[1][0], input_range[1][1], 500)

    pred_matrix = np.zeros((500, 500))

    for i in range(500):
        x_2_input = np.ones_like(x_1_list) * x_2_list[i]
        x_input = np.concatenate((x_1_list.reshape(-1, 1),
                                  x_2_input.reshape(-1, 1)), axis=1)

        pred_matrix[i, :] = model.forward(x_input).reshape(-1)

    mask_1 = (pred_matrix < thred).astype(np.int32)
    mask_2 = (pred_matrix > -thred).astype(np.int32)

    mask_total = np.multiply(mask_1, mask_2)

    point_list = np.argwhere(mask_total == 1)

    point_list = point_list[np.argsort(x_1_list[point_list[:, 1]])]

    return [x_1_list[point_list[:, 1]], x_2_list[point_list[:, 0]]]

def draw(input_x_list, output_y_list, loss_list, split_points_list, count):
    input_x_list = np.array(input_x_list.copy())
    input_dim = np.shape(input_x_list)[1]
    if np.shape(input_x_list)[1] != 2:
        raise ValueError("仅能绘制输入维度为2的结果，但是接受到的维度为{}".format(input_dim))

     # 设置标记点
    input_x_1_range = [np.min(input_x_list[:, 0]), np.max(input_x_list[:, 0])]
    input_x_2_range = [np.min(input_x_list[:, 1]), np.max(input_x_list[:, 1])]
    # 区间长度
    x_1_len = input_x_1_range[1] - input_x_1_range[0]
    x_2_len = input_x_2_range[1] - input_x_2_range[0]

    # 绘制所有训练数据
    plt.cla()
    plt.clf()
    plt.subplot(1, 2, 1)
    plt.scatter(input_x_list[:, 0], input_x_list[:, 1], c=output_y_list)
    plt.scatter(split_points_list[0], split_points_list[1], color="r", s=0.1)

    plt.xlim(input_x_1_range[0]-0.2*x_1_len, input_x_1_range[1]+0.2*x_1_len)
    plt.ylim(input_x_2_range[0]-0.2*x_2_len, input_x_2_range[1]+0.2*x_2_len)
    plt.xlabel("x1")
    plt.ylabel("x2")

    # 绘制损失曲线
    # 绘制训练损失曲线
    plt.subplot(1, 2, 2)
    plt.plot(np.arange(0, len(loss_list), 1), loss_list)
    plt.xlabel("Epochs")
    plt.ylabel("Train loss")
    plt.title("Train loss - epochs")

    plt.savefig("./images/temp/{}.png".format(count), dpi=150)

def create_gif(epochs, output_name, fps=2, split=1):
    """
    生成gif
    """
    with imageio.get_writer("./images/{}.gif".format(output_name), mode="I", fps=fps) as Writer:
        for ind in range(epochs):
            if ind % split == 0:
                image = imageio.imread("./images/temp/{}.png".format(ind))
                os.remove("./images/temp/{}.png".format(ind))
                Writer.append_data(image)

```

### and逻辑

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.1

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_and_data()

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-0.2, 1.2], [-0.2, 1.2]], BP_model)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "add_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### or逻辑

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.1

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_or_data()

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-0.2, 1.2], [-0.2, 1.2]], BP_model)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "or_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### Nor(x1)逻辑

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.1

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_nor_data(0)

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-0.2, 1.2], [-0.2, 1.2]], BP_model)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "nor_x1_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### Nor(x2)逻辑

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.1

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_nor_data(1)

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-0.2, 1.2], [-0.2, 1.2]], BP_model)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "nor_x2_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### xoR逻辑

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.1

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_xor_data()

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-0.2, 1.2], [-0.2, 1.2]], BP_model, thred=0.01)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "xor_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### 线性可分样本簇

```python
input_dims = 2
output_dims = 1
hidden_num = 5
lr = 0.05

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 100
# 获取训练数据
train_x_all, train_y_all = data_creator.get_blob_data(100, std=0.3)

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-1, 1.6], [-0.5, 3.2]], BP_model)
    draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)
# 绘制
create_gif(max_epoch, "blobs_BPNetwork", fps=5)
plt.show()
```

#### 训练过程

### 线性不可分样本簇

```python
input_dims = 2
output_dims = 1
hidden_num = 15
lr = 0.005

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)
# 数据生成器
data_creator = DataCreator()
# 最大训练次数
max_epoch = 700
# 获取训练数据
train_x_all, train_y_all = data_creator.get_moon_data(200)

# 训练方式
# 可选 BGD、SGD、MBGD
train_method = "MBGD"
# batch size
# 若为BGD，则batch_size为所有训练数据的总和
# 若为SGD，则batch_size为1
# 若为MBGD，则batch_size为自定义批大小，大于1小于所有训练数据的总和
if train_method == "BGD":
    batch_size = len(train_x_all)
elif train_method == "MBGD":
    batch_size = 2
else:
    batch_size = 1

# 训练集
train_dataset = DataSet(train_x_all, train_y_all)
# 数据加载器
train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True)
train_num = len(train_y_all)
train_loss_list = list()

plt.figure(figsize=(10, 5), dpi=150)
for epoch in range(max_epoch):

    train_loss = 0
    for batch_loop, batch_ in enumerate(train_dataloader):
        # 读取数据
        train_x = batch_["inputs"]
        train_y = batch_["label"]

        # 预测
        pred_ = BP_model.forward(train_x)
        # 反向传播
        BP_model.backward(train_y, pred_)
        # 优化
        BP_model.optimize()

        train_loss += np.sum((pred_ - train_y) ** 2)

    train_loss_list.append(train_loss / len(train_dataset))

    if epoch % 50 == 0:
        print(epoch, train_loss_list[-1])

    if epoch == 600:
        BP_model.lr = 1e-4

    # 获取分离面对应的预测矩阵
    split_points = get_split_face_point([[-1.2, 2.2], [-0.7, 1.2]], BP_model)

    if epoch % 20 == 0:
        draw(train_x_all, train_y_all, train_loss_list, split_points, epoch)

# 绘制
create_gif(max_epoch, "moons_BPNetwork", fps=5, split=20)
plt.show()
```

#### 训练过程

### 非线性回归

```python
# training data
np.random.seed(1024)
random.seed(1024)
Epochs = 600
x = np.sort(np.random.normal(-1, 1.0, (100, 1)), axis=0).reshape(-1, 1)
y = np.exp(x) - 3

plt.figure(figsize=(10, 5), dpi=150)

input_dims = 1
output_dims = 1
hidden_num = 10
lr = 2e-4

BP_model = MyBPNetwork(input_dims=input_dims,
                       output_dims=output_dims,
                       hidden_dims=hidden_num,
                       lr=lr)

loss_list = []
for e in range(1, Epochs+1):

    # 预测
    pred_ = BP_model.forward(x)
    # 反向传播
    BP_model.backward(y, pred_)
    # 优化
    BP_model.optimize()

    loss = np.mean((pred_ - y) ** 2)

    loss_list.append(loss)
    if e % 40 == 0:
        print("epoch-%i \tloss: %.4f" % (e, loss))

    # 记录图片
    if e % 10 == 0:
        plt.cla()
        plt.clf()
        plt.subplot(1, 2, 1)
        plt.ylim(-4, 3)
        plt.scatter(x.reshape(-1), y.reshape(-1), c="b", label="data")
        plt.plot(x.reshape(-1), pred_.reshape(-1), c="r", label="regression")
        plt.legend()
        plt.subplot(1, 2, 2)
        plt.plot(loss_list)
        plt.xlabel("Epochs")
        plt.ylabel("loss")
        plt.savefig("./images/temp/{}.png".format(e), dpi=150)

with imageio.get_writer("./images/{}.gif".format("nonlinear_regression_BPNetwork"), mode="I", fps=2) as Writer:
    for ind in range(1, Epochs+1):
        if ind % 10 == 0:
            image = imageio.imread("./images/temp/{}.png".format(ind))
            os.remove("./images/temp/{}.png".format(ind))
            Writer.append_data(image)

plt.show()
```

#### 训练过程


# Lecture 6, Heat transports

> Two-dimensional advection-diffusion of heat

- toc: False
- badges: true
- comments: False
- categories: [jupyter]

```python
import xarray as xr
import numpy as np
import matplotlib.pyplot as plt

from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets

from IPython.display import HTML
from IPython.display import display
```

# 1) Background: two-dimensional advection-diffusion

## 1.1) The two-dimensional advection-diffusion equation
Recall from **Lecture 5** that the one-dimensional advection-diffusion equation is written as

$$\frac{\partial T(x,t)}{\partial t} = -U \frac{\partial T}{\partial x} + \kappa \frac{\partial^{2} T}{\partial x^{2}},$$

where $T(x, t)$ is the temperature, $U$ is a constant advective velocity and $\kappa$ is the diffusivity.

The two-dimensional advection diffusion equation simply adds advection and diffusion operators acting in a second dimensions $y$ (orthogonal to $x$).

$$\frac{\partial T(x,y,t)}{\partial t} = u(x,y) \frac{\partial T}{\partial x} + v(x,y) \frac{\partial T}{\partial y} + \kappa \left( \frac{\partial^{2} T}{\partial x^{2}} + \frac{\partial^{2} T}{\partial y^{2}} \right),$$

where $\vec{u}(x,y) = (u, v) = u\,\mathbf{\hat{x}} + v\,\mathbf{\hat{y}}$ is a velocity vector field.

Throughout the rest of the Climate Modelling module, we will consider $x$ to be the *longitundinal* direction (positive from west to east) and $y$ to the be the *latitudinal* direction (positive from south to north).

## 1.2) Multivariable shorthand notation

Conventionally, the two-dimensional advection-diffusion equation is written more succintly as

$$\frac{\partial T(x,y,t)}{\partial t} = - \vec{u} \cdot \nabla T + \kappa \nabla^{2} T,$$

using the following shorthand notation.

The **gradient** operator is defined as

$\nabla \equiv (\frac{\partial}{\partial x}, \frac{\partial }{\partial y})$

such that

$\nabla T = (\frac{\partial T}{\partial x}, \frac{\partial T}{\partial y})$ and

$\vec{u} \cdot \nabla T = (u, v) \cdot (\frac{\partial T}{\partial x}, \frac{\partial T}{\partial y}) = u \frac{\partial T}{\partial x} + v\frac{\partial T}{\partial y}.$

The **Laplacian** operator $\nabla^{2}$ (sometimes denoted $\Delta$) is defined as

$\nabla^{2} = \frac{\partial^{2}}{\partial x^{2}} + \frac{\partial^{2}}{\partial y^{2}}$

such that

$\nabla^{2} T = \frac{\partial^{2} T}{\partial x^{2}} + \frac{\partial^{2} T}{\partial y^{2}}.$

The **divergence** operator is defined as $\nabla \cdot [\quad]$, such that

$\nabla \cdot \vec{u} = \left(\frac{\partial}{\partial x}, \frac{\partial}{\partial x} \right) \cdot (u,v) = \frac{\partial u}{\partial x} + \frac{\partial v}{\partial y}.$

**Note:** Since seawater is largely incompressible, we can approximate ocean currents as a *non-divergent flow*, with $\nabla \cdot \vec{u} = \frac{\partial u}{\partial x} + \frac{\partial v}{\partial y} = 0$. Among other implications, this allows us to write:

\begin{align}
\vec{u} \cdot \nabla T&=
u\frac{\partial T(x,y,t)}{\partial x} + v\frac{\partial T(x,y,t)}{\partial y}\newline &=
u\frac{\partial T}{\partial x} + v\frac{\partial T}{\partial y} + T\left(\frac{\partial u}{\partial x} + \frac{\partial v}{\partial y}\right)\newline &=
\left( u\frac{\partial T}{\partial x} + T\frac{\partial u}{\partial x} \right) +
\left( v\frac{\partial T}{\partial y} + \frac{\partial v}{\partial y} \right)
\newline &=
\frac{\partial (uT)}{\partial x} + \frac{\partial (vT)}{\partial x}\newline &=
\nabla \cdot (\vec{u}T)
\end{align}

using the product rule (separately in both $x$ and $y$).

##### 1.3) The flux-form two-dimensional advection-diffusion equation

This lets us finally re-write the two-dimensional advection-diffusion equation as:

$\frac{\partial T}{\partial t} = - \nabla \cdot (\vec{u}T) + \kappa \nabla^{2} T$

which is the form we will use in our numerical algorithm below.

# 2) Numerical implementation

# 2.1) Discretizing advection in two dimensions

In Lecture XX we saw that in one dimension we can discretize a first-partial derivative in space using the *centered finite difference*

$$\frac{\partial T(x_{i}, t_{n})}{\partial x} \approx \frac{T_{i+1}^{n} - T_{i-1}^{n}}{2 \Delta x}.$$

In two dimensions, we discretize the partial derivative the exact same way, except that we also need to keep track of the cell index $j$ in the $y$ dimension:

$$\frac{\partial T(x_{i}, y_{j}, t_{n})}{\partial x} \approx \frac{T_{i+1,\, j}^{n} - T_{i-1,\,j}^{n}}{2 \Delta x}.$$

The *x-gradient kernel* below, is shown below, and is reminiscent of the *edge-detection* or *sharpening* kernels used in image processing and machine learning:

```python
plt.imshow([[-1,0,1]], cmap=plt.cm.RdBu_r)
plt.xticks([0,1,2], labels=[-1,0,1]);
plt.yticks([]);
```

The first-order partial derivative in $y$ is similarly discretized as:

$$\frac{\partial T(x_{i}, y_{j}, t_{n})}{\partial y} \approx \frac{T_{i,\, j+1}^{n} - T_{i,\,j-1}^{n}}{2 \Delta y}.$$

Its kernel is shown below.

```python
plt.imshow([[-1,-1],[0,0],[1,1]], cmap=plt.cm.RdBu);
plt.xticks([]);
plt.yticks([0,1,2], labels=[1,0,-1]);
```

Now that we have discretized the two derivate terms, we can write out the *advective tendency* for computing $T_{i, j, n+1}$ as

$$u\frac{\partial T}{\partial x} + v\frac{\partial T}{\partial y} \approx u_{i,\, j}^{n} \frac{T_{i,\, j+1}^{n} - T_{i,\,j-1}^{n}}{2 \Delta y} + v_{i,\, j}^{n} \frac{T_{i,\, j+1}^{n} - T_{i,\,j-1}^{n}}{2 \Delta y}.$$

We implement this as a the `advect` function. This computes the advective tendency for the $(i,j)$ grid cell.

```python
xkernel = np.zeros((1,3))
xkernel[0, :] = np.array([-1,0,1])
```

```python
ykernel = np.zeros((3,1))
ykernel[:, 0] = np.array([-1,0,1])
```

```python
class OceanModel:

    def __init_(self,T, u, v, deltax, deltay):
        self.T = [T]
        self.u = u
        self.v = v
        self.deltax = deltax
        self.deltay = deltay
        self.j = np.arange(0, self.T[0].shape[0])
        self.i = np.arange(0, self.T[0].shape[1])

    def advect(self, T):
        T_advect = np.zeros((self.j.size, self.i.size))

        for j in range(1, T_advect.shape[0] - 1):
            for i in range(1, T_advect.shape[1] - 1):
                T_advect[j, i] = -(self.u[j, i] * np.sum(xkernel[0, :] * T[j, i-1 : i + 2])/(2*self.deltax) +
                                   self.v[j, i] * np.sum(ykernel[:, 0] * T[j-1:j+2, i])/(2*self.deltay))
        return T_advect
```

## 2.2) Discretizing diffusion in two dimensions

Just as with advection, the process for discretizing the diffusion operators effectively consists of repeating the one-dimensional process for $x$ in a second dimension $y$, separately:

$$\kappa \left( \frac{\partial^{2} T}{\partial x^{2}} + \frac{\partial^{2} T}{\partial y^{2}} \right) = \kappa \left(
\frac{T_{i+1,\;j}^{n} - 2T_{i,\;j}^{n} + T_{i-1,\;j}^{n}}{\left( \Delta x \right)^{2}} +
\frac{T_{i,\;j+1}^{n} - 2T_{i,\;j}^{n} + T_{i,\;j-1}^{n}}{\left( \Delta y \right)^{2}}
\right)$$

The corresponding $x$ and $y$ second-derivative kernels are shown below:

```python
plt.imshow([[1,1],[0,0],[1,1]], cmap=plt.cm.RdBu_r);
plt.xticks([]);
plt.yticks([0,1,2], labels=[1,-1,1]);
```

```python
plt.imshow([[1,0,1]], cmap=plt.cm.RdBu_r);
plt.xticks([0,1,2], labels=[1,-1,1]);
plt.yticks([]);
```

```python
xkernel_diff = np.zeros((1, 3))
xkernel_diff[0, :] = np.array([1,-2,1])
```

```python
ykernel_diff = np.zeros((3,1))
ykernel_diff[:, 0] = np.array([1,-2,1])
```

Just as we did with advection, we implement a diffuse function using multiple dispatch:

```python
class OceanModel():

    def __init__(self, T, deltat, windfield, grid, kappa):

        self.T = [T]
        self.u = windfield.u
        self.v = windfield.v
        self.deltay = grid.deltay
        self.deltax = grid.deltax
        self.deltat = deltat
        self.kappa = kappa
        self.i = np.arange(0, self.T[0].shape[1])
        self.j = np.arange(0, self.T[0].shape[0])
        self.t = [0]
        self.diffuse_out = []
        self.advect_out = []
        self.t_ = []

    def advect(self, T):
        T_advect = np.zeros((self.j.size, self.i.size))
        for j in range(1, T_advect.shape[0] - 1):
            for i in range(1, T_advect.shape[1] - 1):
                T_advect[j, i] = -(self.u[j, i] * np.sum(xkernel[0, :] * T[j, i-1:i+2])/(2*self.deltax) + \
                                   self.v[j, i] * np.sum(ykernel[:, 0] * T[j-1:j+2, i])/(2*self.deltay))
        return T_advect

    def diffuse(self, T):
        T_diffuse= np.zeros((self.j.size, self.i.size))
        for j in range(1, T_diffuse.shape[0] - 1):
            for i in range(1, T_diffuse.shape[1] - 1):
                T_diffuse[j, i] = self.kappa*(np.sum(xkernel_diff[0, :] * T[j, i-1:i+2])/(self.deltax**2) + \
                                              np.sum(ykernel_diff[:, 0] * T[j-1:j+2, i])/(self.deltay**2))
        return T_diffuse
```

## 2.3) No-flux boundary conditions

We want to impose the no-flux boundary conditions, which states that

$$u\frac{\partial T}{\partial x} = \kappa \frac{\partial T}{\partial x} = 0$$ at $x$ boundaries and

$$v\frac{\partial T}{\partial y} = \kappa \frac{\partial T}{\partial y} = 0$$ at the $y$ boundaries.

To impose this, we treat $i=1$ and $i=N_{x}$ as *ghost cells*, which do not do anything expect help us impose these boundaries conditions. Discretely, the boundary fluxes between $i=1$ and $i=2$ vanish if

$$\dfrac{T_{2,\,j}^{n} -T_{1,\,j}^{n}}{\Delta x} = 0$$ or

$$T_{1,\,j}^{n} = T_{2,\,j}^{n}.$$

Thus, we can implement the boundary conditions by updating the temperature of the ghost cells to match their interior-point neighbors:

```python
def update_ghostcells(var):
    var[0, :] = var[1, :];
    var[-1, :] = var[-2, :]
    var[:, 0] = var[:, 1]
    var[:, -1] = var[:, -2]
    return var
```

```python
B = np.random.rand(6,6)
plt.pcolor(B)
plt.colorbar()
```

```python
B = update_ghostcells(B)
```

```python
plt.imshow(B)
plt.colorbar()
```

## 2.4) Timestepping

```python
class OceanModel():

    def __init__(self, T, deltat, windfield, grid, kappa):

        self.T = [T]
        self.u = windfield.u
        self.v = windfield.v
        self.deltay = grid.deltay
        self.deltax = grid.deltax
        self.deltat = deltat
        self.kappa = kappa
        self.i = np.arange(0, self.T[0].shape[1])
        self.j = np.arange(0, self.T[0].shape[0])
        self.t = [0]
        self.diffuse_out = []
        self.advect_out = []
        self.t_ = []

    def advect(self, T):
        T_advect = np.zeros((self.j.size, self.i.size))
        for j in range(1, T_advect.shape[0] - 1):
            for i in range(1, T_advect.shape[1] - 1):
                T_advect[j, i] = -(self.u[j, i] * np.sum(xkernel[0, :] * T[j, i-1:i+2])/(2*self.deltax) + \
                                   self.v[j, i] * np.sum(ykernel[:, 0] * T[j-1:j+2, i])/(2*self.deltax))
        return T_advect

    def diffuse(self, T):
        T_diffuse= np.zeros((self.j.size, self.i.size))
        for j in range(1, T_diffuse.shape[0] - 1):
            for i in range(1, T_diffuse.shape[1] - 1):
                T_diffuse[j, i] = self.kappa*(np.sum(xkernel_diff[0, :] * T[j, i-1:i+2])/(self.deltax**2) + \
                                              np.sum(ykernel_diff[:, 0] * T[j-1:j+2, i])/(self.deltay**2))
        return T_diffuse

    def timestep(self):
        T_ = self.T[-1].copy()
        T_ = update_ghostcells(T_)
        tendencies = self.advect(self.T[-1]) + self.diffuse(self.T[-1])
        T_ = self.deltat * tendencies + T_
        self.T.append(T_)
        self.t.append(self.t[-1] + self.deltat)
```

```python
class grid:
    """ Creates grid for ocean model
    Args:
    -------

    N: Number of grid points (NxN)
    L: Length of domain
    """
    def __init__(self, N, L):
        self.deltax = L / N
        self.deltay = L / N
        self.xs = np.arange(0-self.deltax/2, L+self.deltax/2, self.deltax)
        self.ys = np.arange(-L-self.deltay/2, L+self.deltay/2, self.deltay)

        self.N = N
        self.L = L
        self.Nx = len(self.xs)
        self.Ny = len(self.ys)
        self.grid, _= np.meshgrid(self.ys, self.xs, sparse=False, indexing='ij')

class windfield:
    """Calculates wind velocity

    Args:
    ------
    Grid: class grid()
    option: zero, or single gyre

    """

    def __init__(self, grid, option = "zero"):
        if option == "zero":
            self.u = np.zeros((grid.grid.shape))
            self.v = np.zeros((grid.grid.shape))
        if option == "single_gyre":
            u, v = PointVortex(my_grid)
            self.u = u
            self.v = v

```

Initialize $10x10$ grid domain with $L=6000000m$ and a windfield, where all velocities are set to zero.
This should result in diffusion only.

Initialize temperature field with rectangular non-zero box.

```python
my_grid = grid(N = 11, L = 6e6)
my_wind = windfield(my_grid, option = "zero")
my_temp = np.zeros((my_grid.grid.shape))
my_temp[9:13, 4:8] = 1
```

```python
my_grid.grid.shape
```

    (23, 12)

```python
plt.pcolor(my_grid.xs, my_grid.ys, my_temp)
plt.colorbar(label = "Temperature in °C");
plt.ylabel("Kilometers Latitude");
plt.xlabel("Kilometers Longitude");
```

```python
deltat = 12*60*60 # 12 Hours
```

```python
def plot_func(steps, temp, deltat, windfield, grid, kappa, anomaly = False):
    my_model = OceanModel(T=temp,
                      deltat=deltat,
                      windfield=windfield,
                      grid=grid,
                      kappa = kappa)
    for step in range(steps):
        my_model.timestep()
    plt.figure(figsize=(8,8))
    lon, lat = np.meshgrid(grid.xs, grid.ys)
    if anomaly == True:
        plt.pcolor(my_grid.xs, my_grid.ys, my_model.T[-1] - my_model.T[-2], cmap = plt.cm.RdBu_r,
                  vmin = -0.001, vmax = 0.001)
    else:
        plt.pcolor(my_grid.xs, my_grid.ys, my_model.T[-1], vmin = 0, vmax = 1)
    plt.quiver(lon, lat, windfield.u, windfield.v)
    plt.colorbar(label = "Temperature in °C");
    plt.ylabel("Kilometers Latitude");
    plt.xlabel("Kilometers Longitude");
    plt.title("Time: {} days".format(int(step/2)))
    plt.show()
```

```python
interact(plot_func,
         temp = fixed(my_temp),
         deltat = fixed(deltat),
         windfield = fixed(my_wind),
         grid = fixed(my_grid),
         kappa = fixed(1e4),
         anomaly = widgets.Checkbox(description = "Anomaly", value=False),
         steps = widgets.IntSlider(description="Timesteps", value = 2, min = 1, max = 10000, step = 1));
```

    interactive(children=(IntSlider(value=2, description='Timesteps', max=10000, min=1), Checkbox(value=False, des…

## Create single gyre windfield

```python
def diagnose_velocities(stream, G):
    dx = np.zeros((G.grid.shape))
    dy = np.zeros((G.grid.shape))

    dx = (stream[:, 1:] - stream[:, 0:-1])/G.deltax
    dy = (stream[1:, :] - stream[0:-1, :])/G.deltay

    u = np.zeros((G.grid.shape))
    v = np.zeros((G.grid.shape))

    u = 0.5*(dy[:, 1:] + dy[:, 0:-1])
    v = -0.5*(dx[1:, :] + dx[0:-1, :])

    return u, v
```

```python
def impose_no_flux(u, v):
    u[0, :]  = 0; v[0, :] = 0;
    u[-1,:]  = 0; v[-1,:] = 0;
    u[:, 0]  = 0; v[:, 0] = 0;
    u[:,-1]  = 0; v[:,-1] = 0;

    return u, v
```

```python
def PointVortex(G, omega = 0.6, a = 0.2, x0=0.5, y0 = 0.):
    x = np.arange(-G.deltax/G.L, 1 + G.deltax / G.L, G.deltax / G.L).reshape(1, -1)
    y = np.arange(-1-G.deltay/G.L, 1+ G.deltay / G.L, G.deltay / G.L).reshape(-1, 1)

#     x = x[:,:-1]

    r = np.sqrt((y - y0)**2 + (x - x0)**2)
    stream = -omega/4*r**2
    stream[r > a] =  -omega*a**2/4*(1. + 2*np.log(r[r > a]/a))

    u, v =  diagnose_velocities(stream, G)
    u, v = impose_no_flux(u, v)
    u = u * G.L; v = v * G.L
    return u, v
```

```python
my_wind = windfield(my_grid, option="single_gyre")
plt.quiver(my_wind.u, my_wind.v)
```

```python
my_temp = np.zeros((my_grid.grid.shape))
my_temp[8:13, :] = 1
```

```python
plt.pcolor(my_temp)
```

```python
my_model = OceanModel(T=my_temp,
                      deltat=deltat,
                      windfield=my_wind,
                      grid=my_grid,
                      kappa = 1e4)
```

```python
interact(plot_func,
         temp = fixed(my_temp),
         deltat = fixed(deltat),
         windfield = fixed(my_wind),
         grid = fixed(my_grid),
         kappa = widgets.IntSlider(description="kappa", value = 1e4, min=1e3, max = 1e5, step = 100),
         anomaly = widgets.Checkbox(description = "Anomaly", value=False),
         steps = widgets.IntSlider(description="Timesteps", value = 1, min = 1, max = 1000, step = 1));
```

    interactive(children=(IntSlider(value=1, description='Timesteps', max=1000, min=1), IntSlider(value=10000, des…

ToDo add two gyre

```python

```

```python

```

```python

```

```python

```

```python
u, v = PointVortex(my_grid)
u, v = u[1:], v[1:]
```

```python
my_grid.grid.shape
```

```python
print(u.shape)
```

```python
epsilon = 0.05
xpsi = np.arange(-deltax, 1+deltax, deltax)
ypsi = np.arange(-1-deltay, 1+deltay, deltay)

def psihat(x, y):
    psi = np.zeros((x.size, y.size))
    np.psi

```

```python
psi = np.zeros((ypsi.size, xpsi.size))

for i, x in enumerate(xpsi):
    for j, y in enumerate(ypsi):
        psi[j, i] = np.pi * np.sin(np.pi * y) * (
        1 - x - np.exp(-x/(2*epsilon)) *
            (np.cos(np.sqrt(3)*x/(2*epsilon)) + (1 / np.sqrt(3)) * np.sin(np.sqrt(3)*x/(2*epsilon)))+epsilon*np.exp((x-1)/epsilon))

```

```python

```

```python
plt.pcolor(psi)
plt.colorbar()
```

```python
np.pi * np.sin(np.pi * y) * (
        1 - x - np.exp(-x/(2*epsilon)) *
            (np.cos(np.sqrt(3)*x/(2*epsilon)) + (1 / np.sqrt(3)) * np.sin(np.sqrt(3)*x/(2*epsilon)))+epsilon*np.exp((x-1)/epsilon))
```


```python
#In this program, we calculate the largest water cluster on Earth in the presence of different levels of water
#Libraries
from scipy.ndimage.filters import gaussian_filter
import matplotlib.pyplot as plt
from netCDF4 import Dataset
import numpy as np
import os
from tqdm import tqdm
from scipy.ndimage import measurements
import time
import sympy
import concurrent.futures
```

```python
#This function gives the different resolution of the earth map. we use it to extract the orginal resolution.
#the code has been written by
def Etopo(lon_area, lat_area, resolution):
    ### Input
    # resolution: resolution of topography for both of longitude and latitude [deg]
    # (Original resolution is 0.0167 deg)
    # lon_area and lat_area: the region of the map which you want like [100, 130], [20, 25]
    ###
    ### Output
    # Mesh type longitude, latitude, and topography data
    ###

    # Read NetCDF data
    data = Dataset("ETOPO1_Bed_c_gdal.grd", "r")
    # Get data
    lon_range = data.variables['x_range'][:]
    lat_range = data.variables['y_range'][:]
    topo_range = data.variables['z_range'][:]
    spacing = data.variables['spacing'][:]
    dimension = data.variables['dimension'][:]
    z = data.variables['z'][:]
    lon_num = dimension[0]
    lat_num = dimension[1]

    # Prepare array
    lon_input = np.zeros(lon_num); lat_input = np.zeros(lat_num)
    for i in range(lon_num):
        lon_input[i] = lon_range[0] + i * spacing[0]
    for i in range(lat_num):
        lat_input[i] = lat_range[0] + i * spacing[1]

    # Create 2D array
    lon, lat = np.meshgrid(lon_input, lat_input)

    # Convert 2D array from 1D array for z value
    topo = np.reshape(z, (lat_num, lon_num))

    # Skip the data for resolution
    if ((resolution < spacing[0]) | (resolution < spacing[1])):
        print('Set the highest resolution')
    else:
        skip = int(resolution/spacing[0])
        lon = lon[::skip,::skip]
        lat = lat[::skip,::skip]
        topo = topo[::skip,::skip]

    topo = topo[::-1]

    # Select the range of map
    range1 = np.where((lon>=lon_area[0]) & (lon<=lon_area[1]))
    lon = lon[range1]; lat = lat[range1]; topo = topo[range1]
    range2 = np.where((lat>=lat_area[0]) & (lat<=lat_area[1]))
    lon = lon[range2]; lat = lat[range2]; topo = topo[range2]

    # Convert 2D again
    lon_num = len(np.unique(lon))
    lat_num = len(np.unique(lat))
    lon = np.reshape(lon, (lat_num, lon_num))
    lat = np.reshape(lat, (lat_num, lon_num))
    topo = np.reshape(topo, (lat_num, lon_num))

    return lon, lat, topo
```

```python
# convert degrees to radians
def degree2radians(degree):
    return degree*np.pi/180
```

```python
#defining the orginal resoluton of the data
resolution = 0.0167
# resolution = 1

lon_area = [-180., 180.]
lat_area = [-90., 90.]
# Get mesh-shape topography data
lon_topo, lat_topo, topo = Etopo(lon_area, lat_area, resolution)
lon_topo = degree2radians(lon_topo)
lat_topo = degree2radians(lat_topo)
```

```python
#The Course graining part. we define the of the course graining window with N and course grain it with thw function rebin
N = 10
def rebin(a, shape):
    sh = shape[0],a.shape[0]//shape[0],shape[1],a.shape[1]//shape[1]
    return a.reshape(sh).mean(-1).mean(1)

Width = int(10800/N)
Length = int(21600/N)
topo = rebin(topo,(Width, Length))
lat_topo = rebin(lat_topo,(Width, Length))
lon_topo = rebin(lon_topo,(Width, Length))
```

```python
#we extract all the elevations and sort then in an ascending order
indexes = np.argsort(topo.flatten())
hieght = np.sort(topo.flatten())
#defining the parallelization variables
l = len(hieght)
d = sympy.divisors(l)
num_iter = d[int(len(d)/2)]
num_proc = d[int(len(d)/2) - 1]
```

```python
#water clusters

total_landmass = np.zeros(len(hieght))
big_cluster = np.zeros(len(hieght))
total_lat = np.sum(np.abs(np.cos(lat_topo)))
all_pos = np.shape(lat_topo)[0]*np.shape(lat_topo)[1]

squares1 = np.zeros((num_proc, num_iter))
squares2 = np.zeros((num_proc, num_iter))

def water_function(ii):
    total_landmass = np.zeros(num_iter)
    big_cluster = np.zeros(num_iter)
    iso_hight = np.zeros(num_iter)
    iterr = 0
    for i in range(ii*num_iter, ii*num_iter + num_iter):
        sphere = 0
        new_topo=np.zeros(hieght.shape, bool)
        new_topo[indexes[:(i+1)]] = True
        new_topo = new_topo.reshape((Width, Length))
        pp = (i+1)/(all_pos)
        lw, num = measurements.label(new_topo)
        for l in range(len(lw)):
            if(lw[l][0] != 0 and lw[l][-1] != 0 and lw[l][-1] != lw[l][0]):
                lw[lw == lw[l][0]] = lw[l][-1]

        biggest_size = 0
        elements = np.unique(lw)
        elements = elements[elements != 0]
        unique_clustersize = np.zeros(len(elements))
        iterat = 0
        for j in elements:
            unique_clustersize[iterat] = np.sum(lw == j)
            iterat +=1

        elements = elements[unique_clustersize.argsort()]

        a = 0
        if len(elements)>2:
            for n in elements[[-1,-2,-3]]:
                mask = (lw == n)
                a = np.sum(np.abs(np.cos((mask*lat_topo)[(mask*lat_topo) != 0])))
                if a>biggest_size:
                    biggest_size = a

        elif len(elements) == 2:
            for n in elements[[-1,-2]]:
                mask = (lw == n)
                a = np.sum(np.abs(np.cos((mask*lat_topo)[(mask*lat_topo) != 0])))
                if a>biggest_size:
                    biggest_size = a

        else:
            mask = (lw == elements[0])
            biggest_size = np.sum(np.abs(np.cos((mask*lat_topo)[(mask*lat_topo) != 0])))

        total_landmass[iterr] = pp
        big_cluster[iterr] = float(biggest_size / total_lat)
        iterr += 1

    return ii, big_cluster, total_landmass, iso_hight

with concurrent.futures.ProcessPoolExecutor() as executor:
    for row, result1, result2 in executor.map(water_function, range(num_proc)):
        print(row)
        squares1[row] = result1
        squares2[row] = result2

np.save("/home/complex/c++/Earth/big_cluster_" + str(N) + ".npy", squares1.flatten())
np.save("/home/complex/c++/Earth/total_landmass_" + str(N) + ".npy", squares2.flatten())
```


# ufunc 对象

**Numpy** 有两种基本对象：`ndarray (N-dimensional array object)` 和 `ufunc (universal function object)`。`ndarray` 是存储单一数据类型的多维数组，而 `ufunc` 则是能够对数组进行处理的函数。

例如，我们之前所接触到的二元操作符对应的 **Numpy** 函数，如 `add`，就是一种 `ufunc` 对象，它可以作用于数组的每个元素。

```python
import numpy as np
```

```python
a = np.array([0,1,2])
b = np.array([2,3,4])

np.add(a, b)
```

    array([2, 4, 6])

查看支持的方法：

```python
dir(np.add)
```

    ['__call__',
     '__class__',
     '__delattr__',
     '__doc__',
     '__format__',
     '__getattribute__',
     '__hash__',
     '__init__',
     '__name__',
     '__new__',
     '__reduce__',
     '__reduce_ex__',
     '__repr__',
     '__setattr__',
     '__sizeof__',
     '__str__',
     '__subclasshook__',
     'accumulate',
     'at',
     'identity',
     'nargs',
     'nin',
     'nout',
     'ntypes',
     'outer',
     'reduce',
     'reduceat',
     'signature',
     'types']

除此之外，大部分能够作用于数组的数学函数如三角函数等，都是 `ufunc` 对象。

特别地，对于二元操作符所对应的 `ufunc` 对象，支持以下方法：

## reduce 方法

    op.reduce(a)

将`op`沿着某个轴应用，使得数组 `a` 的维数降低一维。

add 作用到一维数组上相当于求和：

$$
\begin{align}
y & = add.recuce(a) \\
& = a[0] + a[1] + ... + a[N-1] \\
& = \sum_{n=0}^{N-1} a[n]
\end{align}
$$

```python
a = np.array([1,2,3,4])

np.add.reduce(a)
```

    10

多维数组默认只按照第一维进行运算：

```python
a = np.array([[1,2,3],[4,5,6]])

np.add.reduce(a)
```

    array([5, 7, 9])

指定维度：

```python
np.add.reduce(a, 1)
```

    array([ 6, 15])

作用于字符串：

```python
a = np.array(['ab', 'cd', 'ef'], np.object)

np.add.reduce(a)
```

    'abcdef'

逻辑运算：

```python
a = np.array([1,1,0,1])

np.logical_and.reduce(a)
```

    False

```python
np.logical_or.reduce(a)
```

    True

## accumulate 方法

    op.accumulate(a)

`accumulate` 可以看成保存 `reduce` 每一步的结果所形成的数组。

$$
\begin{align}
y & = add.accumulate(a) \\
& = \left[\sum_{n=0}^{0} a[n], \sum_{n=0}^{1} a[n], ..., \sum_{n=0}^{N-1} a[n]\right]
\end{align}
$$

与之前类似：

```python
a = np.array([1,2,3,4])

np.add.accumulate(a)
```

    array([ 1,  3,  6, 10])

```python
a = np.array(['ab', 'cd', 'ef'], np.object)

np.add.accumulate(a)
```

    array(['ab', 'abcd', 'abcdef'], dtype=object)

```python
a = np.array([1,1,0,1])

np.logical_and.accumulate(a)
```

    array([ True,  True, False, False], dtype=bool)

```python
np.logical_or.accumulate(a)
```

    array([ True,  True,  True,  True], dtype=bool)

## reduceat 方法

    op.reduceat(a, indices)

`reduceat` 方法将操作符运用到指定的下标上，返回一个与 `indices` 大小相同的数组：

$$
\begin{align}
y & = add.reduceat(a, indices) \\
& = \left[\sum_{n=indice[0]}^{indice[1]-1} a[n], \sum_{n=indice[1]}^{indice[2]-1} a[n], ..., \sum_{n=indice[-1]}^{N-1} a[n]\right]
\end{align}
$$

```python
a = np.array([0, 10, 20, 30, 40, 50])
indices = np.array([1,4])

np.add.reduceat(a, indices)
```

    array([60, 90])

这里，`indices` 为 `[1, 4]`，所以 `60` 表示从下标1（包括）加到下标4（不包括）的结果，`90` 表示从下标4（包括）加到结尾的结果。

## outer 方法

    op.outer(a, b)

对于 `a` 中每个元素，将 `op` 运用到它和 `b` 的每一个元素上所得到的结果：

```python
a = np.array([0,1])
b = np.array([1,2,3])

np.add.outer(a, b)
```

    array([[1, 2, 3],
           [2, 3, 4]])

注意有顺序的区别：

```python
np.add.outer(b, a)
```

    array([[1, 2],
           [2, 3],
           [3, 4]])


```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as axes3d
import seaborn
import sympy as sm
sm.init_printing()
```

## Άσκηση 2
Κυλινδρική έλικα έχει εξίσωση:
\begin{align}
\varpi &=R \\
\phi &= A \lambda t \\
z &= RA(1-e^{\phi/A})
\end{align}

όπου $R$ η κυλινδρική ακτίνα και $A$ σταθερά. Δαχτυλίδι είναι περασμένο και κινείται στην έλικα αυτή με τρόπο ώστε η αζιμουθιακή
γωνία αυξάνεται γραμμικά με το χρόνο με σταθερό $\lambda$.
* Βρείτε την ταχύτητα και την επιτάχυνση του δαχτυλιδιού σε κάθε χρόνο.
* Ποιά είναι η ακτίνα Καμπυλότητας
* Ποια η ακτίνα καμπυλότητας της έλικας;
* Αν η μάζα του δαχτυλιδιού είναι m πόση ενέργεια δίνει ή παίρνει συνολικά (από $t = 0$ ως πολύ μεγάλους χρόνους) αυτός που φροντίζει η κίνηση του δαχτυλιδιού να γίνεται με $\phi = A \lambda t$; (Αγνοήστε βάρος και τριβές.)

```python
p,z,f,t=sm.symbols('\\varpi,z,\phi,t', real=True, positive=True)
R,A,l,m=sm.symbols('R,A,\lambda,m', real=True, positive=True)
p
```

```python
p=R #(από εξίσωση 1)
f=A*l*t #(από εξίσωση 2)
z=R*A*(sm.Float(1)-sm.exp(-f/A)) #(από εξίσωση 3)
```

```python
r=sm.Matrix([[p],[f],[z]])
r
```

### Ταχύτητα σε κυλνδρικές συντεταγμένες
$$
\vec{v}=\dot{p} \hat{p} +p\dot{\phi} \hat{\phi}+\dot{z}\hat{z}
$$

```python
v=sm.Matrix([[sm.diff(p,t)],[p*sm.diff(f,t)],[sm.diff(z,t)]])
v
```

### Επιτάχυνση σε κυλνδρικές συντεταγμένες
$$
\vec{a}=(\ddot{p}-p\dot{\phi}^2)\hat{p} +(2\dot{p}\dot{\phi}+p\ddot{\phi}) \hat{\phi}+\ddot{z}\hat{z}
$$

```python
a=sm.Matrix([[sm.diff(p,t,2)-p*sm.diff(f,t)**2],[2*sm.diff(p,t)*sm.diff(f,t)+p*sm.diff(f,t,2)],[sm.diff(z,t,2)]])
a
```

### Τροχία σε καρτεσιανές συντεταγμένες

```python
rk = sm.Matrix([[p*sm.cos(f)],[p*sm.sin(f)],[z]])
rk
```

```python
sub={R:1.,A:5.,l:0.1}
```

```python
xn= sm.lambdify(t,rk[0].subs(sub),"numpy")
yn = sm.lambdify(t,rk[1].subs(sub),"numpy")
zn = sm.lambdify(t,rk[2].subs(sub),"numpy")
tt=np.arange(0.,150.,0.2)
```

```python
fig = plt.figure(figsize=(12,12))
ax = fig.add_subplot(1,1,1, projection='3d')
ax.view_init(elev=20)#, azim=40)
ax.set_zlim([-0.5, 5.2])
ax.set_zlabel('z')
ax.set_ylim([-1.5, 1.5])
ax.set_ylabel('y')
ax.set_xlim([-1.5, 1.5])
ax.set_xlabel('x')
ax.plot(xn(tt),yn(tt),zn(tt))
ax.plot(xn(tt),zn(tt),  zdir='y', zs=1.5,alpha=0.2)
ax.plot(yn(tt),zn(tt),  zdir='x', zs=-1.5,alpha=0.2)
ax.plot(xn(tt),yn(tt),  zdir='z', zs=-0.5,alpha=0.2)
```

## Επιτρόχιος και κεντρομόλος επιτάχυνση
Η επιτρόχιος επιτάχυνση βρίσκεται από τη:
$$
\vec{a}_e = (\vec{a}\cdot \vec{e}) \vec{e}
$$
με το επιτρόχιο διάνυσμα $\vec{e} =\frac{\vec{V}}{|V|}$

```python
e=v/v.norm()
a_e=a.dot(e)*e
sm.simplify(a_e)
```

ενώ η κεντρομόλος επιτάχυνση βρίσκεται από τη:
$$
\vec{a}_k = (\vec{a}\cdot \vec{n}) \vec{n}
$$
με το κάθετο διάνυσμα $\vec{n} =\frac{\vec{a}}{|a|}$

```python
n=a/a.norm()
a_k=a.dot(n)*n
sm.simplify(a_k)
```

```python
b=e.cross(n)
b=b/b.norm()
e,n,b
```

## Ακτίνα Καμπυλότητας
Η ακτίνα καμπυλότητας δίνεται από τη σχέση:
$$
R=k^{-1}=\frac{|\vec{v}^3|}{|\vec{v}\times\vec{a}|}
$$

```python
s=v.cross(a)
RR=v.norm()**3/s.norm()
sm.simplify(RR)
```

## Ενέργεια
Το έργο της δύναμης είναι:
$$
E=\int _0 ^\infty m \vec{a}\cdot{d\vec{r}} = \int _0 ^\infty m \vec{a}\cdot\vec{v} dt
$$

```python
sm.integrate(m*a.dot(v),(t,0,sm.oo))
```

# Επίλυση του προβλήματος των 3 σωμάτων
Θεωρούμε το σύστημα Ήλιος - Γη - Διας, Η χαμιλτονιανή του συστήματος είναι η:
$$
\mathcal{H}=\frac{P_0 ^2}{2M}+\frac{P_1 ^2}{2m_1}+
\frac{P_2^2}{2m_2}-\frac{GMm_1}{|\vec{R}_1-\vec{R}_0}-\frac{GMm_2}{|\vec{R}_2-\vec{R}_0|}-
\frac{Gm_1m_2}{|\vec{R}_2-\vec{R}_1|}
$$
ή σε ηλιοκεντρικές συντεταγμένες $\vec{r}_1 = \vec{R}_1-\vec{R}_0$,  $\vec{r}_2 = \vec{R}_2-\vec{R}_0$

$$
\mathcal{H}=\frac{(P_0+P_1+P_2) ^2}{2M}+\frac{P_1 ^2}{2m_1}+
\frac{P_2^2}{2m_2}-\frac{GMm_1}{|\vec{r}_1}-\frac{GMm_2}{|\vec{r}_2|}-
\frac{Gm_1m_2}{|\vec{\Delta}|}
$$
με $\vec{\Delta}=\vec{R}_2-\vec{R}_1$ τη σχετική θέση των δύο πλανητών.

Αν θεωρήσουμε ότι $P_0=0$ και αναπτύξουμε το πρώτο όρο βρίσκουμε:
$$
\mathcal{H}=\frac{P_1 ^2}{2\mu_1}-\frac{G(M+m_1)\mu_1}{|\vec{r}_1|}
+\frac{P_2^2}{2\mu_2}-\frac{G(M+m_2)\mu_2}{|\vec{r}_2|}
-\frac{Gm_1m_2}{|\vec{\Delta}|}+\frac{\vec{P}_1 \cdot \vec{P}_2}{M}
$$
όπου $\mu _i =\frac{Mm_i}{M+m_i}$ η ανηγμένη μάζα του εκάστοτε σώματος.

### Ορισμός Συμβόλων

```python
m1,m2,mu1,mu2,M=sm.symbols('m_j,m_\oplus,\mu_j,\mu_\oplus, M_\odot', real=True, positive=True)
x1,y1,x2,y2,p1x,p1y,p2x,p2y,p1,p2,r1,r2=sm.symbols('x_1,y_1,x_2,y_2,p_{1x},p_{1y},p_{2x},p_{2y},p_1,p_2,r_1,r_2',real=True, positive=True)
a1,a2,e1,e2=sm.symbols('a_1,a_2,e_1,e_2', real=True, positive=True)
x10,x20,y10,y20,p1x0,p1y0,p2x0,p2y0=sm.symbols('x_{10},x_{20},y_{10},y_{20},p_{1x0},p_{1y0},p_{2x0},p_{2y0}', real=True, positive=True)
W=sm.symbols('\Omega')
G,D=sm.symbols('G,\Delta', real=True, positive=True)
p1,p2,r1,r2,D=sm.symbols('p_1,p_2,r_1,r_2,\Delta')
```

### Hamiltonian

```python
pp=p1*p2
H1=p1**2/(2*mu1)-G*(M+m1)*mu1/r1
H2=p2**2/(2*mu2)-G*(M+m2)*mu2/r2
H3=-G*m1*m2/D + pp/M
H=H1+H2+H3
H
```

### Hamiltonian (σε καρτεσιανές συντεταγμένες)

```python
p1=sm.sqrt(p1x**2+p1y**2)
p2=sm.sqrt(p2x**2+p2y**2)
r1=sm.sqrt(x1**2+y1**2)
r2=sm.sqrt(x2**2+y2**2)
pp=p1x*p2x+p1y*p2y
D=sm.sqrt((y2-y1)**2+(x2-x1)**2)

H1=p1**2/(2*mu1)-G*(M+m1)*mu1/r1
H2=p2**2/(2*mu2)-G*(M+m2)*mu2/r2
H3=-G*m1*m2/D + pp/M
H=H1+H2+H3
H
```

## Εξισώσεις κινησεις
\begin{align}
\dot{x}_1 &= \frac{\partial \mathcal{H}}{\partial P_{1x}} &&
\dot{y}_1 = \frac{\partial \mathcal{H}}{\partial P_{1x}} \\
\dot{x}_2 &= \frac{\partial \mathcal{H}}{\partial P_{2x}} &&
\dot{y}_2 = \frac{\partial \mathcal{H}}{\partial P_{2x}} \\
\dot{P}_{1x} &= -\frac{\partial \mathcal{H}}{\partial P_{1x}} &&
\dot{P}_{1y} = -\frac{\partial \mathcal{H}}{\partial P_{1y}}\\
\dot{P}_{2x} &= -\frac{\partial \mathcal{H}}{\partial P_{2x}} &&
\dot{P}_{2y} = -\frac{\partial \mathcal{H}}{\partial P_{2y}}
\end{align}

```python
dotx1=H.diff(p1x)
doty1=H.diff(p1y)
dotx2=H.diff(p2x)
doty2=H.diff(p2y)

dotpx1=-H.diff(x1)
dotpy1=-H.diff(y1)

dotpx2=-H.diff(x2)
dotpy2=-H.diff(y2)
dotx1,doty1,dotx2,doty2,dotpx1,dotpy1,dotpx2,dotpy2
```

### Εισαγωγή αριθμητικών τιμών
* Μάζα Αστέρα: $1 M_\odot$
* Μάζα πρώτου πλανήτη: $0.0008 M_\odot$
* Μάζα δεύτερου πλανήτη: $0.0002 M_\odot$
* Ημιάξονας και εκκεντρότητα του πρώτου πλανήτη: $a_1=1.3 \mathtt{AU}, \, e_1 = 0.06$
* Ημιάξονας και εκκεντρότητα του δεύτερου πλανήτη: $a_1=2.9 \mathtt{AU}, \, e_1 = 0.02$

```python
Gn=4.*np.pi**2
m1n=1./1047. #0.01
m2n=3.0e-6 #0.005
Mn=1.
a1n=5.20260
a2n=1.000001018
e1n=0.048498
e2n=0.0167086
apj=273.867
ape=114.207
Wn=(apj-ape)*np.pi/180.
mu1n=m1n*Mn/(Mn+m1n)
mu2n=m2n*Mn/(Mn+m2n)
sub={mu1:mu1n,mu2:mu2n,G:Gn,m1:m1n,m2:m2n,M:Mn,a1:a1n,a2:a2n,e1:e1n,e2:e2n,W:Wn}
```

## Αρχικές συνθήκες
$$
x_{10},y_{10},x_{20},y_{20},p_{x10},p_{y10},p_{x20},p_{y20}
$$

```python
x10=a1*(1.+e1)*sm.cos(W)
y10=a1*(1.+e1)*sm.sin(W)
x20=a2*(1.+e2)
y20=sm.Float(0)

p1vec =mu1*sm.sqrt(G*(M+m1)*a1*(1.-e1**2))/(a1*(1.+e1))
p1x0=-p1vec*sm.sin(sm.pi-W)
p1y0=-p1vec*sm.cos(sm.pi-W)
p2x0=sm.Float(0)
p2y0=mu2*sm.sqrt(G*(M+m2)*a2*(1.-e2**2))/(a2*(1.+e2))

x10,y10,x20,y20,p1x0,p1y0,p2x0,p2y0
```

### Lambdification των εξισώσεων κίνησης
Μετατροπή των εξισώσεων κίνησης σε μορφή
$$
\frac{d\mathbf{Y}}{dt}=\mathbf{F}(\mathbf{Y},t)=\mathbf{F}(\vec{r}_1,\vec{r}_2,\vec{p}_1,\vec{p}_2,t)
$$
για την αριθμητική ολοκλήρωση τους.

```python
dx1 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotx1.subs(sub),"numpy")
dx2 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotx2.subs(sub),"numpy")
dy1 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),doty1.subs(sub),"numpy")
dy2 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),doty2.subs(sub),"numpy")
dpx1 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotpx1.subs(sub),"numpy")
dpx2 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotpx2.subs(sub),"numpy")
dpy1 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotpy1.subs(sub),"numpy")
dpy2 = sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),dotpy2.subs(sub),"numpy")
def dY(y,t):
    a= np.array([
            dx1(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dx2(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dy1(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dy2(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dpx1(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dpx2(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dpy1(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7]),
            dpy2(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7])
        ])
    return a
```

### Αρχικές αριθμητικές συνθήκες

```python
Y0=np.array([x10.subs(sub),
             x20.subs(sub),
             y10.subs(sub),
             y20.subs(sub),
             p1x0.subs(sub),
             p2x0.subs(sub),
             p1y0.subs(sub),
             p2y0.subs(sub)
            ]).astype('float')
Y0
```

    array([ -5.11478261e+00,   1.01670964e+00,   1.89607598e+00,
             0.00000000e+00,  -8.70771912e-04,   0.00000000e+00,
            -2.34896126e-03,   1.85371568e-05])

```python
from scipy.integrate import odeint
```

## Αριθμητική ολοκλήρωση
Αριθμητική ολοκλήρωση για 20000 χρόνια, με βήμα 3 ημερών

```python
%%time
t=np.arange(0.,20000.,3./365.)
sol=odeint(dY,Y0,t,rtol=1e-12,atol=1e-13,printmessg=True)
x1t,y1t=sol[:,0],sol[:,2]
x2t,y2t=sol[:,1],sol[:,3]
px1t,py1t=sol[:,4],sol[:,6]
px2t,py2t=sol[:,5],sol[:,7]
```

    CPU times: user 22min 57s, sys: 9.76 s, total: 23min 7s
    Wall time: 24min 25s

    /usr/local/lib/python2.7/dist-packages/scipy/integrate/odepack.py:221: ODEintWarning: Integration successful.
      warnings.warn(warning_msg, ODEintWarning)

```python
ten=100
lw=0.5
plt.figure(figsize=(15,10))
plt.plot([0],[0],'o')
plt.axis('equal')
plt.plot(x1t[::ten],y1t[::ten],linewidth=lw,label='Jupiter Orbit')
plt.plot(x2t[::ten],y2t[::ten],linewidth=lw,label='Earth Orbit')
plt.xlabel('X (A.U.)')
plt.ylabel('Y (A.U.)')
plt.tight_layout()
plt.legend(loc='best')
```

```python
Energy=sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),H.subs(sub),"numpy")
Energy1=sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),H1.subs(sub),"numpy")
Energy2=sm.lambdify((x1,x2,y1,y2,p1x,p2x,p1y,p2y),H2.subs(sub),"numpy")
Et=Energy(x1t,x2t,y1t,y2t,px1t,px2t,py1t,py2t)
E1t=Energy1(x1t,x2t,y1t,y2t,px1t,px2t,py1t,py2t)
E2t=Energy2(x1t,x2t,y1t,y2t,px1t,px2t,py1t,py2t)
```

```python
plt.loglog(t,np.abs(Et-Et[0])/np.abs(Et[0]))
plt.xlabel('Time log(yrs)')
plt.ylabel('Relative Energy Error')
plt.tight_layout()
```

```python
a1t=-Gn*(Mn+m1n)*mu1n/(2.*E1t)
G1=x1t*py1t-y1t*px1t
e1t=np.sqrt(1.-G1**2/(mu1n**2 *Gn*(Mn+m1n)*a1t))

a2t=-Gn*(Mn+m2n)*mu2n/(2.*E2t)
G2=x2t*py2t-y2t*px2t
e2t=np.sqrt(1.-G2**2/(mu2n**2 *Gn*(Mn+m2n)*a2t))
```

```python
plt.plot(t[::ten],a1t[::ten],linewidth=lw,label=u'Μεγάλος Ημιάξονας Δια')
plt.xlabel('Time (yrs)')
plt.ylabel(u'Μεγάλος Ημιάξονας (A.U.)')
plt.legend(loc='best')
plt.tight_layout()
```

```python
plt.plot(t[::ten],a2t[::ten],linewidth=0.5,label=u'Μεγάλος Ημιάξονας Γης')
plt.xlabel('Time (yrs)')
plt.ylabel(u'Μεγάλος Ημιάξονας (A.U.)')
plt.legend(loc='best')
plt.tight_layout()
```

```python
plt.plot(t[::ten],e1t[::ten],label=u'Εκκεντρότητα Δία',linewidth=0.1)
plt.xlabel('Time (yrs)')
plt.ylabel(u'Εκκεντρότητα')
plt.tight_layout()
plt.legend(loc='best')
```

```python
plt.plot(t[::ten],e2t[::ten],label=u'Εκκεντρότητα Γης',linewidth=0.1)
plt.xlabel('Time (yrs)')
plt.ylabel(u'Εκκεντρότητα')
plt.tight_layout()
plt.legend(loc='best')
```

```python

```