# NUME

Instructor: Keny **ORDAZ**

# Selected Topics

$\newcommand{\vb}[1]{\boldsymbol{#1}}\newcommand{\RR}[1]{\mathbb{R}^{#1}}
\newcommand{\vt}[1]{\vb{#1}^\top}$

$\newcommand{\x}{\vb{x}}
\newcommand{\xT}{\vt{x}}
\newcommand{\b}{\vb{b}}
\newcommand{\A}{\vb{A}}
\newcommand{\bT}{\vt{b}}
\newcommand{\AT}{\vt{A}}$

```python
import numpy as np
import scipy
import sympy as sp
import matplotlib.pyplot as plt
%matplotlib widget
```

# Conjugate Gradient Visualization

```python
A = np.array([[4, -1], [-1, 3]])
b = np.array([9, -16])

np.linalg.solve(A, b)
```

    array([ 1., -5.])

```python

def conj_grad(Av, x, b, tol=1.0e-9, callback=None):
    ''' x, numIter = conj_grad(Av,x,b,tol=1.0e-9)
    Conjugate gradient method for solving [A]{x} = {b}.
    The matrix [A] should be sparse. User must supply
    the function Av(v) that returns the vector [A]{v}
    and the starting vector x.
'''
    R = []
    n = len(b)
    r = b - Av(x)
    s = r.copy()
    R.append(x)
    for i in range(n):
        u = Av(s)
        alpha = np.dot(s,r)/np.dot(s,u)
        x = x + alpha*s
        r = b - Av(x)
        R.append(x)
        #print(x)
        if (np.linalg.norm(r)) < tol:
            break
        else:
            beta = -np.dot(r,u)/np.dot(s,u)
            s = r + beta*s
    return x, i, np.asarray(R)
```

```python
import scipy.sparse.linalg

nb_iters = 0

def cg_callback(xk):
    global nb_iters
    nb_iters = nb_iters + 1
    print(f'xk: {xk}')

x_py, i_py = scipy.sparse.linalg.cg(A, b, [0, 17], callback=cg_callback)
x_py, i_py, nb_iters
```

    xk: [ 6.83235818 -0.60646146]
    xk: [ 1. -5.]

    (array([ 1., -5.]), 0, 2)

```python
v_x, v_i, v_r = conj_grad(lambda x : A @ x, np.array([0., 17]), b)
```

```python
def visualization(A, b, v_r):
    N = 31
    x = np.linspace(-48, 48, N)
    y = np.linspace(-20, 20, N)

    X, Y = np.meshgrid(x, y)
    Z = np.zeros_like(Y)
    XY = np.array([X, Y])
    grad = np.zeros_like(XY)
    for i in range(N):
        for j in range(N):
            Z[i, j] = 0.5 * XY[:, i, j] @ A @ XY[:, i, j] - b @ XY[:, i, j]
            grad[:, i, j] = A @ XY[:, i, j] - b

    U, V = grad

    fig, ax = plt.subplots()
    #cs = ax.contourf(X, Y, Z)
    CS = ax.contour(X, Y, Z)
    ax.quiver(X, Y, U, V)
    #P = np.array([[0, 17], [ 6.83235818, -0.60646146], [ 1., -5.]])

    #ax.plot(*P.T, 'r:o')
    ax.plot(*v_r.T, 'b:.')
    ax.set_aspect('equal')
    print(v_r.shape)

visualization(A, b, v_r)
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

    (3, 2)

```python
v_r[0] @ v_r[1], v_r[0] @ v_r[2]
v_r
```

    array([[ 2.60000000e+01, -6.70000000e+01],
           [-1.89358942e+01, -7.34825744e+00],
           [-8.88178420e-15,  7.10542736e-15]])

```python
P = np.array([[0, 17], [ 6.83235818, -0.60646146], [ 1., -5.]])
```

```python
x = P [-1]
x + v_r[1] + v_r[0]
```

    array([  8.06410583, -79.34825744])

```python

```

```python

```

```python

```

# Visualización

```python
%matplotlib widget
```

```python
import ipywidgets as widgets
```

```python
def three(x, y, z):
    return (x, y, z)

_ = widgets.interact(
    three,
    x=(0, 10, 1),
    y=True,
    z=['a', 'b', 'c']
)
```

    interactive(children=(IntSlider(value=5, description='x', max=10), Checkbox(value=True, description='y'), Drop…

```python
# set up plot
fig, ax = plt.subplots(figsize=(6, 4))
ax.set_ylim([-4, 4])
ax.grid(True)

# generate x values
x = np.linspace(0, 2 * np.pi, 100)

def my_sine(x, w, amp, phi):
    """
    Return a sine for x with angular frequeny w and amplitude amp.
    """
    return amp*np.sin(w * (x-phi))

@widgets.interact(w=(0, 10, 1), amp=(0, 4, .1), phi=(0, 2*np.pi+0.01, 0.01))
def update(w = 1.0, amp=1, phi=0):
    """Remove old lines from plot and plot new one"""
    [l.remove() for l in ax.lines]
    ax.plot(x, my_sine(x, w, amp, phi), color='C0')
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

    interactive(children=(IntSlider(value=1, description='w', max=10), FloatSlider(value=1.0, description='amp', m…

```python
# Repaso
```

```python
## LU
```

```python
A = np.array([[10, 4, 7.], [2, 5, 8], [3, 6, 10]])
A
```

    array([[10.,  4.,  7.],
           [ 2.,  5.,  8.],
           [ 3.,  6., 10.]])

```python
M = []
T = np.array([[1, 0, 0], [-1/5, 1, 0], [-3/10, 0, 1]])
M.append(T)
M[0] @ A
```

    array([[10. ,  4. ,  7. ],
           [ 0. ,  4.2,  6.6],
           [ 0. ,  4.8,  7.9]])

```python
T = np.array([[1, 0, 0], [0, 1, 0], [0, -8/7, 1]])
M.append(T)
M[1] @ M[0] @ A
```

    array([[10.        ,  4.        ,  7.        ],
           [ 0.        ,  4.2       ,  6.6       ],
           [ 0.        ,  0.        ,  0.35714286]])

```python
 M[1] M[0] A = (M[1] M[0] L) U = U
    => L = M[0]^{-1} M[1]^{-1}
```

```python
M
```

    [array([[ 1. ,  0. ,  0. ],
            [-0.2,  1. ,  0. ],
            [-0.3,  0. ,  1. ]]),
     array([[ 1.        ,  0.        ,  0.        ],
            [ 0.        ,  1.        ,  0.        ],
            [ 0.        , -1.14285714,  1.        ]])]

```python
[np.linalg.inv(M[i]) for i in range(2)]
```

    [array([[1. , 0. , 0. ],
            [0.2, 1. , 0. ],
            [0.3, 0. , 1. ]]),
     array([[ 1.        ,  0.        ,  0.        ],
            [-0.        ,  1.        , -0.        ],
            [ 0.        ,  1.14285714,  1.        ]])]

```python
np.any(np.isclose(, np.zeros(3)))
```

    False

1 0 1 == 0 0 0 ---> F V F ---> $!\exists A^{-1}$

```python
np.diag(M[1] @ M[0]).prod() == 0.0
```

    False

```python
np.isclose(np.diag(M[1] @ M[0]).prod(), 0.0)
```

    False

```python
a = 1e-300
f'{a:0.0f}', a == 0.0, (1. + a)/1.0 == 1.0
```

    ('0', False, True)

```python
from decimal import Decimal
Decimal.from_float(1e-17)
```

    Decimal('1.00000000000000007154242405462192450852805618492324772617063644020163337700068950653076171875E-17')

```python
Decimal.from_float((1. + a)/1.0), Decimal.from_float(a * b)
```

    (Decimal('1'), Decimal('5.46699999999999963762320476234890520572662353515625'))

```python
b = 5.467e300
a * b
```

    5.467

```python
0.1 + 0.1 + 0.1 == 0.3
```

    False

```python
A
```

    array([[10.,  4.,  7.],
           [ 2.,  5.,  8.],
           [ 3.,  6., 10.]])

```python
M
```

    [array([[ 1. ,  0. ,  0. ],
            [-0.2,  1. ,  0. ],
            [-0.3,  0. ,  1. ]]),
     array([[ 1.        ,  0.        ,  0.        ],
            [ 0.        ,  1.        ,  0.        ],
            [ 0.        , -1.14285714,  1.        ]])]

```python
Ac = A.copy()
Ac
```

    array([[10.,  4.,  7.],
           [ 2.,  5.,  8.],
           [ 3.,  6., 10.]])

```python
Ac[[1, 2]] = Ac[[2, 1]]
Ac
```

    array([[10.,  4.,  7.],
           [ 3.,  6., 10.],
           [ 2.,  5.,  8.]])

```python
P = np.eye(3)
P[[1, 2]] = P[[2, 1]]
P
```

    array([[1., 0., 0.],
           [0., 0., 1.],
           [0., 1., 0.]])

```python
P @ A
```

    array([[10.,  4.,  7.],
           [ 3.,  6., 10.],
           [ 2.,  5.,  8.]])

```python
Ac[1,:] += (-Ac[1,0]/Ac[0,0]) * Ac[0,:]
```

```python
Ac
```

    array([[10. ,  4. ,  7. ],
           [ 0. ,  4.8,  7.9],
           [ 2. ,  5. ,  8. ]])

```python
# P[1,:] += (-A[1,0]/A[0,0]) * P[0,:]
P
```

    array([[1., 0., 0.],
           [0., 0., 1.],
           [0., 1., 0.]])

```python
P @ P.T == np.eye(3)
# Ortho: P^T == P^{-1}
# Sim: A = A^T
#
```

    array([[ True,  True,  True],
           [ True,  True,  True],
           [ True,  True,  True]])

```python
  A = P @ L @ U
```

```python
x = np.array([5, 1, 0])
```

```python
P @ x
```

    array([5., 0., 1.])

```python
A
```

    array([[10.,  4.,  7.],
           [ 2.,  5.,  8.],
           [ 3.,  6., 10.]])

```python
Mi = [np.linalg.inv(M[i]) for i in range(2)]
```

```python
Minv = Mi[0] @ Mi[1]
Minv
```

    array([[1.        , 0.        , 0.        ],
           [0.2       , 1.        , 0.        ],
           [0.3       , 1.14285714, 1.        ]])

```python
D = np.diag([1,0,3])
np.linalg.det(Minv)
```

    1.0

```python
# A es triangular (diagonal)
#det(A) = ∏aii

det(A): O(n!)
A = LU => det(A) = det(LU) = det(L) det(U) = 1 * det(U) = ∏Uii, O(n)
LU: O(n^3)

    O(n^3) + O(n) = O(n^3) det por LU
```

```python
import math
[(i, i**3, math.factorial(i)) for i in range(15)]
```

    [(0, 0, 1),
     (1, 1, 1),
     (2, 8, 2),
     (3, 27, 6),
     (4, 64, 24),
     (5, 125, 120),
     (6, 216, 720),
     (7, 343, 5040),
     (8, 512, 40320),
     (9, 729, 362880),
     (10, 1000, 3628800),
     (11, 1331, 39916800),
     (12, 1728, 479001600),
     (13, 2197, 6227020800),
     (14, 2744, 87178291200)]

```python
Minv
```

    array([[1.        , 0.        , 0.        ],
           [0.2       , 1.        , 0.        ],
           [0.3       , 1.14285714, 1.        ]])

```python
M[1] @ M[0]
```

    array([[ 1.        ,  0.        ,  0.        ],
           [-0.2       ,  1.        ,  0.        ],
           [-0.07142857, -1.14285714,  1.        ]])

```python
Minv[[1,2],:][:,[1,2]]
```

    array([[1.        , 0.        ],
           [1.14285714, 1.        ]])

```python
1 ()
-0.2 () + 1()
0.3() - 1.14...() + 1()
```

```python
adjoint ---- adjugate
```

```python
Mi[0] @ Mi[1] = Ai @ Bi = (B A)^{-1} => M[1] @ M[0]
```


# 1.2 概率论

加法率：

$$
P(X) = \sum_Y P(X,Y)
$$

乘法率：

$$
P(X,Y)=P(Y|X)P(X)=P(Y,X)=P(X|Y)P(Y)
$$

由乘法率可得 `Bayes` 公式：

$$
P(Y|X)=\frac{P(X|Y)P(Y)}{P(X)}
$$

再由加法率和乘法率，`Bayes` 公式的分母可以写成：

$$
P(X) = \sum_Y P(X,Y) = \sum_Y P(X|Y)P(Y)
$$

所以 `Bayes` 公式的分母可以看成是一个归一化项，使得条件概率 $P(Y|X)$ 对所有的 $Y$ 求和之后和为 `1`。

如果
$P(X,Y)=P(X)P(Y)
$，那么 $X, Y$ 是独立的。

## 1.2.1 概率密度函数

### 概率密度函数

对于实值变量 $x$，概率密度函数（`probability density function`） $p(x)$ 定义为：当$\delta x>0, \delta x\rightarrow 0$ 时，变量 $x$ 落在区间 $(x,x+\delta x)$ 范围内的概率为 $p(x)\delta x$。

$x$ 落在任意区间 $(a,b)$ 的概率为：

$$
P(x\in (a,b)) = \int_{a}^b p(x) dx
$$

对于概率密度函数，它需要满足两个条件：

$$
\begin{align}
p(x)&\geq 0 \\
\int_{-\infty}^\infty p(x) dx&=1
\end{align}
$$

对于随机变量间这样的一个变换 $x=g(y)$，设 $x,y$ 的概率密度函数分别为 $p_x(x)$ 和 $p_y(y)$，那么 $x$ 落在 $(x, x+\delta x)$ 可以对应于 $y$ 落在 $(y, y+\delta y)$ 的情况，并有 $p_x(x)\delta x\simeq p_y(y)\delta y$，从而：

$$
p_y(y)=p_x(x)\left|\frac{dy}{dx}\right|=p_x(g(y))\left|g'(y)\right|
$$

### 累积分布函数

累积分布函数（`cumulative distribution function`）定义为：

$$
P(z)=\int_{-\infty}^zp(x) dx
$$

满足：

$$
P'(x) = p(x)
$$

### 联合概率密度分布

对于多个变量 $x_1, \dots, x_D$（用 $\mathbf x$ 表示），定义联合概率密度为 $p(\mathbf x)=p(x_1,\dots,x_D)$，满足当 $\mathbf x$ 落在一个包含 $\mathbf x$ 的足够小的空间体积 $\delta \mathbf x$ 中时，其概率为 $p(\mathbf x)\delta \mathbf x$。

它也需要满足：

$$
\begin{align}
p(\mathbf x)&\geq 0 \\
\int p(\mathbf x) dx&=1
\end{align}
$$

其中积分是对整个空间进行积分。

对于概率密度函数，我们的加法法则乘法法则仍然适用：

$$
\begin{align}
p(x) & = \int p(x,y) dy \\
p(x,y) & = p(y|x)p(x)
\end{align}
$$

### 概率质量函数

对于离散的 $x$ 我们有时候将概率密度函数叫做概率质量函数，因为它的质量相当于集中在了某个允许的 $x$ 上。

## 1.2.2 期望和方差

### 期望

函数 $f(x)$ 在概率密度函数 $p(x)$ 下的均值叫做 $f(x)$ 的期望（`expectation`）。

离散分布下，定义为：

$$\mathbb E\left[f\right] = \sum_x p(x) f(x)$$

连续分布下，定义为：

$$
\mathbb E\left[f\right] = \int p(x) f(x) dx
$$

如果给定了 $N$ 个从 $p(x)$ 中随机抽样的点，那么期望可以近似为

$$
\mathbb E\left[f\right] \simeq \frac 1 N \sum_{n=1}^N f(x_n)
$$

当 $N\rightarrow \infty$ 是，等式成立。

多元函数可以对其中的一个参数求期望，例如 $\mathbb E_x\left[f(x,y)\right]$ 是函数 $f(x,y)$ 在概率密度 $f(x)$ 上的期望，注意，它是一个关于 $y$ 的函数。

### 条件期望

我们可以考虑一个函数 $f(x)$ 在条件分布 $p(x|y)$ 下的条件期望（`conditional expectation`），当 $x$ 是离散变量，定义为：

$$
\mathbb E_x[f|y]= \sum_x p(x|y) f(x)
$$

当 $x$ 是连续变量时，定义为

$$
\mathbb E_x[f|y]= \int p(x|y) f(x) dx
$$

### 方差

$f(x)$ 的方差（`variance`）定义为：

$$
\operatorname{var}[f] = \mathbb E\left[(f(x) - \mathbb E[f(x)])^2\right]
$$

它表示 $f(x)$ 偏离其均值 $\mathbb E[f(x)]$ 的程度。

对平方进行展开之后，方差可以写成：

$$
\operatorname{var}[f] = \mathbb E[f(x)^2] - \mathbb E[f(x)]^2
$$

特别地，我们考虑 $x$ 本身的方差：

$$
\operatorname{var}[x] = \mathbb E[x^2] - \mathbb E[x]^2
$$

### 协方差

对于两个随机变量 $x, y$，其协方差（`covariance`）定义为：

$$
\begin{align}
\operatorname{cov}[x,y]
& = \mathbb E_{x,y} \left[(x-\mathbb E[x])(y-\mathbb E[y])\right] \\
& = \mathbb E_{x,y} [xy] - \mathbb E[x]\mathbb E[y]
\end{align}
$$

当变量 $x,y$ 独立时，协方差为 $0$。

### 协方差矩阵

对于两个随机向量 $\mathbf{x, y}$，其协方差为一个矩阵：

$$
\begin{align}
\operatorname{cov}[\mathbf{x, y}]
& = \mathbb E_{\mathbf{x, y}} \left[(\mathbf x-\mathbb E[\mathbf x])(\mathbf y^\top-\mathbb E[\mathbf y^\top])\right] \\
& = \mathbb E_{\mathbf{x, y}} [\mathbf{xy^\top}] - \mathbb E[\mathbf x]\mathbb E[\mathbf y^\top]
\end{align}
$$

如果只是考虑随机向量 $\mathbf x$ 自身分量之间的协方差，那么我们有 $\operatorname{cov}[\mathbf x] \equiv \operatorname{cov}[\mathbf{x, x}]$

## 1.2.3 Bayes 概率

假设我们有一组模型的参数 $\mathbf w$，并且做出假定：这组参数服从一定的先验概率分布 $p(\mathbf w)$。

$\mathcal D = \{t_1, \dots, t_n\}$ 是我们观测到的一组数据，这组数据在参数 $\mathbf w$ 下的条件概率分布为 $p(\mathcal D|\mathbf w)$。

`Bayes` 公式告诉我们：

$$
p(\mathbf w|\mathcal D)=\frac{p(\mathcal D|\mathbf w)p(\mathbf w)}{p(\mathcal D)}
$$

这给了我们一种衡量在观测到数据 $\mathcal D$ 的情况下，参数 $\mathbf w$ 的不确定性的方法。

### 似然函数

$p(\mathcal D|\mathbf w)$ 可以看成是给定观测数据 $\mathcal D$ 的情况下关于参数向量 $\mathbf w$ 的一个函数，通常叫做似然函数（`likelihood function`）。

似然函数反映了在给定一组参数 $\mathbf w$ 的情况下，生成这组观测数据的一种可能性。注意它并不是一个关于 $w$ 的概率分布。

给定似然函数的定义，我们可以将上面的 Bayes 公式表示为：

$$
\text{posterior} \propto \text{likelihood} \times \text{prior}
$$

这三个量都是 $\mathbf w$ 的函数。

对于分母，在给定观测数据 $\mathcal D$ 的情况下是一个归一化常数，可以写成：

$$
p(\mathcal D)=\int p(\mathcal D|\mathbf w)p(\mathbf w)d\mathbf w
$$

在 Bayes 学派和频率学派的眼中，似然函数 $p(\mathcal D|\mathbf w)$ 都扮演了一个重要角色。但二者对于似然函数的使用方式是截然不同的。

在频率学派眼中，$\mathbf w$ 被看成是一个固定的参数，其值由某些估计量来决定，误差的计算要考虑数据 $\mathcal D$ 的分布；在 Bayes 学派眼中，数据集 $\mathcal D$ 是唯一的，参数 $\mathbf w$的不确定性只来自于 $\mathbf w$ 的一个概率分布。

### 频率估计量：最大似然估计

一个最常用的频率估计量就是最大似然（`maximum likelihood`）估计：

$$
\mathbf w_{ML} = \operatorname{argmax}_{\mathbf w} p(\mathcal D|\mathbf w)
$$

这相当于选择使得观测数据 $\mathcal D$ 出现的概率最大化的 $\mathbf w$。

在机器学习的文献中，似然函数的负对数通常被叫做一个损失函数（`error function`），因为负对数函数是单调递减的函数，因此最大似然就相当于最小化损失函数。

一种衡量频率估计量误差大小的方法是 `bootstrap`：假设我们有 $N$ 个数据 $\mathbf X=\{\mathbf x_1,\dots,\mathbf x_N\}$，我们从这写数据中有放回的抽样 $N$ 个数据得到一组新的数据 $\mathbf X_{\text B}$。
重复进行 $L$ 次这样的操作，得到 $L$ 组这样的抽样数据，然后可以通过这 $L$ 组抽样数据的统计结果估计最大似然估计的误差。

### Bayes 估计

`Bayes` 估计的一个重要观点是先验知识的引入，根据后验概率来决定参数 $\mathbf w$。

考虑抛硬币的情况，假设我们抛三次，每次都得到正面，那么最大似然估计会得到这枚硬币会 `100%` 得到正面的结论，而 `Bayes` 估计不会得到这么极端的结论。

两者并没有什么好坏之分，只不过是看问题的角度不同。`Bayes` 估计如果选定的先验不好，也可能得到很差的结果。

## 1.2.4 高斯分布

高斯分布（`Gaussian distribution`），又叫正态分布（`normal distribution`）。

对于实值变量 $x$，高斯分布定义为：

$$
\mathcal{N}\left(x\left|~\mu,\sigma^2\right.\right) = \frac{1}{(2\pi\sigma^2)^{1/2}} \exp\left\{-\frac{1}{2\sigma^2}(x-\mu)^2\right\}
$$

参数为均值 $\mu$ 和方差 $\sigma^2$。方差的平方根 $\sigma$ 叫做标准差，方差的倒数 $\beta = \frac{1}{\sigma^2}$ 叫做精度。

其图像如下所示：

```python
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

%matplotlib inline

from scipy.stats import norm

xx = np.linspace(-3, 3, 200)

norm_xx = norm.pdf(xx)

fig, ax = plt.subplots()

ax.plot(xx, norm_xx, "r")
ax.set_ylim(0, 0.5)
ax.set_ylabel(r"$\mathcal{N}\left(x|\mu,\sigma^2\right)$", fontsize="xx-large")
ax.set_yticks([])
ax.set_yticklabels([])

ax.set_xticks([0])
ax.set_xticklabels([r"$\mu$"], fontsize="xx-large")

ax.text(-.1, 0.25, "$2\sigma$", fontsize="xx-large")

ax.annotate("",
            xy=(-1, 0.24), xycoords='data',
            xytext=(1, 0.24), textcoords='data',
            arrowprops=dict(arrowstyle="<->",
                            connectionstyle="arc3"),
            )

plt.show()
```

下面证明这是一个概率分布，首先，从定义中可以看出：

$$\mathcal{N}\left(x\left|~\mu,\sigma^2\right.\right) > 0$$

然后是积分：

$$\int_{-\infty}^{\infty} \mathcal{N}\left(x\left|~\mu,\sigma^2\right.\right) dx = 1$$

这个结果可以通过计算 $I = \int_{\infty}^{\infty} \exp (x^2)dx$ 得到（计算 $I^2$，并换成极坐标计算）。

$x$ 的期望为（令 $y = x + \mu$，对 $y$ 的积分变成一个奇函数的积分加上 $\mu$ 乘以一个高斯分布的积分）：

$$
\mathbb E[x] = \int_{-\infty}^{\infty} \mathcal{N}\left(x\left|~\mu,\sigma^2\right.\right)x~dx = \mu
$$

其方差为（等式 $\int_{-\infty}^{\infty} \exp\left\{ -\frac{1}{2\sigma^2}(x-\mu)^2 \right\} dx = (2\pi \sigma^2)^{1/2}$ 两边对 $\sigma^2$ 求导）：

$$
\text var[x] = \int_{-\infty}^{\infty} \mathcal{N}\left(x\left|~\mu,\sigma^2\right.\right)(x-\mu)^2dx =\sigma^2
$$

因此：

$$
\mathbb E[x^2] = \mathbb E[x]^2 + \text{var}[x] = \mu^2 + \sigma^2
$$

概率分布的最大值叫做众数（`mode`），高斯分布的众数就是均值 $\mu$。

### 多维高斯分布

对于 $D$ 维的向量 $\mathbf x$，高斯分布定义为：

$$
\mathcal{N}\left(\mathbf x\left|~\mathbf{\mu, \Sigma}\right.\right) = \frac{1}{(2\pi)^{D/2}} \frac{1}{|\mathbf\Sigma|^{1/2}} \exp \left\{-\frac{1}{2}(\mathbf x - \mathbf \mu)^\top\mathbf\Sigma^{-1}(\mathbf x - \mathbf \mu)\right\}
$$

其中，$D$ 维向量 $\mathbf \mu$ 是均值，$D\times D$ 矩阵 $\mathbf\Sigma$ 是方差，$|\mathbf\Sigma|$ 是其行列式。

### 最大似然估计

假设我们现在有 $N$ 组对 $x$ 的观测数据 $\mathsf x = (x_1,\dots,x_N)^{\text T}$，这些数据是独立同分布（`independent and identically distributed, i.i.d.`）的，都服从一个均值 $\mu$，方差 $\sigma^2$ 的高斯分布。那么在给定这些参数的情况下，出现这些观测数据的概率，或者从参数的角度来说，似然函数为：

$$
p(\mathsf x~|~\mu, \sigma^2)=\prod_{n=1}^N \mathcal N\left(x_n \left|~\mu, \sigma^2\right.\right)
$$

通常最大似然的问题经常转化为求最大对数似然的问题：

$$
\ln p(\mathsf x~|~\mu, \sigma^2) = -\frac{1}{2\sigma^2} \sum_{n=1}^N(x_n - \mu)^2 - \frac N 2 \ln \sigma^2 - \frac N 2 \ln(2\pi)
$$

对 $\mu$ 最大化，我们得到最大似然解：

$$
\mu_{ML} = \frac 1 N \sum_{n=1}^N x_n
$$

即样本均值。

对 $\sigma^2$ 最大化，我们得到：

$$
\sigma^2_{ML} = \frac 1 N \sum_{n=1}^N (x_n-\mu_{ML})^2
$$

即样本方差。

但是这个解不是无偏的，我们可以计算它们的期望：

$$
\begin{align}
\mathbb E[\mu_{ML}] & = \frac 1 N \sum_{n=1}^N \mathbb E[x_n] = \mu \\
\mathbb E[\sigma^2_{ML}] & = \mathbb E \left[\frac{1}{N} \sum_{n=1}^N(x_n-\frac 1 N \sum_{m=1}^N x_m)\right] \\
& = \frac 1 N \sum_{i=1}^N \mathbb E \left[x_n^2-\frac 2 N x_n\sum_{m=1} x_m + \frac{1}{N^2} \sum_{m=1}^N\sum_{l=1}^N x_mx_l\right] \\
& = (\mu^2 + \sigma^2) - 2 (\mu^2+ \frac 1 N \sigma^2) + \mu^2+ \frac 1 N \sigma^2 \\
& = \left(\frac{N-1}{N}\right)\sigma^2
\end{align}
$$

用到了：
- 当 $m=n$ 时，$\mathbb E[x_mx_n]=\mathbb E[x_n^2] = \mu^2 + \sigma^2$
- 当 $m\neq n$ 时，$\mathbb E[x_mx_n]=\mathbb E[x_n]\mathbb E[x_m] = \mu^2$

因此，方差的一个无偏估计为：

$$
\tilde \sigma^2 = \frac{N}{N-1}\sigma^2_{ML}=\frac{1}{N-1}\sum_{n=1}^N(x_n-\mu_{ML})^2
$$

随着 $N$ 的增大，方差估计的误差也随之增大。

## 1.2.5 重新理解曲线拟合

对于曲线拟合的问题，设训练集输入为 $\mathsf x=(x_1, \dots, x_N)^\top$，对应的目标值为 $\mathsf t=(t_1, \dots, t_N)^\top$。

我们将我们的不确定性用高斯分布来表示，假设给定 $x$，对应的目标值 $t$ 服从一个均值为 $y(x,\mathbf w)$ 的高斯分布：

$$
p(t\left|~x,\mathbf w,\beta\right.)=\mathcal N\left(t\left|~y(x,\mathbf w), \beta^{-1}\right.\right)
$$

```python
xx = np.linspace(-0.9, 0.9, 100)
yy = 4 * xx - np.sin(xx * np.pi)

fig, ax = plt.subplots()
ax.plot(xx, yy, color="red")

ax.set_xlim(-1, 1)
ax.set_ylim(-4, 4)

ax.set_xticks([0])
ax.set_xticklabels([r'$x_0$'], fontsize="xx-large")
ax.set_yticks([0])
ax.set_yticklabels([r'$y(x_0, \mathbf{w})$'], fontsize="xx-large")

xx = np.linspace(-4, 4, 100)
yy = norm.pdf(xx, scale=0.5) / 5

ax.plot([-1, 0], [0, 0], "g--")
ax.plot([0, 0], [-4, 4], "k")
ax.plot(yy, xx)

ax.annotate("",
            xy=(0.75, -0.5), xycoords='data',
            xytext=(0.75, 0.5), textcoords='data',
            arrowprops=dict(arrowstyle="<->",
                            connectionstyle="arc3"),
            )

ax.text(0.77, -0.2, r'$2\sigma$', fontsize="xx-large")
ax.text(0.15, -1, r'$p(t|x_0,\mathbf{w}, \beta)$', fontsize="xx-large")
ax.text(0.5, 3, r'$y(x, \mathbf{w})$', fontsize="xx-large")

plt.show()
```

### 最大似然

设训练集数据是独立同分布的，那么似然函数为：

$$
p(\mathsf t \left|\mathsf x, \mathbf w, \beta\right.)
= \sum_{i=1}^N \mathcal{N}\left(t_n\left|~y(x,\mathbf w), \beta^{-1}\right.\right)
$$

对数似然为：

$$
\ln p(\mathsf t \left|\mathsf x, \mathbf w, \beta\right.) = -\frac{\beta}{2} \sum_{n=1}^N \{y(x,\mathbf w) - t_n\}^2 + \frac N 2 \ln \beta - \frac N 2 \ln(2\pi)
$$

设系数的最大似然解为 $\mathbf w_{ML}$，从最大化对数似然的角度来看，求它的问题相当于最小化：

$$
\frac{1}{2} \sum_{n=1}^N \{y(x,\mathbf w) - t_n\}^2
$$

这就是之前最小化平方误差和的结果。

因此最小化平方误差和可以看成是高斯噪音假设下的最大似然的结果。

再对精度 $\beta$ 求最大似然，我们有（可以理解为照搬之前求 $\sigma^2$ 的结果）：

$$
\frac{1}{\beta_{ML}} = \frac{1}{N}\sum_{i=1}^N \{y(x,\mathbf w) - t_n\}^2
$$

我们有了最大似然的结果之后，对于一个新的输入 $x$，其输出 $t$ 应当满足：

$$
p\left(t\left|~x,\mathbf w_{ML}, \beta_{ML}\right.\right) = \mathcal N\left(t\left|~y(x,\mathbf w_{ML}), \beta_{ML}^{-1}\right.\right)
$$

### 最大化后验分布

假设我们对系数 $\mathbf w$ 有一个先验的知识（$M$ 是多项式阶数，加上常数项一共 $M+1$ 维）：

$$
p(\mathbf w~|~\alpha) = \mathcal{N}(\mathbf w~|~\mathbf 0, \alpha^{-1} I)
= \left(\frac{\alpha}{2\pi}\right)^{(M+1)/2} \exp\left\{-\frac{\alpha}{2}\mathbf{w}^\top\mathbf{w}\right\}
$$

$\alpha$ 控制这个模型的先验分布，这一类的参数通常被叫做超参（`hyperparameters`），Bayes 公式告诉我们，后验概率正比与先验概率和似然函数的乘积：

$$
p(\mathbf w~|~\mathsf{x, t}, \alpha, \beta) \propto p(\mathsf t~|~\mathsf x, \mathbf w, \beta)~p(\mathbf w~|~\alpha)
$$

我们可以通过最大化后验概率（`maximum posterior, MAP`）来决定参数 $\mathbf w$ 的值，对上式求对数，并去掉跟 $\mathbf w$ 无关的项，我们相当于要最大化：

$$
-\frac{\beta}{2} \sum_{n=1}^N \{y(x,\mathbf w) - t_n\}^2 -\frac{\alpha}{2}\mathbf{w}^\top\mathbf{w}
$$

即最小化

$$
\frac{\beta}{2} \sum_{n=1}^N \{y(x,\mathbf w) - t_n\}^2 +\frac{\alpha}{2}\mathbf{w}^\top\mathbf{w}
$$

因此，`MAP` 的结果相当于给多项式拟合加二范数正则化的结果，其中正则参数 $\lambda = \alpha / \beta$。

## 1.2.6 Bayes 曲线拟合

虽然在 `MAP` 中，我们引入了先验分布，但是本质上它还是一个点估计，本质上并不是一个完全的 `Bayes` 估计。

一个完全的 `Bayes` 估计要求我们对 $\mathbf w$ 的所有值进行积分。

对于之前的曲线拟合问题，给定训练集 $\mathsf x$ 和 $\mathsf t$，对于一个新的测试样例 $x$，其目标值为 $t$，我们考虑预测的分布 $p(t~|~x,\mathsf{x, t})$（这里我们假定 $\beta, \alpha$ 两个参数已经给定了）

Bayes 公式给出：

$$
p(t~|~x,\mathsf{x, t}) = \int p(t~|~x,\mathbf w) p(w~|~\mathsf{x,t})d\mathbf w
$$

其中 $p(t~|~x,\mathbf w)$ 是之前给定的高斯分布，$p(w~|~\mathsf{x,t})$ 是训练集上的后验概率（也是一个高斯分布）。

由于高斯分布的性质，上面的式子本质上也是一个高斯分布，因此可以写成

$$
p(t~|~x,\mathsf{x, t})=\mathcal{N}\left(t~|~m(x), s^2(x)\right)
$$

其中均值和方差分别为

$$
\begin{align}
m(x) & = \beta \phi(x)^\text{T} \mathbf S \sum_{n=1}^N \phi(x_n) t_n \\
s^2(x) & = \beta^{-1} + \phi(x)^\top \mathbf S \phi(x)
\end{align}
$$

其中，$\phi_i(x) = x^i, i = 0, \dots, M$，矩阵 $\mathbf S$：

$$
\mathbf S^{-1}=\alpha I +\beta \sum_{n=1}^N \phi(x_n)^\top\phi(x_n)
$$

下图粉红色部分就是 Bayes 估计给出的结果，红色曲线是 `MAP` 给出的结果。

```python
def phi(x, M):
    return x[:,None] ** np.arange(M + 1)

N = 10

# 生成 0，1 之间等距的 N 个 数
x_tr = np.linspace(0, 1, N)

# 计算 t
t_tr = np.sin(2 * np.pi * x_tr) + 0.25 * np.random.randn(N)

# 加正则项的解
M = 9
alpha = 5e-3
beta = 11.1
lam = alpha / beta

phi_x_tr = phi(x_tr, M)
A_0 = phi_x_tr.T.dot(phi_x_tr) + lam * np.eye(M+1)
y_0 = t_tr.dot(phi_x_tr)

# 求解 Aw=y
coeff = np.linalg.solve(A_0, y_0)[::-1]

f = np.poly1d(coeff)

# 绘图

xx = np.linspace(0, 1, 500)

# Bayes估计的均值和标准差
S = np.linalg.inv(A_0 * beta)

m_xx = beta * phi(xx, M).dot(S).dot(y_0)
s_xx = np.sqrt(1 / beta + phi(xx, M).dot(S).dot(phi(xx, M).T).diagonal())

fig, ax = plt.subplots()
ax.plot(x_tr, t_tr, 'co')
ax.plot(xx, np.sin(2 * np.pi * xx), 'g')
ax.plot(xx, f(xx), 'r')
ax.fill_between(xx, m_xx-s_xx, m_xx+s_xx, color="pink")
ax.set_xlim(-0.1, 1.1)
ax.set_ylim(-1.5, 1.5)
ax.set_xticks([0, 1])
ax.set_yticks([-1, 0, 1])
ax.set_xlabel("$x$", fontsize="x-large")
ax.set_ylabel("$t$", fontsize="x-large")

plt.show()
```


Non Linear Regression Analysis

If the data shows a curvy trend, then linear regression will not produce very accurate results when compared to a non-linear regression because, as the name implies, linear regression presumes that the data is linear.
Let's learn about non linear regressions and apply an example on python. In this notebook, we fit a non-linear model to the datapoints corrensponding to China's GDP from 1960 to 2014.

Importing required libraries

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

Though Linear regression is very good to solve many problems, it cannot be used for all datasets. First recall how linear regression, could model a dataset. It models a linear relation between a dependent variable y and independent variable x. It had a simple equation, of degree 1, for example y = $2x$ + 3.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 2*(x) + 3
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
#plt.figure(figsize=(8,6))
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

Non-linear regressions are a relationship between independent variables $x$ and a dependent variable $y$ which result in a non-linear function modeled data. Essentially any relationship that is not linear can be termed as non-linear, and is usually represented by the polynomial of $k$ degrees (maximum power of $x$).

$$ \ y = a x^3 + b x^2 + c x + d \ $$

Non-linear functions can have elements like exponentials, logarithms, fractions, and others. For example: $$ y = \log(x)$$

Or even, more complicated such as :
$$ y = \log(a x^3 + b x^2 + c x + d)$$

Let's take a look at a cubic function's graph.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 1*(x**3) + 1*(x**2) + 1*x + 3
y_noise = 20 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

As you can see, this function has $x^3$ and $x^2$ as independent variables. Also, the graphic of this function is not a straight line over the 2D plane. So this is a non-linear function.

Some other types of non-linear functions are:

### Quadratic

$$ Y = X^2 $$

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

y = np.power(x,2)
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Exponential

An exponential function with base c is defined by $$ Y = a + b c^X$$ where b ≠0, c > 0 , c ≠1, and x is any real number. The base, c, is constant and the exponent, x, is a variable.

```python
X = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

Y= np.exp(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Logarithmic

The response $y$ is a results of applying logarithmic map from input $x$'s to output variable $y$. It is one of the simplest form of **log()**: i.e. $$ y = \log(x)$$

Please consider that instead of $x$, we can use $X$, which can be polynomial representation of the $x$'s. In general form it would be written as
\begin{equation}
y = \log(X)
\end{equation}

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = np.log(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Sigmoidal/Logistic

$$ Y = a + \frac{b}{1+ c^{(X-d)}}$$

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = 1-4/(1+np.power(3, X-2))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

# Non-Linear Regression example

For an example, we're going to try and fit a non-linear model to the datapoints corresponding to China's GDP from 1960 to 2014. We download a dataset with two columns, the first, a year between 1960 and 2014, the second, China's corresponding annual gross domestic income in US dollars for that year.

```python
import numpy as np
import pandas as pd

#downloading dataset
!wget -nv -O china_gdp.csv https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-ML0101EN-Coursera/labs/Data_files/china_gdp.csv

df = pd.read_csv("china_gdp.csv")
df.head(10)
```

    2021-05-29 14:56:29 URL:https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-ML0101EN-Coursera/labs/Data_files/china_gdp.csv [1218/1218] -> "china_gdp.csv" [1]

      Year
      Value

      0
      1960
      5.918412e+10

      1
      1961
      4.955705e+10

      2
      1962
      4.668518e+10

      3
      1963
      5.009730e+10

      4
      1964
      5.906225e+10

      5
      1965
      6.970915e+10

      6
      1966
      7.587943e+10

      7
      1967
      7.205703e+10

      8
      1968
      6.999350e+10

      9
      1969
      7.871882e+10

**Did you know?** When it comes to Machine Learning, you will likely be working with large datasets. As a business, where can you host your data? IBM is offering a unique opportunity for businesses, with 10 Tb of IBM Cloud Object Storage: [Sign up now for free](http://cocl.us/ML0101EN-IBM-Offer-CC)

### Plotting the Dataset

This is what the datapoints look like. It kind of looks like an either logistic or exponential function. The growth starts off slow, then from 2005 on forward, the growth is very significant. And finally, it decelerate slightly in the 2010s.

```python
plt.figure(figsize=(8,5))
x_data, y_data = (df["Year"].values, df["Value"].values)
plt.plot(x_data, y_data, 'ro')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

### Choosing a model

From an initial look at the plot, we determine that the logistic function could be a good approximation,
since it has the property of starting with a slow growth, increasing growth in the middle, and then decreasing again at the end; as illustrated below:

```python
X = np.arange(-5.0, 5.0, 0.1)
Y = 1.0 / (1.0 + np.exp(-X))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

The formula for the logistic function is the following:

$$ \hat{Y} = \frac1{1+e^{\beta_1(X-\beta_2)}}$$

$\beta_1$: Controls the curve's steepness,

$\beta_2$: Slides the curve on the x-axis.

### Building The Model

Now, let's build our regression model and initialize its parameters.

```python
def sigmoid(x, Beta_1, Beta_2):
     y = 1 / (1 + np.exp(-Beta_1*(x-Beta_2)))
     return y
```

Lets look at a sample sigmoid line that might fit with the data:

```python
beta_1 = 0.10
beta_2 = 1990.0

#logistic function
Y_pred = sigmoid(x_data, beta_1 , beta_2)

#plot initial prediction against datapoints
plt.plot(x_data, Y_pred*15000000000000.)
plt.plot(x_data, y_data, 'ro')
```

Our task here is to find the best parameters for our model. Lets first normalize our x and y:

```python
# Lets normalize our data
xdata =x_data/max(x_data)
ydata =y_data/max(y_data)
```

#### How we find the best parameters for our fit line?

we can use **curve_fit** which uses non-linear least squares to fit our sigmoid function, to data. Optimal values for the parameters so that the sum of the squared residuals of sigmoid(xdata, *popt) - ydata is minimized.

popt are our optimized parameters.

```python
from scipy.optimize import curve_fit
popt, pcov = curve_fit(sigmoid, xdata, ydata)
#print the final parameters
print(" beta_1 = %f, beta_2 = %f" % (popt[0], popt[1]))
```

     beta_1 = 690.451715, beta_2 = 0.997207

Now we plot our resulting regression model.

```python
x = np.linspace(1960, 2015, 55)
x = x/max(x)
plt.figure(figsize=(8,5))
y = sigmoid(x, *popt)
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(x,y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

## Practice

Can you calculate what is the accuracy of our model?

```python
# write your code here
# split data into train/test
msk = np.random.rand(len(df)) < 0.8
train_x = xdata[msk]
test_x = xdata[~msk]
train_y = ydata[msk]
test_y = ydata[~msk]

# build the model using train set
popt, pcov = curve_fit(sigmoid, train_x, train_y)

# predict using test set
y_hat = sigmoid(test_x, *popt)

# evaluation
print("Mean absolute error: %.2f" % np.mean(np.absolute(y_hat - test_y)))
print("Residual sum of squares (MSE): %.2f" % np.mean((y_hat - test_y) ** 2))
from sklearn.metrics import r2_score
print("R2-score: %.2f" % r2_score(y_hat , test_y) )
```

    Mean absolute error: 0.03
    Residual sum of squares (MSE): 0.00
    R2-score: 0.96

Double-click **here** for the solution.

Want to learn more?

IBM SPSS Modeler is a comprehensive analytics platform that has many machine learning algorithms. It has been designed to bring predictive intelligence to decisions made by individuals, by groups, by systems – by your enterprise as a whole. A free trial is available through this course, available here: SPSS Modeler

Also, you can use Watson Studio to run these notebooks faster with bigger datasets. Watson Studio is IBM's leading cloud solution for data scientists, built by data scientists. With Jupyter notebooks, RStudio, Apache Spark and popular libraries pre-packaged in the cloud, Watson Studio enables data scientists to collaborate on their projects without having to install anything. Join the fast-growing community of Watson Studio users today with a free account at Watson Studio

Thanks for completing this lesson!

Author:  Saeed Aghabozorgi

Saeed Aghabozorgi, PhD is a Data Scientist in IBM with a track record of developing enterprise level applications that substantially increases clients’ ability to turn data into actionable knowledge. He is a researcher in data mining field and expert in developing advanced analytic methods like machine learning and statistical modelling on large datasets.

| Date (YYYY-MM-DD) | Version | Changed By | Change Description    |
| ----------------- | ------- | ---------- | --------------------- |
| 2020-08-04        | 0       | Nayef      | Upload file to Gitlab |
|                   |         |            |                       |

Copyright © 2018 Cognitive Class. This notebook and its source code are released under the terms of the MIT License.

```python

```


[ADS Entry](http://adsabs.harvard.edu/abs/1998ApJ...497L..17S)

```python
import sympy
sympy.init_printing()
```

Supra - thermal electron distribution

```python
gamma_e = sympy.Symbol('gamma_e')
C_e = sympy.Symbol('C_e')
p = sympy.Symbol('p')
gamma_m = sympy.Symbol('gamma_m')
electron_distribution_ansatz = C_e*gamma_e**(-p)
electron_distribution_ansatz
```

Shocked number density

```python
gamma = sympy.Symbol('gamma')
n = sympy.Symbol('n', positive=True)
shocked_number_density = gamma*n
```

Shocked energy density

```python
m_p = sympy.Symbol('m_p', positive=True)
c = sympy.Symbol('c', positive=True)
shocked_energy_density = gamma**2*m_p*c**2*n
```

```python
m_e = sympy.Symbol('m_e', positive=True)
epsilon_e = sympy.Symbol('epsilon_e', positive=True)
_ = [sympy.integrate(electron_distribution_ansatz,gamma_e,conds='none'),
     sympy.integrate(electron_distribution_ansatz*gamma_e*m_e*c**2,gamma_e,conds='none')]
_ = [-itm.subs(gamma_e,gamma_m).simplify() for itm in _]
_ = [sympy.Eq(_[0],shocked_number_density),
     sympy.Eq(_[1],shocked_energy_density*epsilon_e)]
_ = sympy.solve(_,[C_e,gamma_m])[0]
C_e_expr, gamma_m_expr = _
```

```python
sympy.Eq(gamma_m,gamma_m_expr)
```

Magnetic field strength

```python
epsilon_B = sympy.Symbol('epsilon_B',positive=True)
B = sympy.Symbol('B')
magnetic_field = sympy.sqrt(epsilon_B*shocked_energy_density)
sympy.Eq(B,magnetic_field)
```

Synchrotron radiation power

```python
P_gamma_e = sympy.Symbol(r'P\left(\gamma_e\right)')
sigma_T = sympy.Symbol('sigma_T', positive=True)
synchrotron_radiation_power = sigma_T*c*gamma**2*gamma_e**2*B**2
sympy.Eq(P_gamma_e,synchrotron_radiation_power)
```

Characteristic frequency

```python
nu_gamma_e = sympy.Symbol(r'\nu \left(\gamma_e\right)')
q_e = sympy.Symbol('q_e', positive=True)
characteristic_frequency = gamma*gamma_e**2*q_e*B/(m_e*c)
sympy.Eq(nu_gamma_e, characteristic_frequency)
```

Peak power

```python
P_nu_max = sympy.Symbol(r'P_{\nu,\max}')
peak_power = synchrotron_radiation_power/ characteristic_frequency
sympy.Eq(P_nu_max, peak_power)
```

Critical cooling Lorentz factor

```python
gamma_c = sympy.Symbol('gamma_c')
t = sympy.Symbol('t',positive=True)
_ = sympy.Eq(gamma*gamma_c*m_e*c**2,
             P_gamma_e*t)
_ = _.subs(P_gamma_e, synchrotron_radiation_power)
_ = _.subs(gamma_e,gamma_c).simplify()
critical_lorentz_factor = sympy.solve(_,gamma_c)[1]
sympy.Eq(gamma_c, critical_lorentz_factor)
```

Adiabatic shock trajectory

```python
E = sympy.Symbol('E',positive=True)
R = sympy.Symbol('R')
adiabatic_shock_energy = gamma**2*R**3*n*m_p*c**2
sympy.Eq(E,adiabatic_shock_energy)
```

```python
adiabatic_shock_trajectory = sympy.solve(adiabatic_shock_energy-E,gamma)[1]
sympy.Eq(gamma,adiabatic_shock_trajectory)
```

Relativistic Sedov Length

```python
L = sympy.Symbol('L', positive=True)
gamma_0 = sympy.Symbol('gamma_0', positive=True)
relativistic_sedov_length = sympy.solve(sympy.Eq(E,L**3*n*m_p*c**2*gamma_0),L)[0]
sympy.Eq(L,relativistic_sedov_length)
```

Raditative shock trajectory

```python
radiative_shock_trajectory = (R/L)**(-3)
sympy.Eq(gamma,radiative_shock_trajectory)
```

Observers time

```python
observer_time = R/(c*gamma**2)
sympy.Eq(t,observer_time)
```

Equation 9 - adiabatic case

```python
_ = adiabatic_shock_trajectory
_ = sympy.solve(_-sympy.solve(observer_time-t,gamma)[1],R)[1]
adiabatic_radius_time = _
sympy.Eq(R,adiabatic_radius_time)
```

Equation 9 - radiative case

```python
_ = radiative_shock_trajectory
_ = sympy.solve(_-sympy.solve(observer_time-t,gamma)[1],R)[0]
radiative_radius_time = _
sympy.Eq(R,radiative_radius_time)
```

Equation 10 - adiabatic case

```python
_ = adiabatic_shock_trajectory.subs(R,adiabatic_radius_time)
adiabatic_gamma_time = sympy.expand_power_base(_,force=True)
sympy.Eq(gamma, adiabatic_gamma_time)
```

Equation 10 - radiative case

```python
_ = radiative_shock_trajectory.subs(R,radiative_radius_time)
radiative_gamma_time = _
sympy.Eq(gamma,radiative_gamma_time)
```

Adiabatic critical frequency

```python
nu_c = sympy.Symbol('nu_c')
_ = characteristic_frequency.subs(gamma_e, critical_lorentz_factor)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma,adiabatic_gamma_time)
_ = sympy.expand_power_base(_,force=True)
adiabatic_nu_c = _
sympy.Eq(nu_c,adiabatic_nu_c)
```

Adiabatic minimal frequency

```python
nu_m = sympy.Symbol('nu_m')
_ = characteristic_frequency
_ = _.subs(gamma_e, gamma_m_expr)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma, adiabatic_gamma_time)
_ = sympy.expand_power_base(_,force=True)
adiabatic_nu_m = _
sympy.Eq(nu_m, adiabatic_nu_m)
```

Adiabatic peak flux

```python
D = sympy.Symbol('D')
F_nu_max = sympy.Symbol(r'F_{\nu, \max}')
_ = peak_power*n*R**3/D**2
_ = _.subs(R, adiabatic_radius_time)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma,adiabatic_gamma_time)
_ = sympy.expand_power_base(_,force=True)
adiabatic_peak_flux = _
sympy.Eq(F_nu_max, adiabatic_peak_flux)
```

Radiative critical frequency

```python
_ = characteristic_frequency.subs(gamma_e, critical_lorentz_factor)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma,radiative_gamma_time)
_ = _.subs(L,relativistic_sedov_length)
_ = sympy.expand_power_base(_,force=True)
radiative_nu_c = _
sympy.Eq(nu_c,radiative_nu_c)
```

Radiative minimal frequency

```python
_ = characteristic_frequency
_ = _.subs(gamma_e, gamma_m_expr)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma, radiative_gamma_time)
_ = _.subs(L,relativistic_sedov_length)
_ = sympy.expand_power_base(_,force=True)
radiative_nu_m = _
sympy.Eq(nu_m, radiative_nu_m)
```

Radiative peak flux

```python
_ = peak_power*n*R**3/D**2
_ = _.subs(R, radiative_radius_time)
_ = _.subs(B,magnetic_field)
_ = _.subs(gamma,radiative_gamma_time)
_ = _.subs(L,relativistic_sedov_length)
_ = sympy.expand_power_base(_,force=True)
radiative_peak_flux = _
sympy.Eq(F_nu_max, radiative_peak_flux)
```

Adiabatic transition time

```python
t_0 = sympy.Symbol('t_0')
_ = sympy.Eq(adiabatic_nu_c,
             adiabatic_nu_m)
adiabatic_transition_time = sympy.solve(_,t)[0]
sympy.Eq(t_0, adiabatic_transition_time)
```

Radiative transition time

```python
xi = sympy.Symbol('xi',positive=True)
_ = sympy.Eq(radiative_nu_c,
            radiative_nu_m)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = sympy.solve(sympy.expand_power_base(_.rhs/_.lhs,force=True)-1,t)[-1]
_ = _.subs(xi,(p-2)/(p-1))
radiative_transition_time = _
sympy.Eq(t_0, radiative_transition_time)
```

Critical time for a given frequency, adiabatic case

```python
t_c = sympy.Symbol('t_c')
nu = sympy.Symbol('nu', positive=True)
_ = sympy.Eq(nu,adiabatic_nu_c)
_ = sympy.solve(_,t)[0]
adiabatic_t_c_nu = _
sympy.Eq(t_c, adiabatic_t_c_nu)
```

Critical time for a given frequency, radiative case

```python
_ = sympy.Eq(nu,radiative_nu_c)
_ = sympy.solve(_,t)[0]
_ = sympy.expand_power_base(_,force=True)
radiative_t_c_nu = _
sympy.Eq(t_c, radiative_t_c_nu)
```

$t_m$ for the adiabatic case

```python
t_m = sympy.Symbol('t_m')
_ = sympy.Eq(nu,adiabatic_nu_m)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = sympy.solve(_,t)[0]
_ = _.subs(xi,(p-2)/(p-1))
adiabatic_t_m_nu = _
sympy.Eq(t_m, adiabatic_t_m_nu)
```

$t_m$ for the radiative case

```python
_ = sympy.Eq(nu,radiative_nu_m)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = sympy.solve(_,t)[0]
_ = _.subs(xi,(p-2)/(p-1))
radiative_t_m_nu = _
sympy.Eq(t_m, radiative_t_m_nu)
```

$\nu_0$ for the adiabatic case

```python
nu_0 = sympy.Symbol('nu_0')
_ = adiabatic_nu_c.subs(t,adiabatic_transition_time)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = _.subs(xi,(p-2)/(p-1))
adiabatic_nu_0 = _
sympy.Eq(nu_0,_)
```

$\nu_0$ for the radiative case

```python
_ = radiative_nu_c.subs(t,radiative_transition_time)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = _.subs(xi,(p-2)/(p-1))
radiative_nu_0 = _
sympy.Eq(nu_0,_)
```

Transition from radiative to adiabatic regime

```python
_ = gamma**2*R**3*n*m_p*c**2
_ = _.subs(R, radiative_radius_time)
_ = _.subs(gamma, radiative_gamma_time)
_ = _.subs(L, relativistic_sedov_length)
_ = _.subs(t, radiative_transition_time)
_ = _.subs(sympy.solve(sympy.Eq(xi,(p-2)/(p-1)))[0]).simplify()
_ = _.subs(xi,(p-2)/(p-1))
_
```

```python

```


# 5. Active Learning

 * Make big data algorithms ask when they're unsure.
 * Avoid silly questions and ask questions which promise to yield the most information.
     - Want to minimize number of requested labels (expensive).

## Overview
Two main approaches:
 * **Stream-based (online) active learning**: need to decide on the spot whether to request a label for a data point
 * **Pool-based active learning**: sequentially request point labels from an unlabeled dataset

### Simple example
Learning a 1D threshold function or a liner threshold function.

\begin{equation}
    H = \left\{ h : [0, 1] \mapsto \{+1, -1\}, h(x) = \operatorname{sign}(x - \tau), \tau \in [0, 1] \right\}
\end{equation}

## Pool-based active learning
 * request labels intelligently until we can infer all remaining labels
 * efficiency varies (can reduce nr. of requested labels exponentially, or have almost no impact)

### Uncertainty sampling
 * Assign uncertainty score to all points and pick most uncertain example. Widely used heuristic.
 * SVM: pick point closest to separating hyperplane: $x^{*} = \operatorname{arg min}_{x_i \in \mathcal{U}} | w^T x_i |$
 * Classifier retrained after every new label (Can we avoid this?).
 * Complexity to pick $m$ labels: $O\left( n \cdot m \cdot d + \frac{m^2}{\epsilon} \right)$, with $n$ datapoints (need to compute uncertainty score for each), $m$ max requested labels, $d$ dimensions and $\epsilon$ our error threshold (TODO please confirm).

Note: AUROC = Area Under ROC (receiver operating characteristic) Curve; measures the performance of a binary classifier system as its discrimination threshold is varied (i.e. it's big when a classifier has a lot of true positives vs. false positives).

## Sub-linear time active learning
 * Store mappings from hyperplanes (hyperplane query) to close points.
 * Hyperplane query = give me the closest point(s) to this hyperplane. TODO: why more?
 * This helps us avoid recomputing uncertainty scores every time we update our model (i.e. the separating hyperplane).
 * The closer a point's vector is to be perpendicular to the normal $w$, the more likely it is to be close to it. There are exceptions, of course, but most of the time this rule works very well.

So how can we hash these queries?

Use two random vectors, $u$ and $v$.

\begin{equation}
    h_{u, v}(a, b) = [h_u(a), h_v(b)] = [\operatorname{sign}(u^{T} a), \operatorname{sign}(v^{T} b)], \quad u, v \sim \mathcal{N}(0, I)
\end{equation}

Define a hash family (why?):

\begin{equation}
h_{\mathcal{H}}(z) = \left\{
 \begin{array}
        h_{u, v}(z, z), \> & \text{if } z \text{ is a database point vector} \\
        h_{u, v}(z, -z), \> & \text{if } z \text{ is a query hyperplane vector} \\
    \end{array}
    \right.
\end{equation}

With this we get an LSH collision probability:

\begin{equation}
\begin{aligned}
    Pr[h_H(w) & = h_H(x)] = Pr[h_u(w) = h_u(x)] Pr[h_v(-w) = h_v(x)] \\
    & = \frac{1}{4} - \frac{1}{\pi^2}\left( \theta_{x, w} - \frac{\pi}{2}\right)^2
\end{aligned}
\end{equation}

(A collision being a query hyperplane vector and another point get put into the same bucket, representing the fact that they are close, and that its label should be requested)

For small enough $\theta_{x, w}$ this is OK but still not great. However, we can use multiple hash functions together in an OR-construction to increase the number of positives. (TODO: not in slides but logical; find reference)

## Improving active learning

Key insight/problem: uncertain $\nRightarrow$ informative

Better to quantify how much information we get from each label, rather than just how uncertain it is.

### Version space
A **version space** is a set of all classifiers consistent with the current data.

\begin{equation}
\mathcal{V}(D) = \{ w : \forall \> (x, y) \in D, \operatorname{sign}(w^Tx) = y \}
\end{equation}

Note: this does not account for noisy data.

In an SVM case, the original version space could be the whole $l_2$-norm ball of radius $\frac{1}{\sqrt{\lambda}}$.

We want to always request a label for the point which will make our version space shrink the most.

### Relevant version space

A **relevant version space**, given a pool of unlabeled points, is the set of possible labeling consistent with the data.

\begin{equation}
\hat{\mathcal{V}}(D; U) = \left\{ h: U \rightarrow \{+1, -1\} : \exists w \in \mathcal{V}(D) : \forall \> x \in U, \operatorname{sign}(w^Tx) = h(x) \right\}
\end{equation}

We use this to conduct a *generalized binary search* to find the most informative point.

$v^{+}(x)$ - how many elemenents we would have in the relevant version space if x were labelled +.
$v^{-}(x)$ - same, but if x were labelled -.
Pick $x$ where the minimum possible change $\min{(v^{-}(x), v^{+}(x))}$ is the largest. This is the x which, regardless of what label it will get, will shrink the relevant version space the most ("balanced" split). (TODO: clarify)

In other words for each possible label (+ or -) of an uncertain point, calculate resulting SVMs with margings $m^{+}$ and $m^{-}$. Pick point which produces most "balanced" difference between margins:

 * Max-min margin: $x_t = \operatorname{arg max}_x \left\{ \min \{ m_x^-, m_x^+ \} \right\}$ (used in generalized binary search (GBS) example)
 * Ratio margin: $x_t = \operatorname{arg max}_x \left\{ \min \left\{ \frac{m_x^-}{m_x^+}, \frac{m_x^+}{m_x^-} \right\} \right\}$

Wait, "calculate resulting SVM**s**"? Isn't that expensive?

It is. Optimal max-min margins and ratio margins are expensive to compute. But there exist some practical tricks:

 * Only score and pick from *small random subsample* of data
 * Only use informativeness at first (e.g. first 10 examples), then switch to (faster) uncertainty sampling
 * Occasionally "cheat" and pick points *uniformly at random*

## Dealing with noise

Thus far, we assumed there was **no noise**. How to deal with noise?

 * In practice, it doesn't matter (SVM + slack variables)
 * In theory, the analysis is **much harder**.

## Open questions
 * Uncertainty sampling for SVMs: What about kernelized SVMs? Can you still tell what point(s) are closest to the decision boundary (easily)? (Note: just picking the smallest kernel function value between all available training points and the new one doesn't work.)

```python

```


# Introduction to the Finite Element Method (FEM)

#### *Marco Lúcio Bittencourt* (mlb@fem.unicamp.br) and *Cláudio Alessandro de Carvalho Silva* (cacs@thorus-scisoft.com.br)

In the previous classes, we looked into the Finite Difference Method (FDM) and the Finite Volume Method (FVM). Now, we will introduce a numerical method called the Finite Element Method (FEM).

In our discusssion about the Finite Element Method, we will deal with the 1-D steady state diffusion equation, present in several engineering applications. In Heat Transfer, it describes steady state conduction. In Structural Engineering, that is the equation of deformation of axially loaded bars.

## The Finite Element Method (FEM)

- The FEM is a numerical procedure for the determination of approximate solution for Boundary Value Problems (BVP).
- The **strong form** of a BVP corresponds to the differential equation plus the boundary conditions of the problem at hand.
- A solution of the strong form is also a solution of the integral form of the BVP or **weak form**.
- In a **weak formulation**, an equation is no longer required to hold absolutely and has instead *weak solutions*. Integrating the weak form by parts (applying the divergence theorem by Gauss or Green’s formula) provides the numerical benefit of reduced differentiation order.
- Using FEM  we are able to obtain an approximate solution of the **weak form** associated to the BVP.
- The approximate solution consists of a linear combination of interpolation polynomial functions.
- There are different versions of FEM according to the type of interpolation functions used.

#### *h* and *p* refinements.

## Equations for Axially Loaded Bars: Strong Formulation

#### Axially loaded bar.

The static equilibrium principle leads to a BVP in terms of the normal force:
\begin{equation}
\left\{ \begin{array}{l}
\frac{dN_x(x)}{dx} = -q_x(x)\\
N_x(x=0)=-P_0\\
N_x(x=L)=P_L
\end{array}\right.
\end{equation}

The classic cinematic assumption for axially loaded bars composed by linear material and subjected to small displacements and strains (plane sections remain plane after loading) leads to the differential equation in terms of displacements for $x\in(0,L)$
\begin{equation}
\frac{d}{dx}\underbrace{\left(E(x)A(x)\frac{du_x(x)}{dx}\right)}_{N_x(x)} = -q_x(x)\;\;\;\;
%\frac{dN_x(x)}{dx} = -q_x(x)
EA\frac{du_x^2(x)}{dx^2}=-q_x(x)
\end{equation}
with kinematic and natural boundary conditions respectively in terms of $u_x(x)$ and $N_x(x)$. For the bar illustrated below we have
\begin{equation}
u_x(x=0) = 0 \;\;\;\;\mbox{ and }\;\;\;\; N_x(x=L)=P
\end{equation}

Cinematic and natural boundary conditions.

## Example: Solving the Strong Formulation

Consider a bar subjected to a distributed constant axial load $q_x(x)=q_0$ and the boundary conditions shown in the following figure.

###

A function $u_x(x)$ is called **strong solution** if it is defined for any $0\leq x \leq L$, has continuous derivatives up to second order (thus, $u_x(x)\in C^2[0,L]$) and is a solution of the BVP:

- Differential equation
    \begin{equation}
    EAu_x''(x)=-q_0
    \end{equation}
- Boundary conditions: $u_x(x=0)=0$ and $N_x(x=L)=P$.

#### Integrating the differential equation:
1. First integration: normal force
    \begin{equation}
     N_x(x) = EAu_x'(x)=-q_0x+C_1
    \end{equation}
2. Second integration: axial displacement
    \begin{equation}
    EAu_x(x) = -\frac{q_0}{2}x^2+C_1x+C_2
    \end{equation}
3. Computing constants $C_1$ and $C_2$
    \begin{equation}
     N_x(x=L) = -(q_0)(L)+C_1=P\;\rightarrow\;C_1=P+q_oL
    \end{equation}
    \begin{equation}
    u_x(x=0) = \frac{1}{EA}\left[-\frac{q_0}{2}    (0)^2+C_1(0)+C_2\right]=0\;\rightarrow\;C_2=0
    \end{equation}

#### Final solution
- Normal force
    \begin{equation}
    N_x(x) = -q_0(x-L)+P
    \end{equation}
- Axial displacement
    \begin{equation}
    u_x(x) = \frac{1}{EA}\left(-\frac{q_0}{2}x^2+(P+q_oL)x\right)
    \end{equation}
- Support reaction
    \begin{equation}
    R=-N_x(x=0)=-(P+q_oL)
    \end{equation}

Let´s assume some numerical values and plot the solution:

```python
import numpy
from matplotlib import pyplot
%matplotlib inline
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

```python
L = 1    # length L=1m
q0=100   # Distributed load q0=100 N/m
P=200    # Normal force at the end of the bar.
nx=51    #
x=numpy.linspace(0,L,nx)
N=-q0*(x-L)+P                 # Internal distribution of normal force
EAu=-(q0/2)*x**2+(P+q0*L)*x   # Displacement times EA
```

```python
pyplot.figure(figsize=(12,3))
pyplot.subplot(121)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'$N(x)$', fontsize=18)
pyplot.title('Normal Force Diagram', fontsize=18)
pyplot.plot(x,N, 'b-', lw=2);

pyplot.subplot(122)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'$EAu(x)$', fontsize=18)
pyplot.title('Displacement', fontsize=18)
pyplot.plot(x,EAu, 'r-', lw=2);
```

## Equations for Axially Loaded Bars: Weak Formulation

Multiplying the diferential equation in terms of displacements by a test function $v_x(x)$ and integrating over the length of the bar, we obtain the statement
\begin{equation}
\int_0^L \frac{d}{dx}\left(E(x)A(x)\frac{du_x(x)}{dx}\right)v_x(x)dx = -\int_0^L q_x(x) v_x(x) dx
\end{equation}

Integrating the previous expression by parts, we have
\begin{equation}
-\int_0^L E(x)A(x)u_x'v_x'dx + \left.\underbrace{E(x)A(x)u_x'(x)}_{N_x(x)}v_x(x)\right|_0^L =
-\int_0^L q_x(x)v_x(x) dx
\end{equation}

and rearranging the terms
\begin{equation}
\int_0^L E(x)A(x)u_x'(x)v_x'(x)dx = \int_0^L q_x(x)v_x(x) dx + N_x(L)v_x(L) - N_x(0)v_x(0)
\end{equation}

Using the boundary conditions in terms of normal forces, we obtain
\begin{equation}
\int_0^L E(x)A(x)u_x'(x)v_x'(x)dx = \int_0^L q_x(x)v_x(x) dx + P_Lv_x(L) + P_0v_x(0)
\end{equation}

The expression above is a equilibrium integral equation involving only first order derivatives of $u_x(x)$ and $v_x(x)$. The ogirinal operator was thus "weakened" and then that expression is called **weak form** of the original BVP.

The test function $v_x(x)$ must satisfy the corresponding homogeneous boundary conditions. In the preceeding example: $v_x(0)=0$.

In FEM, the approximation $u_n$ of $u_x$ is obtained through a linear combination of global polynomial interpolation functions $\{\phi_j(x)\}_{j=1}^n$
\begin{equation}
u_x(x) \approx u_{n}(x) = \sum_{j=1}^n a_j\phi_j(x)\rightarrow
u_{n}'(x) = \sum_{j=1}^n a_j\phi_j'(x)
\end{equation}
where $a_j\;(j=1,\ldots,n)$ are the coefficients to be determined.

The same interpolation functions are used for $v_x(x)$ in the Galerkin method. Therefore,
\begin{equation}
v_x(x) \approx v_{n}(x) = \sum_{i=1}^n b_i\phi_i(x)\rightarrow
v_{n}'(x) = \sum_{i=1}^n b_i\phi_i'(x)
\end{equation}

Substituting both approximations in the weak form, we have
\begin{equation}
\sum_{i,j=1}^n \left[ \left( \int_0^L E(x)A(x)\phi_i'(x)\phi_j'(x) dx\right)a_j
- \int_0^L q_x\phi_i(x) dx - P_L\phi_i(L) - P_0\phi_i(0) \right] b_i = 0
\end{equation}

Note that the $b_i$ are arbitrary coefficients and then the term inside brackets in the previous expression must be null for all $i=1,\ldots,n$. In other words,
\begin{equation}
\sum_{j=1}^n\left( \underbrace{\int_0^L E(x)A(x)\phi_i'(x)\phi_j'(x)dx}_{K_{ij}}\right)a_j =
\underbrace{\int_0^L q_x(x)\phi_i(x) dx + P_L\phi_i(L) + P_0\phi_i(0)}_{f_i}
\end{equation}
which constitutes a system of linear equations for $i,j=1,\ldots,n$
\begin{equation}
K_{ij}a_j=f_i
\end{equation}
In matricial form
\begin{equation}
[K]\{a\}=\{f\}
\end{equation}

In the present context of Structural Engineering, $[K]$ is the *stiffness matrix*; $\{a\}$ is the *vector of approximation coefficients* and $\{f\}$ is the *vector of equivalent forces*.

It is easily observed that the formulation is fundamentally associated to the selection of the interpolation functions $\{\phi_j(x)\}_{j=1}^n$.

## An Approximation of the Weak Formulation

For $n=2$ we have two interpolation functions and the axial displacement approximation is
\begin{equation}
u_n(x) = a_1\phi_1(x)+a_2\phi_2(x) = [\begin{array}{cc} \phi_1(x) & \phi_2(x)]
\end{array} \left\{ \begin{array}{c} a_1 \\ a_2 \end{array} \right\} = [N]\{a\}
\end{equation}
where $[N]=[\phi_1(x)\;\; \phi_2(x)]$ is the matrix of interpolation funcions.

We use the following linear interpolation functions (also called basis functions) and their derivatives:
\begin{equation}
\phi_1(x) = 1-\frac{x}{L} \;\;\rightarrow \;\; \phi_1'(x) = -\frac{1}{L} \;\;\;\;\mbox{and}\;\;\;\;
\phi_2(x) = \frac{x}{L} \;\; \rightarrow \;\;  \phi_2'(x) = \frac{1}{L}
\end{equation}

Using some Python code we can plot those functions:

```python
phi1=1-x/L
dphi1=numpy.ones(nx)
dphi1[0 : ]=-1/L
phi2=x/L
dphi2=numpy.ones(nx)
dphi2[0 : ]=1/L
#
pyplot.figure(figsize=(12,3))
#
pyplot.subplot(121)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'$\phi_i(x)$', fontsize=18)
pyplot.tick_params(axis='both', labelsize=12)
pyplot.plot(x,phi1, 'r-', lw=2, label='$\phi_1(x)$')
pyplot.plot(x,phi2, 'b-', lw=2, label='$\phi_2(x)$')
pyplot.text(0.2, 0.85, r'$\phi_1(x)$')
pyplot.text(0.7, 0.85, r'$\phi_2(x)$');
#
pyplot.subplot(122)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=16)
pyplot.ylabel(r'$d\phi_i(x)$', fontsize=16)
pyplot.plot(x,dphi1, 'r-', lw=2)
pyplot.plot(x,dphi2, 'b-', lw=2)
pyplot.tick_params(axis='both', labelsize=12)
pyplot.ylim(-1.2,1.2)
pyplot.text(0.2, -0.9, r'$d\phi_1(x)$')
pyplot.text(0.7, 0.8, r'$d\phi_2(x)$');

```

Now, with a defined set of interpolation functions, let´s check the meaning of the other terms of this formulation.

The approximation coefficients associated to the considered interpolation functions can be computed evaluating the displacement at both ends:
\begin{equation}
u_n(0) = a_1\underbrace{\phi_1(0)}_1 +a_2\underbrace{\phi_2(0)}_0 = a_1 = \bar{u}_1
\end{equation}
\begin{equation}
u_n(L) = a_1\underbrace{\phi_1(L)}_0+a_2\underbrace{\phi_2(L)}_1 = a_2 = \bar{u}_2
\end{equation}
Therefore, they correspond to the displacements at the ends of the bar.

The approximate normal strain is $\epsilon_n(x) = u_n'(x)$, that is,
\begin{equation}
\epsilon_n(x) = a_1\phi_1'(x)+a_2\phi_2'(x) = [\begin{array}{cc} \phi_1'(x) & \phi_2'(x)]
\end{array} \left\{ \begin{array}{c} a_1 \\ a_2 \end{array} \right\} = [B]\{a\}
\end{equation}
$[B]=\frac{1}{L}[-1\;\; 1]$ is called *strain matrix*.

For $EA=\mathrm{cte}$, the stiffness matrix is given by
\begin{eqnarray*}
[K] &=& EA \left[ \begin{array}{cc}
\int_0^L \phi'_1(x)\phi'_1(x)dx & \int_0^L \phi'_1(x)\phi'_2(x)dx \\
\int_0^L \phi'_1(x)\phi'_2(x)dx & \int_0^L \phi'_2(x)\phi'_2(x)dx
\end{array} \right] \\
&=& \frac{EA}{L} \left[ \begin{array}{rr} 1 & -1 \\ -1 & 1 \end{array} \right] \\
&=&{\int_0^L} [B]^T[D][B]Adx\;\;\;\;\;\;\;[D]=E
\end{eqnarray*}

For a distributed load $q_x(x)=q_0$, the vector of equivalent forces is
\begin{eqnarray*}
\{f\} &=& \left\{ \begin{array}{c}
\int_0^L q_0\phi_1(x)dx + P_0\phi_1(0) + P_L\phi_1(L)\\
\int_0^L q_0\phi_2(x)dx + P_0\phi_2(0) + P_L\phi_2(L)
\end{array} \right\} \\
&=& \left\{ \begin{array}{c}
q_0L/2 + P_0\\ q_0L/2 + P_L
\end{array} \right\} = \int_0^L q_x(x)[N]^Tdx + \left\{ \begin{array}{c}
P_0\\ P_L
\end{array} \right\}
\end{eqnarray*}

Finally, we obtain the system of linear equations
\begin{equation}
\frac{EA}{L} \left[ \begin{array}{rr} 1 & -1 \\ -1 & 1 \end{array} \right]
\left\{ \begin{array}{c} \bar{u}_1 \\ \bar{u}_2 \end{array} \right\} =
\left\{ \begin{array}{c}
q_oL/2 + P_0\\ q_oL/2 + P_L
\end{array} \right\}
\end{equation}

## Example: Approximate Solution for the Weak Formulation

Applying the boundary conditions of the last example to the system of equations above we have

\begin{equation}
\frac{EA}{L} \left[ \begin{array}{rr} 1 & -1 \\ -1 & 1 \end{array} \right]
\left\{ \begin{array}{c} 0 \\ \bar{u}_2 \end{array} \right\} =
\left\{ \begin{array}{c}
q_0L/2 + R\\ q_0L/2 + P
\end{array} \right\}
\end{equation}
where $R$ is the reaction force in the support at $x=0$.

Due to the boundary condition at $x=0$, $\bar{u}_1=0$, the systems of equations reduces to
\begin{equation}
\frac{EA}{L}{\bar{u}_2} = -q_0L/2 - {R},\qquad \frac{EA}{L}{\bar{u}_2} = q_0L/2 + P
\end{equation}
The first equation has two unknowns: $\bar{u}_2$ and $R$, where $\bar{u}_2$ is the displacement at $x=L$.

Solving the second equation, we have
\begin{equation}
\bar{u}_2 = \frac{L}{EA}\left(q_0L/2 + P\right)
\end{equation}

Then we can solve the first equation for $R$:
\begin{equation}
R = -(q_0L + P)
\end{equation}

The exact solution (from the last example) and the approximate solution (computed here) for this problem are
\begin{equation}
u_x(x) = \frac{1}{EA}\left(Px+q_0Lx-\frac{q_0}{2}x^2\right)\;\;
\end{equation}
\begin{equation}
u_{2}(x) =\frac{1}{EA}\left(Px+\frac{q_0L}{2}x\right)
\end{equation}

The *absolute error function* is
\begin{equation}
e(x)=|u_x(x)-u_{2}(x)| = \left|\frac{1}{EA}\left(\frac{q_0L}{2}x - \frac{q_0}{2}x^2\right)\right|
\end{equation}

The *relative error* is given by
\begin{equation}
e_r = \frac{||e(x)||}{||u_x(x)||} = \frac{\sqrt{\int_0^L e^2(x)dx}}{\sqrt{\int_0^L u_x^2(x)dx}}
\end{equation}

A comparison of the exact and approximate solutions is given below:

```python
EAu2=(P+q0*L/2)*x
err=EAu-EAu2

pyplot.figure(figsize=(16,3))

pyplot.subplot(121)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'Axial displacement', fontsize=14)
pyplot.tick_params(axis='both', labelsize=12)
pyplot.plot(x,EAu, 'r-', lw=2, label='$u_x(x)$')
pyplot.plot(x,EAu2, 'b-', lw=2, label='$u_2(x)$')
pyplot.legend(bbox_to_anchor=(0.35,1));
pyplot.subplot(122)
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'Absolut error', fontsize=14)
pyplot.tick_params(axis='both', labelsize=12)
pyplot.plot(x,err, 'g-', lw=2, label='$e(x)$')
pyplot.legend(bbox_to_anchor=(0.3,1));
```

### Computing the relative error $e_r$:

```python
from scipy import integrate
from math import sqrt
mod1=numpy.zeros(nx)
mod2=numpy.zeros(nx)
for i in range(nx):
    mod1[i]=sqrt(err[i]**2)
    mod2[i]=sqrt(EAu[i]**2)
I1=integrate.simps(mod1,x)
I2=integrate.simps(mod2,x)
rerr=I1/I2  #relative error
print('The relative error of the approximate solution of the weak form is {:.4f}.'.format(rerr))
```

    The relative error of the approximate solution of the weak form is 0.0625.

## A Bar Finite Element

Let a bar of length $L$ be divided in four segments by the introduction of three internal points as in the figure:

We can associate each of the following interpolation functions $\phi_i$ to node $i$:

The general expression of functions $\phi_i$ is

\begin{equation}
\phi _{i}(x)=\left\{
\begin{array}{rrl}
0 &  & x\notin \left[ x_{i-1},x_{i+1}\right]  \\
\dfrac{x-x_{i-1}}{x_{i}-x_{i-1}} &  & x\in \left[ x_{i-1},x_{i}\right]  \\
-\dfrac{x-x_{i+1}}{x_{i+1}-x_{i}} &  & x\in \left[ x_{i},x_{i+1}\right]
\end{array}
\right.
\end{equation}

Note that each of those functions is nonzero on only one segment of the bar. That segment is called the *support* of $\phi_i$. Those functions are then called *compact support functions*. That very characteristic of compact supportness allows us to define the **concept of finite element**:
#### *The Finite Element Method offers a sistematic procedure for building polynomial compact support shape functions.*

1. Consider the partitioned bar according to the figure above. Associate to each bar segment $i$, $i=1,\ldots,4$, the functions $\phi_{i-1}(x)$ and $\phi_i(x)$ to define the **finite element**:

2. The interpolation functions can now the written in local terms for each element as
\begin{equation}
\phi _{i-1}^{e}(x)= -\frac{x-x_i}{h_e} \qquad\text{and}\qquad
\phi _i^{e}(x)= \frac{x-x_{i-1}}{h_e}
\end{equation}

3. The approximations of the axial displacement and strain on the finite element $e$ are
\begin{equation}
u_{n}^{e}(x)=a_{i-1}^{e}\phi _{i-1}^{e}(x)+a_{i}^{e}\phi _{i}^{e}(x) = [N^e]\{a^e\}.
\end{equation}
\begin{eqnarray*}
\frac{du_{n}^{e}(x)}{dx} &=&
\left[ \begin{array}{cc} \frac{d\phi _{i-1}^{e}}{dx} & \frac{d\phi _i^{e}}{dx} \end{array} \right]
\left\{ \begin{array}{c} a_{i-1}^{e} \\ a_{i}^{e} \end{array} \right\} \\ &=&
\dfrac 1{h_e}\left[ \begin{array}{cc} -1 & 1 \end{array} \right]
\left\{ \begin{array}{c} a_{i-1}^{e} \\ a_{i}^{e} \end{array} \right\} = [{B}^{e}]\{a^{e}\}
\end{eqnarray*}

4. The element stiffness matrix is now given by
\begin{equation}
[K^e] = \int_{x_{i-1}}^{x_i} [B^e]^T[D][B^e]Adx =
\frac{EA}{h_e} \left[ \begin{array}{rr} 1 & -1 \\ -1 & 1 \end{array} \right]
\end{equation}

5. **The superposition of local functions generates the global interpolation functions.** That superposition of local functions translate to the assembly of the global linear system of equations.

## Example: Applying the Finite Element Method to the Axially Loaded Bar

Consider the same bar of the two preceeding examples as shown in the figure. We will apply the procedure described above: the domain $0\leq x\leq L$ will be divided in four segments and the steps 1 to 5 will be applied.

The next snippets of Python code implement a simple FEM procedure for solving this problem.

```python
## Physical data
Lb=1.0        # bar length
E  = 210e9       # Young modulus
A  = 1e-4        # cross sectional area
EA = E*A
p0 = 100         # distributed load
Pc = 200         # concentrated load at x = L, b.c. at x = L

## FEM setup: all element with the same length
Nel   = 4        # number of elements
Nvert = Nel+1    # number of nodes (1-D, linear interpolation)
he    = Lb/Nel   # element lenght
Ndofs = Nel+1    # number of d.o.f. (1 d.o.f. per node)

# Allocation of the stifness matrix
Kg = numpy.zeros((Ndofs,Ndofs))

# Allocation of the load vector
Fg = numpy.zeros(Ndofs)

# Allocation of the solution vector
ug = numpy.zeros(Ndofs)

# Allocation of incidence matrix
Incid = numpy.zeros((Nel,2), dtype=numpy.int)

# Auxiliary matrix for accessing submatrix (more comments below)
TIncid = numpy.zeros((2,1), dtype=numpy.int)

# Nodes coordinates
X = numpy.linspace(0, Lb, Nvert)

# Bar element stiffness matrix
Ke = (EA/he)*numpy.array([[1,-1],[-1,1]])

# Bar element load vector due to the distributed load p0
Fe = (p0*he/2)*numpy.array([1, 1])

# Assembly of global matrix and global load vector without b.c.'s
for e in range(0,Nel):
	Incid[e,:]=numpy.array([e,e+1])
	Fg[Incid[e,:]]=Fg[Incid[e,:]]+Fe
	# numpy requires a column matrix to specify submatrix columns
	TIncid[:,0]=Incid[e,:]
	# The general mechanism to access submatrices in numpy returns the
    # transpost of the submatrix. Despite all matrices being symmetric
	# I am transposing Ke to be coherent.
	Kg[Incid[e,:], TIncid] = Kg[Incid[e,:], TIncid] + numpy.transpose(Ke);

# Application of force b.c. at x = L
Fg[-1] = Fg[-1] + Pc

# Application of b.c. disp. condition at x = 0 (elimination of the first line
# and first column of the global system) and linear system solution for ug
ug[1:] = numpy.linalg.solve( Kg[1:,1:],Fg[1:] )

print('Displacement solution:',ug)
```

    Displacement solution: [  0.00000000e+00   3.42261905e-06   6.54761905e-06   9.37500000e-06
       1.19047619e-05]

```python
u=(1/EA)*EAu # exact solution of the strong form

pyplot.figure(figsize=(8,3))
pyplot.grid(True)
pyplot.xlabel(r'$x$', fontsize=18)
pyplot.ylabel(r'Axial displacement', fontsize=14)
pyplot.tick_params(axis='both', labelsize=12)
pyplot.title('Exact and FEM solutions for the bar problem', fontsize=18)
pyplot.plot(x,u, 'r-', lw=2, label='$u_x(x)$')
pyplot.plot(X,ug, 'b--', lw=2, label='$u_5(x)$', marker='o')
pyplot.legend(bbox_to_anchor=(0.35,1));
```

## Error Norms

### $L_2$ norm of the approximation error

\begin{equation}
||e||_{L_2} = \sqrt{\int_\Omega (u-u_n)^2d\Omega} =
\sqrt{\sum_{e=1}^{Nel} \int_{\Omega_e} (u-u_n)^2d\Omega_e}
\end{equation}

### Energy norm of the approximation $u_n$

\begin{eqnarray*}
||u_n||_{A} &=& \sqrt{\int_\Omega EA(u_n')^2 d\Omega} =
\sqrt{\sum_{e=1}^{Nel} \int_{\Omega_e}EA (u_n')^2 d\Omega_e} \\
&=& \sqrt{\{a\}^T[K_g]\{a\}} = \sqrt{\{a\}^T\{f\}} \\
&=& \sqrt{\sum_{e=1}^{Nel}\{a_e\}^T[K_e]\{a_e\} }.
\end{eqnarray*}

```python
NormEner_un = sqrt( numpy.dot( numpy.transpose(ug), numpy.dot(Kg, ug) ) )
print('The energy norm of the FEM solution is {:.4e}.'.format(NormEner_un))
```

    The energy norm of the FEM solution is 5.4894e-02.

### Energy norm of the exact solution $u$

\begin{equation}
||u||_{A} = \sqrt{\int_\Omega EA(u')^2 d\Omega} =
\sqrt{\sum_{e=1}^{Nel} \int_{\Omega_e} EA(u')^2 d\Omega_e}
\end{equation}

```python
du=N/EA
I3=integrate.simps(EA*du**2,x)
NormEner_uex = sqrt(I3)
print('The energy norm of the exact solution is {:.4e}.'.format(NormEner_uex))
```

    The energy norm of the exact solution is 5.4917e-02.

### Energy norm of the relative error

\begin{equation}
||e_r||_{A} = \frac{||u||_{A} - ||u_n||_{A}}{||u||_{A}}
\end{equation}

```python
Error = (NormEner_uex - NormEner_un)/NormEner_uex
print('The relative error is e = {:.4e}.'.format(Error))
```

    The relative error is e = 4.1127e-04.

## Exercise

What is the error behavior with mesh refinement? Try to increase the mesh resolution and check the error behavior. Can you compute the mesh convergence rate?

## References

- G. E. Karniadakis, S. J. Sherwin. *Spectral/hp Element Methods for CFD*. Oxford University
Press, Oxford, 1999.
- M.L. Bittencourt. *Fully tensorial nodal and modal shape functions for triangles and tetrahedra*. IJNME, 63(2):1530-1558, 2005.
- M.L. Bittencourt, M. G. Vazquez, T. G. Vazquez. *Construction of shape functions for the h- and p- versions of the fem using tensorial product*. IJNME, 71(5):529-563, 2007.
- S. Dong, Z. Yosibash. *A parallel spectral element method for dynamic three-dimensional nonlinear
elasticity problems*. Computer & Structures, (4):61-85, 2008.
- M.L. Bittencourt, T. G. Vazquez. *Tensor-based gauss-jacobi numerical integration for high-order
mass and stiffness matrices.* IJNME, 79(5):599-638,2009.
- M.L. Bittencourt, *Análise computacional de estruturas*, Editora da UNICAMP, 2010.

```python

```