```python
from sympy import *
init_session()
```

    IPython console for SymPy 1.4 (Python 3.7.4-64-bit) (ground types: python)

    These commands were executed:
    >>> from __future__ import division
    >>> from sympy import *
    >>> x, y, z, t = symbols('x y z t')
    >>> k, m, n = symbols('k m n', integer=True)
    >>> f, g, h = symbols('f g h', cls=Function)
    >>> init_printing()

    Documentation can be found at https://docs.sympy.org/1.4/

```python
@vectorize(0,1)
def vdiff(f,y):
    return diff(f,y)
```

```python
vdiff([f(x, y, z), g(x, y, z), h(x, y, z)], [x, y, z])
```

```python
s = symbols("s",cls=Function)
vdiff(s(x,y,z),[x,y,z])
```

```python
from sympy.vector import CoordSys3D, gradient, Del
# 1 gradient
C = CoordSys3D('C')

delop = Del() # nabla算子

# 标量场 f = x**2*y-xy
f = C.x**2*C.y - C.x*C.y

res = delop.gradient(f, doit=True) # 使用nabla算子
# res = delop(f).doit()

res = gradient(f) # 直接使用gradient
print(res) # (2*C.x*C.y - C.y)*C.i + (C.x**2 - C.x)*C.j
```

    (2*C.x*C.y - C.y)*C.i + (C.x**2 - C.x)*C.j

```python
f = symbols('f', cls=Function)
```

```python
diff(x**2*y-x*y,x), diff(x**2*y-x*y,y)
```

```python
from sympy.vector import CoordSys3D,gradient,Del
D = CoordSys3D('D')
delop = Del()
s = Function('s')(D.x,D.y,D.z)
delta_s = delop.gradient(s,doit=True)
print(delta_s)
```

    (Derivative(s(D.x, D.y, D.z), D.x))*D.i + (Derivative(s(D.x, D.y, D.z), D.y))*D.j + (Derivative(s(D.x, D.y, D.z), D.z))*D.k

```python
init_printing(use_unicode=True)
```

```python
diff(s,D.x)*D.i + diff(s, D.y)*D.j +  diff(s,D.z)*D.k
```

### Differential Operations for Scalars, Vectors, and Tensors in Cartesian Coordinates

```python
from sympy.vector import CoordSys3D,gradient,Del
C = CoordSys3D('C')
Vx = Function('Vx')
Vy = Function('Vy')
Vz = Function('Vz')
V = Function('Vx')(C.x) + Function('Vy')(C.y) + Function('Vz')(C.z)
delta_V = Del().gradient(V,doit=True)
print(delta_V)
```

    (Derivative(Vx(C.x), C.x))*C.i + (Derivative(Vy(C.y), C.y))*C.j + (Derivative(Vz(C.z), C.z))*C.k

```python
(Derivative(Vx(C.x), C.x))*C.i + (Derivative(Vy(C.y), C.y))*C.j + (Derivative(Vz(C.z), C.z))*C.k
```

```python
V = Function('Vx')(C.x,C.y,C.z) + Function('Vy')(C.x,C.y,C.z) + Function('Vz')(C.x,C.y,C.z)
```

```python
DiffV = Del().gradient(V,doit=True)
```

```python
(C.i).cross(C.i), (C.i).cross(C.j),(C.i).cross(C.k)
```

```python
DiffV
```

```python
DiffV.cross(C.i) + DiffV.cross(C.j) + DiffV.cross(C.k)
```

```python
V1 = (Function('Vx')(C.x,C.y,C.z) + Function('Vy')(C.x,C.y,C.z) + Function('Vz')(C.x,C.y,C.z))*(C.i)
```

```python
Del().cross(V1,doit=True)
```

```python
V2 = (Function('Vx')(C.x,C.y,C.z) + Function('Vy')(C.x,C.y,C.z) + Function('Vz')(C.x,C.y,C.z))*(C.j)
Del().cross(V2,doit=True)
```

```python
V3 = (Function('Vx')(C.x,C.y,C.z) + Function('Vy')(C.x,C.y,C.z) + Function('Vz')(C.x,C.y,C.z))*(C.k)
Del().cross(V3,doit=True)
```

```python
Del().cross(V1,doit=True) + Del().cross(V2,doit=True) + Del().cross(V3,doit=True)
```

```python
from sympy.vector import CoordSys3D,gradient,Del
C = CoordSys3D('C')
Vx = Function('Vx')
Vy = Function('Vy')
Vz = Function('Vz')
V = (Function('Vx')(C.x,C.y,C.z) + Function('Vy')(C.x,C.y,C.z) + Function('Vz')(C.x,C.y,C.z))*(C.i)
delta_V = Del().dot(V,doit=True)
print(delta_V)
```

    Derivative(Vx(C.x, C.y, C.z), C.x) + Derivative(Vy(C.x, C.y, C.z), C.x) + Derivative(Vz(C.x, C.y, C.z), C.x)

```python
Derivative(Vx(C.x, C.y, C.z), C.x) + Derivative(Vx(C.x, C.y, C.z), C.y) + Derivative(Vx(C.x, C.y, C.z), C.z) + Derivative(Vy(C.x, C.y, C.z), C.x) + Derivative(Vy(C.x, C.y, C.z), C.y) + Derivative(Vy(C.x, C.y, C.z), C.z) + Derivative(Vz(C.x, C.y, C.z), C.x) + Derivative(Vz(C.x, C.y, C.z), C.y) + Derivative(Vz(C.x, C.y, C.z), C.z)
```

Del().dot((Vx(C.x,C.y,C.z))*C.i, doit=True) + Del().dot((Vy(C.x,C.y,C.z))*C.j,doit=True) + Del().dot((Vz(C.x,C.y,C.z))*C.k,doit=True)

```python
Del().cross((Vx(C.x,C.y,C.z))*C.i, doit=True) + Del().cross((Vy(C.x,C.y,C.z))*C.j,doit=True) + Del().cross((Vz(C.x,C.y,C.z))*C.k,doit=True)
```

# Transport Phenomena

### Algerbraic Operations for vectors and tensors in cartesian coordinates

```python
from sympy import *
init_session()
```

    IPython console for SymPy 1.4 (Python 3.7.4-64-bit) (ground types: python)

    These commands were executed:
    >>> from __future__ import division
    >>> from sympy import *
    >>> x, y, z, t = symbols('x y z t')
    >>> k, m, n = symbols('k m n', integer=True)
    >>> f, g, h = symbols('f g h', cls=Function)
    >>> init_printing()

    Documentation can be found at https://docs.sympy.org/1.4/

```python
from sympy.vector import CoordSys3D
C = CoordSys3D("C")
V, W = symbols("V W")
Vx,Vy,Vz = symbols('V(x:z)')
Wx,Wy,Wz = symbols("W(x:z)")
V = Vx*C.i+Vy*C.j+Vz*C.k
W = Wx*C.i+Wy*C.j+Wz*C.k
```

```python
V.dot(W)
```

```python
W.dot(V)
```

```python
W.dot(V) == V.dot(W)
```

    True

```python
V.cross(W)
```

```python
(V.cross(W)).dot(C.i), (V.cross(W)).dot(C.j), (V.cross(W)).dot(C.k)
```

```python
(W.cross(V)).dot(C.i), (W.cross(V)).dot(C.j), (W.cross(V)).dot(C.k)
```

```python
((V.cross(W)).dot(C.i), (V.cross(W)).dot(C.j), (V.cross(W)).dot(C.k)) == (-(W.cross(V)).dot(C.i), -(W.cross(V)).dot(C.j), -(W.cross(V)).dot(C.k))
```

    True

```python
𝑇𝑥𝑥, 𝑇𝑥𝑦, 𝑇𝑥𝑧 = symbols("Tx(x:z)")
𝑇𝑦𝑥, 𝑇𝑦𝑦, 𝑇𝑦𝑧 = symbols("Ty(x:z)")
𝑇𝑧𝑥, 𝑇𝑧𝑦, 𝑇𝑧𝑧 = symbols("Tz(x:z)")
```

```python
𝑇𝑥𝑥, 𝑇𝑥𝑦, 𝑇𝑥𝑧, 𝑇𝑦𝑥, 𝑇𝑦𝑦, 𝑇𝑦𝑧, 𝑇𝑧𝑥, 𝑇𝑧𝑦, 𝑇𝑧𝑧 = symbols("Tx(x:z), Ty(x:z), Tz(x:z)")
```

```python
T = symbols("T")
T = Matrix([[𝑇𝑥𝑥, 𝑇𝑥𝑦, 𝑇𝑥𝑧],[𝑇𝑦𝑥, 𝑇𝑦𝑦, 𝑇𝑦𝑧],[𝑇𝑧𝑥, 𝑇𝑧𝑦, 𝑇𝑧𝑧]])
```

```python
V.to_matrix(C)
```

$\displaystyle \left[\begin{matrix}Vx\\Vy\\Vz\end{matrix}\right]$

```python
T*(V.to_matrix(C)) == T*V.to_matrix(C)
```

    True

```python
(T*V.to_matrix(C)).row(0), (T*V.to_matrix(C)).row(1), (T*V.to_matrix(C)).row(2)
```

$\displaystyle \left( \left[\begin{matrix}Txx Vx + Txy Vy + Txz Vz\end{matrix}\right], \  \left[\begin{matrix}Tyx Vx + Tyy Vy + Tyz Vz\end{matrix}\right], \  \left[\begin{matrix}Tzx Vx + Tzy Vy + Tzz Vz\end{matrix}\right]\right)$

```python
T*V.to_matrix(C)
```

$\displaystyle \left[\begin{matrix}Txx Vx + Txy Vy + Txz Vz\\Tyx Vx + Tyy Vy + Tyz Vz\\Tzx Vx + Tzy Vy + Tzz Vz\end{matrix}\right]$

```python
V.to_matrix(C),T
```

$\displaystyle \left( \left[\begin{matrix}Vx\\Vy\\Vz\end{matrix}\right], \  \left[\begin{matrix}Txx & Txy & Txz\\Tyx & Tyy & Tyz\\Tzx & Tzy & Tzz\end{matrix}\right]\right)$

```python
(V.to_matrix(C)).transpose()*T.col(0), (V.to_matrix(C)).transpose()*T.col(1), (V.to_matrix(C)).transpose()*T.col(2)
```

$\displaystyle \left( \left[\begin{matrix}Txx Vx + Tyx Vy + Tzx Vz\end{matrix}\right], \  \left[\begin{matrix}Txy Vx + Tyy Vy + Tzy Vz\end{matrix}\right], \  \left[\begin{matrix}Txz Vx + Tyz Vy + Tzz Vz\end{matrix}\right]\right)$

Note: The above operations may be generalized to cylindrical coordinates by replacing (x,y,z) by (r,theta,z), and to replacing (x, y,z) by (r, theta,phi).

```python
@vectorize(0,1)
def vdiff(f,y):
    return diff(f,y)
```

```python
s = symbols("s",cls=Function)
vdiff(s(x,y,z),[x,y,z])
```

OR

```python
s = (Function('s')(C.x,C.y,C.z))
```

```python
Del().dot(s*(C.i),doit=True), Del().dot(s*(C.j),doit=True), Del().dot(s*(C.k),doit=True)
```

```python
from sympy.vector import CoordSys3D,gradient,Del
C = CoordSys3D('C')
vx = Function('vx')
vy = Function('vy')
vz = Function('vz')
v = Function('vx')(C.x,C.y,C.z)*(C.i) + Function('vy')(C.x,C.y,C.z)*(C.j) + Function('vz')(C.x,C.y,C.z)*(C.k)
delta_v = Del().dot(v,doit=True)
print(delta_v)
```

    Derivative(vx(C.x, C.y, C.z), C.x) + Derivative(vy(C.x, C.y, C.z), C.y) + Derivative(vz(C.x, C.y, C.z), C.z)

```python
Del().cross((vx(C.x,C.y,C.z))*C.i, doit=True) + Del().cross((vy(C.x,C.y,C.z))*C.j,doit=True) + Del().cross((vz(C.x,C.y,C.z))*C.k,doit=True)
```

```python
Del().dot((vx(C.x,C.y,C.z))*C.i, doit=True) + Del().dot((vy(C.x,C.y,C.z))*C.j, doit=True) + Del().dot((vz(C.x,C.y,C.z))*C.k, doit=True)
```

```python
v = (Function('vx')(C.x,C.y,C.z)*(C.i) + Function('vy')(C.x,C.y,C.z)*(C.j) + Function('vz')(C.x,C.y,C.z)*(C.k))
```

```python
(Del() & v).doit() # Scalar
```

```python
delta_S = Del().dot(s*(C.i),doit=True)*(C.i) + Del().dot(s*(C.j),doit=True)*(C.j) + Del().dot(s*(C.k),doit=True)*(C.k)
```

```python
delta_S.dot(v)
```

```python
(Del() & delta_S).doit()
```

```python
delta_V = diff(v.dot(C.i),C.x)*(C.i) + diff(v.dot(C.j),C.y)*(C.j) + diff(v.dot(C.i),C.z)*(C.k)
```

```python
gradient(gradient(vx(C.x,C.y,C.z), coord_sys=C).dot(C.i)).dot(C.i)
```

```python
gradient(gradient(vx(C.x,C.y,C.z), coord_sys=C).dot(C.j)).dot(C.j)
```

```python
gradient(gradient(vx(C.x,C.y,C.z), coord_sys=C).dot(C.k)).dot(C.k)
```

```python
v = (vx(C.x,C.y,C.z))*C.i + (vy(C.x,C.y,C.z))*C.j + (vz(C.x,C.y,C.z))*C.k
```

```python
v.dot(C.i)
```

```python
Del().dot(gradient(v.dot(C.i),coord_sys=C))
```

```python
v.dot(C.j)
```

```python
Del().dot(gradient(v.dot(C.j),coord_sys=C))
```

```python
v.dot(C.k)
```

```python
Del().dot(gradient(v.dot(C.k),coord_sys=C))
```

```python
V.dot(gradient(v.dot(C.i),coord_sys=C))
```

```python
V.dot(gradient(v.dot(C.j),coord_sys=C))
```

```python
V.dot(gradient(v.dot(C.k),coord_sys=C))
```

```python
vv = (v.to_matrix(C))*((v.to_matrix(C)).transpose())
```

```python
type(vv)
```

    sympy.matrices.immutable.ImmutableDenseMatrix

```python
vv
```

$\displaystyle \left[\begin{matrix}\operatorname{vx}^{2}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vx}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vy}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vx}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vz}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\\\operatorname{vx}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vy}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vy}^{2}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vy}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vz}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\\\operatorname{vx}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vz}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vy}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} \operatorname{vz}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)} & \operatorname{vz}^{2}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\end{matrix}\right]$

```python
gradient(vv.row(0).dot(Matrix([1,0,0])),coord_sys =C,doit=False).dot(C.i)
```

```python
gradient(vv.row(0).dot(Matrix([0,1,0])),coord_sys =C,doit=False).dot(C.j)
```

```python
gradient(vv.row(0).dot(Matrix([0,0,1])),coord_sys =C,doit=False).dot(C.k)
```

```python
gradient(vv.row(0).dot(Matrix([1,0,0])),coord_sys =C,doit=False).dot(C.i) + gradient(vv.row(0).dot(Matrix([0,1,0])),coord_sys =C,doit=False).dot(C.j) + gradient(vv.row(0).dot(Matrix([0,0,1])),coord_sys =C,doit=False).dot(C.k)
```

```python
gradient(vv.row(1).dot(Matrix([1,0,0])),coord_sys =C,doit=False).dot(C.i) + gradient(vv.row(1).dot(Matrix([0,1,0])),coord_sys =C,doit=False).dot(C.j) + gradient(vv.row(1).dot(Matrix([0,0,1])),coord_sys =C,doit=False).dot(C.k)
```

```python
gradient(vv.row(2).dot(Matrix([1,0,0])),coord_sys =C,doit=False).dot(C.i) + gradient(vv.row(2).dot(Matrix([0,1,0])),coord_sys =C,doit=False).dot(C.j) + gradient(vv.row(2).dot(Matrix([0,0,1])),coord_sys =C,doit=False).dot(C.k)
```

```python
Txx = Function("Txx")(C.x,C.y,C.z)
Txy = Function("Txy")(C.x,C.y,C.z)
Txz = Function("Txz")(C.x,C.y,C.z)
Tyx = Function("Tyx")(C.x,C.y,C.z)
Tyy = Function("Tyy")(C.x,C.y,C.z)
Tyz = Function("Tyz")(C.x,C.y,C.z)
Tzx = Function("Tzx")(C.x,C.y,C.z)
Tzy = Function("Tzy")(C.x,C.y,C.z)
Tzz = Function("Tzz")(C.x,C.y,C.z)
```

```python
T = Matrix([[Txx,Txy,Txz],[Tyx,Tyy,Tyz],[Tzx,Tzy,Tzz]])
```

```python
gradient(T.row(0).dot((C.i).to_matrix(C)), doit=False).dot(C.i) + gradient(T.row(0).dot((C.j).to_matrix(C)), doit=False).dot(C.j) + gradient(T.row(0).dot((C.k).to_matrix(C)), doit=False).dot(C.k)
```

```python
gradient(T.row(1).dot((C.i).to_matrix(C)), doit=False).dot(C.i) + gradient(T.row(1).dot((C.j).to_matrix(C)), doit=False).dot(C.j) + gradient(T.row(1).dot((C.k).to_matrix(C)), doit=False).dot(C.k)
```

```python
gradient(T.row(2).dot((C.i).to_matrix(C)), doit=False).dot(C.i) + gradient(T.row(2).dot((C.j).to_matrix(C)), doit=False).dot(C.j) + gradient(T.row(2).dot((C.k).to_matrix(C)), doit=False).dot(C.k)
```

```python
v
```

```python
v.to_matrix(C)
```

$\displaystyle \left[\begin{matrix}\operatorname{vx}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\\\operatorname{vy}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\\\operatorname{vz}{\left(\mathbf{{x}_{C}},\mathbf{{y}_{C}},\mathbf{{z}_{C}} \right)}\end{matrix}\right]$

```python
T.row(0).dot(gradient(v.dot(C.i),doit=True).to_matrix(C))
```

```python
T.row(1).dot(gradient(v.dot(C.j),doit=True).to_matrix(C))
```

```python
T.row(1).dot(gradient(v.dot(C.j),doit=True).to_matrix(C))
```

```python
T.row(0).dot(gradient(v.dot(C.i),doit=True).to_matrix(C)) + T.row(1).dot(gradient(v.dot(C.j),doit=True).to_matrix(C)) + T.row(1).dot(gradient(v.dot(C.j),doit=True).to_matrix(C))
```

```python

```


```python
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from scipy.stats import multivariate_normal
```

```python
import matplotlib.pyplot as plt
%matplotlib inline
```

```python
mean1 = np.matrix([1,2])
cov1 = np.matrix([[1.8,-0.7],[-0.7,1.8]])
```

```python
mean2 = np.matrix([-1,-3])
cov2 =np.matrix([[1.5,0.3],[0.3,1.5]])
```

```python
-0.5*mean1*((np.linalg.inv(cov1)))*mean1.T-0.5*np.log(np.linalg.det(cov1))
```

    matrix([[-2.651255]])

```python
-0.5*mean2*((np.linalg.inv(cov2)))*mean2.T-0.5*np.log(np.linalg.det(cov2))
```

    matrix([[-3.44060967]])

```python
-0.5*np.linalg.inv(cov1), -0.5*np.linalg.inv(cov2)
```

    (matrix([[-0.32727273, -0.12727273],
             [-0.12727273, -0.32727273]]),
     matrix([[-0.34722222,  0.06944444],
             [ 0.06944444, -0.34722222]]))

```python
np.linalg.inv(cov1)*mean1.T, np.linalg.inv(cov2)*mean2.T
```

    (matrix([[1.16363636],
             [1.56363636]]),
     matrix([[-0.27777778],
             [-1.94444444]]))

```python
s=-0.5*np.linalg.inv(cov1)+0.5*np.linalg.inv(cov2)
```

```python
m=np.linalg.inv(cov1)*mean1.T  - np.linalg.inv(cov2)*mean2.T
```

```python
b=-0.5*mean1*((np.linalg.inv(cov1)))*mean1.T-0.5*np.log(np.linalg.det(cov1)) - (-0.5*mean2*((np.linalg.inv(cov2)))*mean2.T-0.5*np.log(np.linalg.det(cov2))) + np.log(3)
```

```python
mean1 = ([1,2])
cov1 =([[1.8,-0.7],[-0.7,1.8]])
```

```python
mean2 = ([-1,-3])
cov2 =([[1.5,0.3],[0.3,1.5]])
```

```python
from sympy.solvers import solve
from sympy import Symbol
```

```python
X=Symbol('X')
Y=Symbol('Y')
```

```python
solve(0.01994*(X**2)-0.3934*X*Y+0.01994*(Y**2)+1.4414*X+3.5080*Y+1.88796695,X,Y)
```

    [(9.86459378134403*Y - 34.8089810035731*sqrt(0.0794858297234481*Y**2 - 0.733707899807037*Y + 1) - 36.1434302908726,
      Y),
     (9.86459378134403*Y + 34.8089810035731*sqrt(0.0794858297234481*Y**2 - 0.733707899807037*Y + 1) - 36.1434302908726,
      Y)]

```python
x1 = np.linspace(-5,5,500)
y1 = 9.86459378134403*x1 - 34.8089810035731*np.sqrt(0.0794858297234481*x1**2 - 0.733707899807037*x1 + 1) - 36.1434302908726
y2 = 9.86459378134403*x1 + 34.8089810035731*np.sqrt(0.0794858297234481*x1**2 - 0.733707899807037*x1 + 1) - 36.1434302908726
```

    f:\users\lehrd\anaconda3\envs\tfg\lib\site-packages\ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in sqrt

    f:\users\lehrd\anaconda3\envs\tfg\lib\site-packages\ipykernel_launcher.py:3: RuntimeWarning: invalid value encountered in sqrt
      This is separate from the ipykernel package so we can avoid doing imports until

```python
x = np.linspace(-10,10,500)
y = np.linspace(-10,10,500)
X, Y = np.meshgrid(x,y)
ph=np.zeros([500,500,2])
ph[:,:,0]=X
ph[:,:,1]=Y
rv = multivariate_normal(mean1, cov1)
rv2 = multivariate_normal(mean2, cov2)

plt.contour(X, Y, rv.pdf(ph))
plt.contour(X,Y, rv2.pdf(ph))
plt.colorbar()

#plt.plot(x1,y1,color='b')
plt.plot(x1,y2,color='b')
```

```python
def BayesClassifierL(x):
    d1=9.86459378134403*x[0] - 34.8089810035731*np.sqrt(0.0794858297234481*x[0]**2 - 0.733707899807037*x[0] + 1) - 36.1434302908726
    d2=9.86459378134403*x[0] + 34.8089810035731*np.sqrt(0.0794858297234481*x[0]**2 - 0.733707899807037*x[0] + 1) - 36.1434302908726
    if d1<=x[1] and d2 >=x[1]:
        return 2
    else:
        return 1
```

```python
def BayesClassifier(x):
    d1=9.86459378134403*x[0] - 35.5916254466891*np.sqrt(0.0760285463822325*(x[0]**2) - 0.701794839225714*x[0] + 1) - 36.1434302908726
    d2=9.86459378134403*x[0] + 35.5916254466891*np.sqrt(0.0760285463822325*(x[0]**2) - 0.701794839225714*x[0] + 1) - 36.1434302908726
    if d1<=x[1] and d2 >=x[1]:
        return 2
    else:
        return 1
```

```python
x1=np.random.multivariate_normal(mean1,cov1,1000)
```

```python
x2=np.random.multivariate_normal(mean2,cov2,1000)
```

```python
tp=0
tn=0
tpl=0
tnl=0
for i in range(1000):
    r = BayesClassifier(x1[i])
    if r == 1 :
        tp+=1
    r = BayesClassifier(x2[i])
    if r == 2 :
        tn+=1
    r = BayesClassifierL(x1[i])
    if r == 1 :
        tpl+=1
    r = BayesClassifierL(x2[i])
    if r == 2 :
        tnl+=1
```

    f:\users\lehrd\anaconda3\envs\tfg\lib\site-packages\ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in sqrt

    f:\users\lehrd\anaconda3\envs\tfg\lib\site-packages\ipykernel_launcher.py:3: RuntimeWarning: invalid value encountered in sqrt
      This is separate from the ipykernel package so we can avoid doing imports until

```python
tp
```

    974

```python
tpl
```

    986

```python
tn
```

    876

```python
tnl
```

    844

```python

```


# Variational Bayes Notes

## What is variational Bayes?
A technique that allows us to compute approximations to non-closed form Bayes update equations.

## Why would we use it?
In **Bayesian parameter estimation**, to compute the intractable integration of the denominator in the parameter posterior probability:

\begin{equation}
p(\Theta \vert \Omega, Y) = \frac{p(Y, \Theta \vert \Omega)}{\int p(Y, \Theta \vert \Omega) d\Omega}
\end{equation}

and in **Bayesian model fitting**, to find the denominator of the model posterior probability (for a fixed $\Omega$):

\begin{equation}
p(\Omega \vert Y) = \frac{p(Y \vert \Omega) p( \Omega )}{\sum_{\Omega \in \mathcal{M}} p(Y\vert \Omega)p(\Omega)}
\end{equation}

Where $\Omega$ is the candidate model, $Y = \{(x_1,d_1),\dots,(x_N,d_N)\}$ is the training data, $\Theta$ is the set of unknown model parameters (and prior hyperparameters) under $\Omega$.

Also for **Bayesian data fusion**, if we're using GMM priors with MMS likelihoods, we're trying to evaluate

\begin{equation}\label{eq:bdf}
p(X_k \vert D_k) = \frac{P(D_k \vert X_k) p( X_k )}{\int P(D_k \vert X_k) p( X_k )dX_k} = \frac{p(X_k,D_k)}{P(D_k)}
\end{equation}

Where $p(D_k \vert X_k)$ is the MMS model and

$$
p( X_k ) = p(X_{k} \vert D_{1:k-1}) = \int p(X_k \vert X_{k-1}) p(X_{k} \vert D_{1:k-1}) dX_{k-1}
$$

where $p(X_k \vert X_{k-1})$ is the state transition pdf and we are only fusing $D_k$ sensor data.

## Alternatives & Extensions
 * Grid-based approaches
 * [MCMC](https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo) (and other Monte Carlo/Particle Filter techniques)
 * [Laplace Approximation](https://en.wikipedia.org/wiki/Laplace's_method)
 * VBIS: Use variational bayes outputs as the parameters for imporance distribution $q(X_k)$
 * LWIS: Use prior $p(X_k)$ as imporance distribution $q(X_k)$

## How does it work?

VB minimizes the KLD between the integrable, closed-form *variational posterior* parameter distribution $q(\Theta \vert Y, \Omega)$ and the true posterior $p(\Theta \vert \Omega, Y)$ for $\Omega \in \mathcal{M}$:

$$
KL(q\vert\vert p) = - \int q(\Theta \vert Y, \Omega) log{\frac{p(\Theta \vert Y, \Omega)}{q(\Theta \vert Y, \Omega)}}d\Theta
$$

But, since $p(\Theta \vert Y, \Omega)$ is unavailable, we can instead minimize the KLD by maximizing a lower bound $\mathcal{L}$ to $\log{p(Y\vert\Omega)}$, where:

$$
\mathcal{L} = \int q(\Theta \vert Y, \Omega) log{\frac{p(Y, \Omega \vert \Theta)}{q(\Theta \vert Y, \Omega)}}d\Theta
$$

and

\begin{align*}
\log{p(Y\vert\Omega)} &= \mathcal{L} + KL(q \vert\vert p) \\
&= \int q(\Theta \vert Y, \Omega) log{\frac{p(Y, \Omega \vert \Theta)}{q(\Theta \vert Y, \Omega)}}d\Theta
- \int q(\Theta \vert Y, \Omega) log{\frac{p(\Theta \vert Y, \Omega)}{q(\Theta \vert Y, \Omega)}}d\Theta \\
&= \int q(\Theta \vert Y, \Omega) log{(p(Y, \Omega \vert \Theta))}
- q(\Theta \vert Y, \Omega) log{(p(\Theta \vert Y, \Omega))}d\Theta \\
&= \int q(\Theta \vert Y, \Omega) log{\frac{p(Y, \Omega \vert \Theta)}{p(\Theta \vert Y, \Omega)}}
d\Theta
\end{align*}

## How do we use it?

Take equation \ref{eq:bdf}. We want to approximate $p(X_k,D_k)$ (analytically intractable) with an unnormalized Gaussian lower bound pdf, which leads to a *variational Bayesian* Gaussian posterior approximation $\hat{p}(X_k\vert D_k)$.

If $f(D_k,X_k)$ is an unnormalized Gaussian function that approximates the softmax likelihood $P(D_k \vert X_k)$, then

\begin{align}\label{eq:approx_joint}
p(X_k,D_k) &\approx \hat{p}(X_k,D_k) = p(X_k)f(D_k,X_k) \\
P(D_k) = C &\approx \hat{C} = \int_{-\infty}^{\infty} \hat{p}(X_k,D_k)
\end{align}

Since $p(X_k)$ is Gaussian, $\hat{p}(X_k,D_k)$ is as well (as the product of two gaussians).

### How do we derive $f(D_k,X_k)$?

[2] derives an upper bound to the softmax denominator:

\begin{equation}\label{eq:upper}
\log\left(\sum_{c=1}^m e^{y_c}\right) \leq \alpha + \sum_{c=1}^m \frac{y_c - \alpha - \xi_c}{2} + \lambda(\xi_c)[(y_c - \alpha)^2 - \xi_c^2] + log(1 + e^{\xi_c})
\end{equation}

where $\lambda(\xi_c) = \frac{1}{2\xi_c}\left[\frac{1}{1 + e^{-\xi_c}}\right] - \frac{1}{2}$ and $y_c = w^T_cx + b_c$. $\alpha$ and $\xi_c$ are *free variational parameters*; given $y_c$, $\alpha$ and $\xi_c$ can be selected to minimize the upper bound in \ref{eq:upper}.

Assuming known $\alpha$ and $\xi_c$, we take the log of the softmax likelihood to get:

\begin{align*}
\log{P(D_k=j\vert X_k)} &= w^T_jx + b_j - \log{\left(\sum_{c=1}^m e^{w^T_cx + v_c}\right)} \\
&\leq \log{f(D_j=j,X_k)} =  g_j + h^T_jx - \frac{1}{2}x^TK_jx
\end{align*}

Where

\begin{equation}\label{eq:approx_likelihood}
f(D_j=j,X_k) = \exp\{g_j + h^T_jx - \frac{1}{2}x^TK_jx\}
\end{equation}

The prior $p(X_k)$ can be expressed similarly:

\begin{equation}\label{eq:gaussian_prior}
p(X_k) = \exp\{g_p + h^T_px - \frac{1}{2}x^TK_px\}
\end{equation}

where $g_p = -\frac{1}{2}(log{\lvert2\pi\Sigma\rvert} + \mu^TK_p\mu)$, $h_p = K_p\mu$ $K_p = \Sigma^{-1}$. This is simply a reformulation of the equation of a gaussian:

\begin{align*}
p(X_k) &= \frac{1}{\sqrt{\lvert 2 \pi \Sigma \rvert}} exp{\{-\frac{1}{2}(x - \mu)^T\Sigma^{-1}(x - \mu) \}} \\
&=exp{\{-\frac{1}{2}(x^T - \mu^T)K_p(x - \mu) -\frac{1}{2}\log{\lvert 2 \pi \Sigma \rvert} \}}\\
&=exp{\{-\frac{1}{2}x^TK_px +\frac{1}{2}(x^TK_p\mu +\mu^TK_px) -\frac{1}{2}\mu^TK_p\mu -\frac{1}{2}\log{\lvert 2 \pi \Sigma \rvert} \}}\\
&=exp{\{-\frac{1}{2}(\log{\lvert 2 \pi \Sigma \rvert} + \mu^TK_p\mu ) +\frac{1}{2}(K_p\mu x +K_p \mu x) -\frac{1}{2}x^TK_px \}}\\
&= \exp\{g_p + h^T_px - \frac{1}{2}x^TK_px\}
\end{align*}

Since equation \ref{eq:approx_joint} is simply the product of two Gaussians, it becomes:

\begin{equation}\label{eq:approx_joint_product}
\hat{p}(X_k,D_k) = p(X_k)f(D_k,X_k) = \exp\{g_l + h^T_lx - \frac{1}{2}x^TK_lx\} = \mathcal{N}(\hat{\mu}_{VB},\hat{\Sigma}_{VB})
\end{equation}

### But how do we optimize $\alpha$ and $\xi_c$ ?

Minimizing the RHS of \ref{eq:upper} gives us:

\begin{align}
\xi^2_c &= y^2_c + \alpha^2 - 2\alpha y_c \label{eq:xi} \\
\alpha &= \frac{\left(\frac{m-2}{4}\right) \sum_{c=1}^m\lambda(\xi_c)y_c}{\sum_{c=1}^m\lambda(\xi_c)} \label{eq:alpha}
\end{align}

But, both depend on $X_k$, which is unobserved. Instead, we minimize the *expected value* of the RHS of \ref{eq:upper} with respect to the posterior.

Apparently **(?)** this is equivalent to maximizing $\log{\hat{P}(D_k)}$, the *approximate* the marginal log-likelihood of the observation $D_k = j$:

$$
\log{\hat{P}(D_k)} = \log{\hat{C}} = \log \int_{-\infty}^{\infty} \hat{p}(X_k,D_k)dX_k
$$

We can now use **expectation-maximization (EM)** to iteratively optimize $\alpha$ and $\xi_c$. We take the expectations of \ref{eq:xi} and \ref{eq:alpha} under the current $\hat{p}(X_k \vert D_k)$ estimate. Additionally, we'll need the following:

\begin{align}
\langle y_c\rangle &= w^T_c\hat{\mu}_{VB} + b_c \label{eq:y_expected} \\
\langle y^2_c\rangle &= w^T_c(\hat{\Sigma}_{VB} + \hat{\mu}_{VB}\hat{\mu}_{VB}^T)w_c + 2w^T_c\hat{\mu}_{VB}b_c + b^2_c \label{eq:y2_expected}
\end{align}

The code below demonstrates the expectation maximization algorithm.

## Issues

The approximate posterior is **optimistic** relative to the true posterior, and will be biased due to this optimism as well.

# Examples

## Variational Bayes (VB) with Softmax and Gaussian Prior

Let's restate the bayesian data fusion problem. We want to compute the following:

\begin{equation}\label{eq:vb-bdf}
p(X_k \vert D_k) = \frac{P(D_k = j \vert X_k) p( X_k )}{\int P(D_k = j \vert X_k) p( X_k )dX_k} = \frac{p(X_k,D_k)}{P(D_k)}
\end{equation}

Where, for a softmax likelihood,

\begin{equation}\label{eq:softmax}
P(D_k = j \vert X_k) = \frac{e^{w^T_jx + b_j}}{\sum_{c=1}^m e^{w^T_cx + b_c}}
\end{equation}

and, for a gaussian prior,

\begin{equation}
P(X_k) = \frac{1}{\sqrt{\lvert 2 \pi \Sigma}} \exp{\{-\frac{1}{2}(x - \mu)^T\Sigma^{-1}(x - \mu)\}}
\end{equation}

We show these two below for a one-dimensional problem of estimating the speed of a target.

```python
from cops_and_robots.robo_tools.fusion.softmax import speed_model
%matplotlib inline

sm = speed_model()
sm.plot(plot_classes=False)
```

```python
from __future__ import division
from scipy.stats import norm
import numpy as np
import matplotlib.pyplot as plt

mu = 0.3
sigma = 0.1
min_x = -5
max_x = 5
res = 10000

prior = norm(loc=mu, scale=sigma)
x_space = np.linspace(min_x, max_x, res)

# Plot the frozen distribution
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(8, 8)
ax.plot(x_space, prior.pdf(x_space), lw=2, label='frozen pdf', c='g')
ax.fill_between(x_space, 0, prior.pdf(x_space), alpha=0.2, facecolor='g')
ax.set_xlim([0,.4])
ax.set_ylim([0,10])
ax.set_title('Gaussian prior')
```

While the numerator of \ref{eq:vb-bdf} can be computed easily, there is no closed form solution for its denominator.

We'll follow the following algorithm:

**Inputs**
 * prior $\mu$ and $\Sigma$;
 * $D_k = j$ with likelihood in eq.(7);
 * initial $\alpha$ and $\xi_c$, for j, c ∈ {1, ...,m}

**Outputs**
 * posterior mean $\hat{\mu}_{VB}$
 * Posterior covariance $\hat{\Sigma}_{VB}$

**Steps**
 1. E-step: for all fixed $\xi_c$ and $\alpha$,
   1. compute $\hat{\alpha}_{VB}$ and $\hat{\Sigma}_{VB}$ via eq. (19);
   2. compute $\langle y_c\rangle$ and $\langle y_c^2\rangle$ via eqs. (23)-(24);
 2. M-step: for all fixed $\langle y_c\rangle$ and $\langle y_c^2\rangle$,

       **for $i = 1 : n_{lc}$ do **
   1. compute all $\xi_c$ for fixed $\alpha$ via eq. (20)
   2. compute $\alpha$ for all fixed $\xi_c$ via eq. (21)

**end for **
 3. If converged, return $\hat{C}$ via eq. (25) and stop; otherwise, return to step 1

```python
from numpy.linalg import inv
import pandas as pd
np.set_printoptions(precision=2, suppress=True)
pd.set_option('precision', 3)

# SETTINGS:
n_lc = 15  # number of convergence loops
measurement = 'Medium'
tolerance = 10 ** -3  # for convergence
max_EM_steps = 1000

# INPUT: Define input priors and initial values
prior_mu = np.zeros(1)
prior_sigma = np.array([[1]])
initial_alpha = 0.5
initial_xi = np.ones(4)

# Softmax values
m = 4
w = sm.weights
b = sm.biases
j = sm.class_labels.index(measurement)

# Preparation
xis = initial_xi
alpha = initial_alpha
mu_hat = prior_mu
sigma_hat = prior_sigma

# dataframe for debugging
df = pd.DataFrame({'Alpha': alpha,
                   'g_j' : np.nan,
                   'h_j' : np.nan,
                   'K_j' : np.nan,
                   'Mu': mu_hat[0],
                   'Sigma': sigma_hat[0][0],
                   'Xi': [xis],
                   })
def lambda_(xi_c):
    return 1 / (2 * xi_c) * ( (1 / (1 + np.exp(-xi_c))) - 0.5)

converged = False
EM_step = 0

while not converged and EM_step < max_EM_steps:
    ################################################################
    # STEP 1 - EXPECTATION
    ################################################################
    # PART A #######################################################

    # find g_j
    sum1 = 0
    for c in range(m):
        if c != j:
            sum1 += b[c]
    sum2 = 0
    for c in range(m):
        sum2 = xis[c] / 2 \
            + lambda_(xis[c]) * (xis[c] ** 2 - (b[c] - alpha) ** 2) \
            - np.log(1 + np.exp(xis[c]))
    g_j = 0.5 *(b[j] - sum1) + alpha * (m / 2 - 1) + sum2

    # find h_j
    sum1 = 0
    for c in range(m):
        if c != j:
            sum1 += w[c]
    sum2 = 0
    for c in range(m):
        sum2 += lambda_(xis[c]) * (alpha - b[c]) * w[c]
    h_j = 0.5 * (w[j] - sum1) + 2 * sum2

    # find K_j
    sum1 = 0
    for c in range(m):
        sum1 += lambda_(xis[c]) * w[c].T .dot (w[c])

    K_j = 2 * sum1

    K_p = inv(prior_sigma)
    g_p = -0.5 * (np.log( np.linalg.det(2 * np.pi * prior_sigma))) \
        + prior_mu.T .dot (K_p) .dot (prior_sigma)
    h_p = K_p .dot (prior_mu)

    g_l = g_p + g_j
    h_l = h_p + h_j
    K_l = K_p + K_j

    mu_hat = inv(K_l) .dot (h_l)
    sigma_hat = inv(K_l)

    # PART B #######################################################
    y_cs = np.zeros(m)
    y_cs_squared = np.zeros(m)
    for c in range(m):
        y_cs[c] = w[c].T .dot (mu_hat) + b[c]
        y_cs_squared[c] = w[c].T .dot (sigma_hat + mu_hat .dot (mu_hat.T)) .dot (w[c]) \
            + 2 * w[c].T .dot (mu_hat) * b[c] + b[c] ** 2

    ################################################################
    # STEP 2 - MAXIMIZATION
    ################################################################
    for i in range(n_lc):

        # PART A #######################################################
        # Find xi_cs
        for c in range(m):
            xis[c] = np.sqrt(y_cs_squared[c] + alpha ** 2 - 2 * alpha * y_cs[c])

        # PART B #######################################################
        # Find alpha
        num_sum = 0
        den_sum = 0
        for c in range(m):
            num_sum += lambda_(xis[c]) * y_cs[c]
            den_sum += lambda_(xis[c])
        alpha = ((m - 2) / 4 + num_sum) / den_sum

    ################################################################
    # STEP 3 - CONVERGENCE CHECK
    ################################################################

    new_df = pd.DataFrame([[alpha, g_j, h_j, K_j, mu_hat, sigma_hat,
                            [xis]]],
                          columns=('Alpha','g_j','h_j','K_j','Mu','Sigma',
                                   'Xi',))
    df = df.append(new_df, ignore_index=True)
    EM_step += 1
# df

#plot results
mu_post = mu_hat[0]
sigma_post = np.sqrt(sigma_hat[0][0])

print('Mu and sigma found to be {} and {}, respectively.'.format(mu_hat[0],sigma_hat[0][0]))

ax = sm.plot_class(measurement_i, fill_between=False)
posterior = norm(loc=mu_post, scale=sigma_post)
ax.plot(x_space, posterior.pdf(x_space), lw=2, label='posterior pdf', c='b')
ax.fill_between(x_space, 0, posterior.pdf(x_space), alpha=0.2, facecolor='b')
ax.plot(x_space, prior.pdf(x_space), lw=1, label='prior pdf', c='g')

ax.set_title('Posterior distribtuion')
ax.legend()
ax.set_xlim([0, 0.4])
ax.set_ylim([0, 7])
plt.show()
```

Correct output: sigma 5.926215694086777e-05 and mu 0.2227

```python

```

### Comparison: discretized state space

Using a discrete environment, we get the following:

```python
measurement = 'Slow'
measurement_i = sm.class_labels.index(measurement)

dx = (max_x - min_x)/res

normalizer = 0
for x in x_space:
    lh = sm.probs_at_state(x, measurement)
    if np.isnan(lh):
        lh = 1.00
    normalizer += lh  * gaussian.pdf(x)
normalizer *= dx

posterior = np.zeros_like(x_space)
for i, x in enumerate(x_space):
    lh = sm.probs_at_state(x, measurement)
    if np.isnan(lh):
        lh = 1.00
    posterior[i] = lh * gaussian.pdf(x) / normalizer

ax = sm.plot_class(measurement_i, fill_between=False)
ax.plot(x_space, posterior, lw=3, label='posterior pdf', c='b')
ax.fill_between(x_space, 0, posterior, alpha=0.2, facecolor='b')
ax.plot(x_space, prior.pdf(x_space), lw=1, label='prior pdf', c='g')

ax.set_title('Posterior distribtuion')
ax.legend()
ax.set_xlim([0, 0.4])
plt.show()
```

```python

```

## References
[1] N. Ahmed and M. Campbell, “Variational Bayesian learning of probabilistic discriminative models with latent softmax variables,” Signal Process. IEEE Trans. …, vol. XX, no. c, pp. 1–27, 2011.

[2] G. Bouchard, “Efficient bounds for the softmax function and applications to approximate inference in hybrid models,” in NIPS 2007 Workshop for Approximate Bayesian Inference in Continuous/Hybrid Systems, Whistler, BC, Canada, 2007.

[3] N. Ahmed, E. Sample, and M. Campbell, “Bayesian Multicategorical Soft Data Fusion for Human--Robot Collaboration,” IEEE Trans. Robot., vol. 29, no. 1, pp. 189–206, 2013.

```python
from IPython.core.display import HTML

# Borrowed style from Probabilistic Programming and Bayesian Methods for Hackers
def css_styling():
    styles = open("../styles/custom.css", "r").read()
    return HTML(styles)
css_styling()
```

<link href='http://fonts.googleapis.com/css?family=Roboto:100,100italic,500,300,300italic,400' rel='stylesheet' type='text/css'>