```python
import numpy as np
import numpy.linalg as linalg
from discreteMarkovChain import markovChain

import sympy
sympy.init_printing(use_unicode=True)
```

```python
P = np.matrix([
    [0.6, 0.2, 0.2],
    [0.1, 0.8, 0.1],
    [0.1, 0.1, 0.8]])

(values,vectors) = linalg.eig(np.transpose(P))
print("values",values)
print("vectors\n",vectors)
indexOf1 = np.where(np.abs(values-1)<0.0001)[0][0]
s = vectors[:,indexOf1]
s = np.transpose(s / sum(s))
print("steady state probs", s)
print("s*P", s*P)

mc = markovChain(P)
mc.computePi('linear') #We can also use 'power', 'krylov' or 'eigen'
print("mc.pi", mc.pi)
```

    values [ 0.5  1.   0.7]
    vectors
     [[ -8.16496581e-01   3.33333333e-01  -8.45817455e-17]
     [  4.08248290e-01   6.66666667e-01  -7.07106781e-01]
     [  4.08248290e-01   6.66666667e-01   7.07106781e-01]]
    steady state probs [[ 0.2  0.4  0.4]]
    s*P [[ 0.2  0.4  0.4]]
    mc.pi [ 0.2  0.4  0.4]

```python
P1 = np.matrix([
    [0.5,0.5],
    [0.6,0.4]])

(values,vectors) = linalg.eig(np.transpose(P1))
print("values",values)
print("vectors\n",vectors)
indexOf1 = np.where(np.abs(values-1)<0.0001)[0][0]
s = vectors[:,indexOf1]
s = np.transpose(s / sum(s))
print("steady state probs", s)
print("s*P", s*P1)

mc = markovChain(P1)
mc.computePi('linear') #We can also use 'power', 'krylov' or 'eigen'
print("mc.pi", mc.pi)
```

    values [ 1.  -0.1]
    vectors
     [[ 0.76822128 -0.70710678]
     [ 0.6401844   0.70710678]]
    steady state probs [[ 0.54545455  0.45454545]]
    s*P [[ 0.54545455  0.45454545]]
    mc.pi [ 0.54545455  0.45454545]

```python
def steadyStateSym(transitionMatrix):
    """
    Given a transition matrix of a Markov chain, returns the steady-state vector
    Uses sympy for symbolic computaiton.

    INPUT: transitionMatrix - a list of rows each of which sums up to 1.
    OUTPUT steadyState - a sympy matrix; a row whose entires sum to 1.
    """
    P = sympy.Matrix(transitionMatrix)
    vects = P.T.eigenvects()
    s = [vect[2] for vect in vects if vect[0]==1][0][0].T
    s = sympy.simplify(s)
    s = (s/sum(s))
    s = sympy.simplify(s)
    return s
```

```python
p,q = sympy.symbols('p q')
s = steadyStateSym([
    [q, 1-q, 0],
    [q, 0, 1-q],
    [0, q, 1-q]])
s
```

```python
sympy.simplify(s*P)
```

```python
mat = np.zeros(3)
mat
```

    array([ 0.,  0.,  0.])

```python
def expectedPayment(w,p,q,A,B,kA,kB,kI,txCost,restartCost):
    transition = np.zeros((w+1,w+1))  # Markov process transition matrix
    payment = np.zeros(w+1)           # payment vector
    for k in range(0, kA):   # vanishes if kA=0 (never restart)
        transition[k,k] = 1-p-q
        transition[k,k+B] = q
        transition[k,kI-A] = p
        payment[k] = p*restartCost     # channel initialization
    for k in range(kA, A):  # vanishes if kA=A (never use blockchain)
        transition[k,k] = 1-q
        transition[k,k+B] = q
        payment[k] = p*txCost     # w.p. p, use bitcoin tx to remain in place
    for k in range(A,w-B+1):
        transition[k,k] = 1-p-q
        transition[k,k+B] = q
        transition[k,k-A] = p
    for k in range(w-B+1, kB+1): # vanishes if kB=w-B (never use blockchain)
        transition[k,k] = 1-p
        transition[k,k-A] = p
        payment[k] = q*txCost     # w.p. q, use bitcoin tx to remain in place
    for k in range(kB+1, w+1):  # disappears if kB=w (never restart)
        transition[k,k] = 1-p-q
        transition[k,kI+B] = q
        transition[k,k-A] = p
        payment[k] = q*restartCost     # channel initialization

    mc = markovChain(transition)
    mc.computePi('eigen') # can use 'linear' 'power', 'krylov' or 'eigen'
    steadyState = mc.pi
    expectedPayment = np.dot(payment,steadyState)
    return expectedPayment

ep = expectedPayment(w=100, p=.1, q=.9, A=10, B=1, kA=10, kB=99, kI=90, txCost=5, restartCost=10)
ep
```

```python
def simulate(w,p,q,A,B,txCost,restartCost):
    eps = []
    for kA in [0,A]:
        for kB in [w,w-B]:
            for kI in [A, int(w/2), w-B]:
                ep = expectedPayment(w, p, q, A, B, kA, kB, kI, txCost, restartCost)
                print("kA={} kB={} kI={} ep={}".format(kA,kB,kI,ep))
                eps.append((kA,kB,kI,ep))
            print()
    i = np.argmin([ep[3] for ep in eps])
    best = eps[i]
    print("BEST: Alice uses blockchain in {}-{}, Bob restarts after {}, initialization at {}, payment {}".format(best[0],best[1],best[2],best[3]))

simulate(w=100, p=.1, q=.95, A=10, B=1, txCost=5, restartCost=10)
```

    kA=0 kB=100 kI=10 ep=0.2029853321233756
    kA=0 kB=100 kI=50 ep=0.2029853321233756
    kA=0 kB=100 kI=99 ep=0.2029853321233756

    kA=0 kB=99 kI=10 ep=0.06707861060419912
    kA=0 kB=99 kI=50 ep=0.05863615996388098
    kA=0 kB=99 kI=99 ep=0.36479017950826154

    kA=10 kB=100 kI=10 ep=0.29475787425482014
    kA=10 kB=100 kI=50 ep=0.2373843531586099
    kA=10 kB=100 kI=99 ep=0.38592332458781176

    kA=10 kB=99 kI=10 ep=0.20828645157085246
    kA=10 kB=99 kI=50 ep=0.04065825264181202
    kA=10 kB=99 kI=99 ep=0.7584627330436977

    BEST: Alice restarts before 10, Bob restarts after 99, initialization at 50, payment 0.04065825264181202

```python
from scipy.misc import comb
comb(20,4)
```

    4845.0

```python
comb(6,2)*comb(6,2)/comb(12,4)
```

    0.45454545454545453

```python

```


```python
import sympy as sym
import matplotlib.pyplot as plt

x1,x2,x3,y1,y2,y3,a,b,x,y=sym.symbols('x1,x2,x3,y1,y2,y3,a,b,x,y')
wc1=(a*x1+b-y1)**2
wc2=(a*x2+b-y2)**2
wc3=(a*x3+b-y3)**2

wc = wc1+wc2+wc3
wc= wc.subs(x1,-29).subs(x2,-20).subs(x3,-13)
wc= wc.subs(y1,-9).subs(y2,1).subs(y3,4)
wc

```

$\displaystyle \left(- 29 a + b + 9\right)^{2} + \left(- 20 a + b - 1\right)^{2} + \left(- 13 a + b - 4\right)^{2}$

```python

wc=wc.expand()
wc
wc.collect(a)

```

$\displaystyle 1410 a^{2} + a \left(- 124 b - 378\right) + 3 b^{2} + 8 b + 98$

```python
#b=-(14*a-22)/(2*3)
eq1=sym.Eq(a,-(-124*𝑏-378)/(2*1410))
wc.collect(b)

```

$\displaystyle 1410 a^{2} - 378 a + 3 b^{2} + b \left(8 - 124 a\right) + 98$

```python
eq2=sym.Eq(b,-(8-124*a)/(2*3))
sym.solve([eq1,eq2],[a,b])

```

    {a: 319/386, b: 3039/193}

```python

a=319/386.0
b=3039/193.0

plt.plot(-29,-9,'o')
#plt.text(-29,-20,'x1,y1')

plt.plot(-20,1,'o')
#plt.text(-13,-9,'x2,y3')

plt.plot(-13,4,'o')
#plt.text(1,4,'x2,y3')

plt.plot([-29,-20,-13],[a*-29+b,a*-20+b,a*-13+b],marker='o')

plt.show()
```

```python

```


```python
l=1.0
n=100
h=l/(n-1)
τ = 1e-4
δ = 0.01
Gr = 100
C = 1

from sympy import simplify, collect
from sympy.abc import y, x, z, m, C, h, G, d, t
m = x - t*((d**2)*(y - 2*x + z)/(h**2) - G*(d**2)*m*(y-z)/(2*h) - C)
```

```python
collect(m, m)
```

$\displaystyle - t \left(- C - \frac{G d^{2} m \left(y - z\right)}{2 h} + \frac{d^{2} \left(- 2 x + y + z\right)}{h^{2}}\right) + x$


# Bayesian Inference in the Poisson Generalized Linear Model

**References:**
- Chapter 16 of BDA3 contains background material on generalized linear models.
- Chapter 7.1 of BDA3 introduces notation for model evaluation based on predictive log likelihoods.

## The Poisson GLM

The Poisson distribution is a common model for count data with a single parameter $\lambda \in \mathbb{R}_+$. Its pmf is,

\begin{align}
\Pr(y \mid \lambda) &= \frac{1}{y!} e^{-\lambda} \lambda^y,
\end{align}

for $y \in \mathbb{N}$. Its mean and variance are both equal to $\lambda$.

Suppose we have count observations $y_n \in \mathbb{N}$ along with covariates $x_n \in \mathbb{R}^P$. We construct a Poisson GLM by modeling the expected value as,

\begin{align}
\mathbb{E}[y_n \mid x_n] = f(w^\top x_n),
\end{align}

with $w \in \mathbb{R}^P$ and $f: \mathbb{R} \to \mathbb{R}_+$ is the mean function. The _canonical mean function_ is $f(a) = e^a$; equivalently, the canonical _link_ function is the logarithm.

We assume a Gaussian prior on the weights $w$:
$$
w \sim \mathcal{N}(0, \sigma^2 I),
$$

where $\sigma^2 I$ is the covariance matrix.

Derive the log joint probability,

\begin{align}
\mathcal{L}(w) &\triangleq \log p(\{y_n\}_{n=1}^N, w \mid \{x_n\}_{n=1}^N, \sigma^2) \\
&= \log p(w\mid\sigma^2) + \sum_{n=1}^N\log p(y_n \mid w^Tx_n)
\end{align}

If we replace $\lambda = \exp(w^Tx_n)$ in the Possion pmf we get

\begin{align}
\mathcal{L}(w) &= \log\mathcal{N}(w\mid 0, \sigma^2\mathbf{I})+\sum_{n=1}^N\log\left(\frac{1}{y_n!}\exp\left\{\left \langle y_n, w^Tx_n \right \rangle-\exp(w^Tx_n)\right\}\right)\\
&= \log\mathcal{N}(w\mid 0, \sigma^2\mathbf{I}) + \sum_{n=1}^N\left(\left \langle y_n, w^Tx_n \right \rangle - \exp(w^Tx_n) - \log (y_n!)\right)\\
&= -\frac{w^Tw}{2\sigma^2} + \left \langle \sum_{n=1}^Ny_nx_n, w \right \rangle - \sum_{n=1}^N\exp(w^Tx_n) + C
\end{align}

\begin{align}
\nabla_w \mathcal{L}(w) &= -\nabla_w\frac{w^Tw}{2\sigma^2} +  \nabla_w\left \langle \sum_{n=1}^Ny_nx_n, w \right \rangle -  \nabla_w\sum_{n=1}^N\exp(w^Tx_n)\\
&= -\frac{w}{\sigma^2}+\sum_{n=1}^Ny_nx_n-\sum_{n=1}^Nx_n\exp(w^Tx_n)
\end{align}

\begin{align}
\nabla^2_w \mathcal{L}(w) &= -\nabla_w\frac{w}{\sigma^2}+\nabla_w\sum_{n=1}^Ny_nx_n-\nabla_w\sum_{n=1}^Nx_n\exp(w^Tx_n)\\
&= -\frac{I}{\sigma^2}+0-\sum_{n=1}^Nx_n\nabla_w\exp(w^Tx_n)\\
&= -\frac{I}{\sigma^2}-\sum_{n=1}^Nx_nx_n^T\exp(w^Tx_n)
\end{align}

## Maximum a posterior (MAP) Estimation

### Optimize to find the posterior mode

```python
import sys
sys.path.append('../')
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from src.d01_data.dengue_data_api import DengueDataApi
from src.d04_modeling.poisson_glm import PoissonGLM
```

```python
dda = DengueDataApi()
x_train, x_validate, y_train, y_validate = dda.split_data()

axs0 = sns.heatmap(y_train.unstack('weekofyear'))
```

```python
sigma2 = 1.
poisson_glm1 = PoissonGLM(x_train=x_train, y_train=y_train, sigma2=sigma2)
```

```python
_, w_hist = poisson_glm1.compute_posterior_mode()
w_hist_df = pd.DataFrame(w_hist, columns=x_train.columns)
w_hist_df['log_joint'] = w_hist_df.apply(lambda w: poisson_glm1.log_joint(y_train.values.reshape((-1,1)),
                                                                          x_train.values, 1.,
                                                                            w.values, sigma2), axis=1)
w_hist_df.name = 'iter'
axs1 = sns.lineplot(data=w_hist_df.iloc[1:].reset_index(), x="index", y="log_joint")
```

## Laplace Approximation

### Approximate the covariance at the mode

Solve for $\Sigma_{\mathsf{MAP}} = -[\nabla^2(\mathcal{L}(w_{\mathsf{MAP}})]^{-1}$. Plot the covariance matrix (e.g. with `imshow`). Don't forget to add a colorbar and labels.

```python
poisson_glm1.compute_laplace_approximation()
cov_map = poisson_glm1.get_cov_map()
cov_map_df = pd.DataFrame(cov_map, index=x_train.columns, columns=x_train.columns)
axs2 = sns.heatmap(cov_map_df)
```

# Posterior of the weights

Plot the posterior mean of the weights for features $x_n$ (i.e. not including the bias term). Also plot 95% credible intervals around the mean by using two standard deviations of the marginal distribution of the weights. Note the diagonal of $\Sigma_{\mathsf{MAP}}$ gives the marginal variance of the posterior.

```python
w_map = poisson_glm1.get_w_map()
var_map = np.diagonal(cov_map)

fig, ax = plt.subplots()
n = len(var_map)
ax.errorbar(range(n), w_map, yerr=1.96*np.sqrt(var_map/n), marker='o', linestyle='none')
ax.set_xlabel('Covariate')
ax.set_xticks(range(n))
ax.set_xticklabels(x_train.columns, rotation='vertical')
ax.set_ylabel('$w$')
plt.show()
```

## Model validation

### Approximate the posterior predictive distribution of the rates

We can draw many samples $w^{(s)}$ from the Laplace approximation of the posterior $p(w \mid \{x_n, y_n\})$. Use those samples to approximate the posterior predictive distribution on the **test** dataset,

\begin{align}
p(y_{n'}=k \mid x_{n'}, \{x_n, y_n\}_{n=1}^N) &=
\int p(y_{n'} \mid w, x_{n'}) \, p(w \mid \{x_n, y_n\}_{n=1}^N) \, \mathrm{d} w \\
&\approx \frac{1}{S} \sum_{s=1}^S p(y_{n'}=k \mid w^{(s)}, x_{n'})
\end{align}
where
\begin{align}
w^{(s)} &\sim p(w \mid \{x_n, y_n\}_{n=1}^N \\
&\approx \mathcal{N}(w \mid w_{\mathsf{MAP}}, \Sigma_{\mathsf{MAP}})
\end{align}

Visualize the posterior predictive distribution as an $K \times N_{\mathsf{test}}$ array where row corresponds to possible spike counts $k\in \{0,\ldots, K\}$. You can set $K=5$ for this problem. **Only show the first 100 columns (time bins), otherwise it's hard to see changes in the rate.**

Overlay the actual spike counts for the test dataset.

```python
log_joint, mae, e = poisson_glm1.validate_model(x_validate=x_validate, y_validate=y_validate)
print("Log Joint Probability: %.2f" % log_joint)
print("MAE: %.6f" % mae)
```

## Singular Value Descomposition

```python
x_train, x_validate, y_train, y_validate = dda.split_data()

u, s, vh = np.linalg.svd(x_train, full_matrices=True)
num_components = 4
new_features = ["pc%i" % i for i in range(num_components)]
z_train = pd.DataFrame(np.dot(x_train, vh[:num_components, :].T), columns=new_features, index=x_train.index)
z_validate = pd.DataFrame(np.dot(x_validate, vh[:num_components, :].T), columns=new_features, index=x_validate.index)

sigma2 = 1.
poisson_glm2 = PoissonGLM(x_train=z_train, y_train=y_train, sigma2=sigma2)

poisson_glm2.compute_laplace_approximation()
cov_map = poisson_glm2.get_cov_map()
cov_map_df = pd.DataFrame(cov_map, index=z_train.columns, columns=z_train.columns)
axs2 = sns.heatmap(cov_map_df)
```

```python
w_map = poisson_glm2.get_w_map()
var_map = np.diagonal(cov_map)

fig, ax = plt.subplots()
n = len(var_map)
ax.errorbar(range(n), w_map, yerr=1.96*np.sqrt(var_map), marker='o', linestyle='none')
ax.set_xlabel('Covariate')
ax.set_xticks(range(n))
ax.set_xticklabels(z_train.columns, rotation='vertical')
ax.set_ylabel('$w$')
plt.show()
```

```python
log_joint, mae, e = poisson_glm2.validate_model(x_validate=z_validate, y_validate=y_validate)
print("Log Joint Probability: %.2f" % log_joint)
print("MAE: %.6f" % mae)
```

```python

```


```python
import sympy as sym
import numpy as np
from IPython.display import display, Markdown, Latex, HTML
import scipy as sp
import scipy.linalg
import matplotlib.pyplot as plt
from matplotlib import animation
from mpl_toolkits.mplot3d.art3d import Poly3DCollection

```

```python
# Angulos de rot
Rx= np.deg2rad(45)
Ry= np.deg2rad(0)
Rz= np.deg2rad(0)

# Vetor gravidade
g = sym.Matrix([[0],
                [0],
                [-9.8]])

# Montando as matrizes de rotação (o plano rotaciona, não o vetor)
# por isso, esta não é a matriz de rotação de um vetor no R3 (ela é equivalente à rotação de um angulo em sentido contrário em um vetor)
# ex.: rotacionar um vetor 45º no eixo x é diferente de rotacionar o plano e manter o vetor estacionário, ou seja, rotarcionar o vetor -45º

#rotação no eixo x
Mx = sym.Matrix([[1,             0,          0],
                 [0,    np.cos(Rx), np.sin(Rx)],
                 [0, -1*np.sin(Rx), np.cos(Rx)]])
#rotação no eixo y
My = sym.Matrix([[np.cos(Ry), 0, -1*np.sin(Ry)],
                 [         0, 1,             0],
                 [np.sin(Ry), 0,    np.cos(Ry)]])
#rotação no eixo z
Mz = sym.Matrix([[   np.cos(Rz), np.sin(Rz), 0],
                 [-1*np.sin(Rz), np.cos(Rz), 0],
                 [            0,          0, 1]])

# Pega a rotação do vetor g no R3
Mx@My@Mz@g

# Só subtrair o vetor resultante da leitura do acelerômetro
# isso irá isolar a gravidade ( teremos somente a projeção dos vetores de aceleração no plano XY)
```

$\displaystyle \left[\begin{matrix}0\\-6.92964645562817\\-6.92964645562817\end{matrix}\right]$


## Using Python (and R) for Finance -- (Sanjiv Das)

In this notebook we will get started with using Python for financial computation. You first need to download and install IPython which is an entire system for interactive use of Python. Luckily, this is easily done by visiting the following URL and downloading the version applicable to your machine (Python runs on all platforms).

Visit Continuum Analytics and download Python: http://continuum.io/downloads and install it. Test that it works by using the Launcher (which will be on your desktop) to start up the IPython Notebook.

Also install the R programming language, which is a very useful tool for Machine Learning.
See: http://en.wikipedia.org/wiki/Machine_learning
Get R from: http://www.r-project.org/ (download and install it).

In order to allow Python to interface with R, you also need to install the "rpy2" package from: http://rpy.sourceforge.net/
While you do not need to use R, and can get by mostly with Python, it is also a useful language as there are heaps of finance packages in R. As you will see, we can run both within the IPython Notebook. (Indeed, these lecture notes have been prepared in the Notebook.)

If you want to use R in IDE mode, download RStudio: http://www.rstudio.com/

Particularly useful are linear algebra capabilities in Python and R. In Python the package numPy makes these facile. Also useful are graphical libraries, and in Python, these comes in the matplotlib package.

```python
#Invoke numPy and matplotlib in one command
%pylab inline
```

    Populating the interactive namespace from numpy and matplotlib

```python
#IMPORTING STOCK DATA USING PANDAS
from pandas.io.data import DataReader
from datetime import datetime
goog = DataReader("GOOG",  "yahoo", datetime(2014,4,1), datetime(2015,3,31))
stkp = goog["Adj Close"]
print stkp
```

    Date
    2014-04-01    567.15997
    2014-04-02    567.00000
    2014-04-03    569.73999
    2014-04-04    543.14001
    2014-04-07    538.15002
    2014-04-08    554.90002
    2014-04-09    564.14001
    2014-04-10    540.95001
    2014-04-11    530.59998
    2014-04-14    532.52002
    2014-04-15    536.44000
    2014-04-16    556.53998
    2014-04-17    536.09998
    2014-04-21    528.62000
    2014-04-22    534.81000
    ...
    2015-03-11    551.17999
    2015-03-12    555.51001
    2015-03-13    547.32001
    2015-03-16    554.51001
    2015-03-17    550.84003
    2015-03-18    559.50000
    2015-03-19    557.98999
    2015-03-20    560.35999
    2015-03-23    558.81000
    2015-03-24    570.19000
    2015-03-25    558.78497
    2015-03-26    555.16998
    2015-03-27    548.34003
    2015-03-30    552.03003
    2015-03-31    548.00000
    Name: Adj Close, Length: 252

```python
goog.head()
```

      Open
      High
      Low
      Close
      Volume
      Adj Close

      Date

      2014-04-01
       558.71002
       568.45001
       558.71002
       567.15997
          7900
       567.15997

      2014-04-02
       599.98999
       604.83002
       562.19000
       567.00000
        146700
       567.00000

      2014-04-03
       569.84998
       587.28003
       564.13000
       569.73999
       5085200
       569.73999

      2014-04-04
       574.65002
       577.77002
       543.00000
       543.14001
       6351900
       543.14001

      2014-04-07
       540.73999
       548.47998
       527.15002
       538.15002
       4389600
       538.15002

```python
goog.index
```

    <class 'pandas.tseries.index.DatetimeIndex'>
    [2014-04-01, ..., 2015-03-31]
    Length: 252, Freq: None, Timezone: None

```python
t = goog.index
plot(t,stkp)
xlabel("Date")
ylabel("Stock Price")
```

```python
n = len(t)-1
rets = zeros(n)
for j in range(n):
    rets[j] = log(stkp[j+1]/stkp[j])
plot(rets)
ylabel("Returns")
```

```python
hist(rets,25)
```

```python
goog.describe()
```

      Open
      High
      Low
      Close
      Volume
      Adj Close

      count
       252.000000
       252.000000
       252.000000
       252.000000
           252.000000
       252.000000

      mean
       550.523850
       554.668048
       545.193662
       550.081029
       1922350.000000
       550.081029

      std
        24.599106
        24.145277
        24.955973
        24.524011
        941218.139443
        24.524011

      min
       494.649990
       495.976010
       487.560000
       492.549990
          7900.000000
       492.549990

      25%
       531.007507
       535.871978
       526.500015
       531.240007
       1371625.000000
       531.240007

      50%
       550.000000
       553.994995
       544.024995
       549.170015
       1691350.000000
       549.170015

      75%
       571.622515
       576.689985
       566.231262
       571.405017
       2193975.000000
       571.405017

      max
       599.989990
       604.830020
       592.500000
       596.080020
       6790900.000000
       596.080020

```python
import scipy.stats as ss
print("Skewness = ",ss.skew(rets))
print("Kurtosis = ",ss.kurtosis(rets))

#CHECK IF THIS IS EXCESS KURTOSIS or PLAIN KURTOSIS
x = randn(1000000)
print(ss.kurtosis(x))
```

    ('Skewness = ', -0.173028777259206)
    ('Kurtosis = ', 0.9993234508893414)
    0.00302654770593

### SHARING VARIABLES FROM PYTHON TO R

```python
#SENDING DATA VARIABLES TO R
%load_ext rpy2.ipython
#THIS ALLOWS US TO USE R INSIDE THE NOTEBOOK
```

```python
#SENDS DATA FROM PYTHON TO R
%Rpush stkp
```

```python
#PREFIX NEEDED TO CALL R INSTEAD OF PYTHON
%R plot(stkp,type="l",col="red",lwd=2)
```

```python
#GETTING DATA BACK FROM R TO PYTHON
%R ret = diff(log(stkp))
#GET DATA BACK FROM R TO PYTHON
ret = %Rget ret
plot(ret)
%R print(summary(ret))
```

### GETTING STOCK DATA FROM R INTO PYTHON

```r
%%R
library(quantmod)
getSymbols(c("C","AAPL","CSCO","YHOO","IBM"))
```

    Loading required package: xts
    Loading required package: zoo

    Attaching package: ‘zoo’

    The following objects are masked from ‘package:base’:

        as.Date, as.Date.numeric

    Loading required package: TTR
    Version 0.4-0 included new data defaults. See ?getSymbols.
        As of 0.4-0, ‘getSymbols’ uses env=parent.frame() and
     auto.assign=TRUE by default.

     This  behavior  will be  phased out in 0.5-0  when the call  will
     default to use auto.assign=FALSE. getOption("getSymbols.env") and
     getOptions("getSymbols.auto.assign") are now checked for alternate defaults

     This message is shown once per session and may be disabled by setting
     options("getSymbols.warning4.0"=FALSE). See ?getSymbol for more details
    [1] "C"    "AAPL" "CSCO" "YHOO" "IBM"

```r
%%R
citi = as.matrix(C$C.Adjusted)
aapl = as.matrix(AAPL$AAPL.Adjusted)
csco = as.matrix(CSCO$CSCO.Adjusted)
yhoo = as.matrix(YHOO$YHOO.Adjusted)
ibm = as.matrix(IBM$IBM.Adjusted)
```

```r
%%R
stkdata = data.frame(cbind(citi,aapl,csco,yhoo,ibm))
rn = rownames(stkdata)
head(stkdata)
```

               C.Adjusted AAPL.Adjusted CSCO.Adjusted YHOO.Adjusted IBM.Adjusted
    2007-01-03   497.5920      11.29213      24.94332         25.61     83.41148
    2007-01-04   495.8808      11.54276      25.59996         26.85     84.30331
    2007-01-05   493.2690      11.46057      25.60896         27.74     83.54011
    2007-01-08   495.7907      11.51716      25.75288         27.92     84.80925
    2007-01-09   491.4678      12.47390      25.60896         27.58     85.81255
    2007-01-10   487.5050      13.07084      25.79785         28.70     84.80067

```python
stkdata = %Rget stkdata
rn = %Rget rn
```

```python
stkdata
```

    array([[ 497.59198,  495.8808 ,  493.26904, ...,   53.76   ,   53.32   ,
              53.76   ],
           [  11.29213,   11.54276,   11.46057, ...,  128.64   ,  125.15   ,
             128.95   ],
           [  24.94332,   25.59996,   25.60896, ...,   29.05   ,   28.83   ,
              29.13   ],
           [  25.61   ,   26.85   ,   27.74   , ...,   43.28   ,   42.57   ,
              42.51   ],
           [  83.41148,   84.30331,   83.54011, ...,  174.39999,  171.28999,
             173.67   ]])

```python
rn
```

    array(['2007-01-03', '2007-01-04', '2007-01-05', ..., '2015-04-29',
           '2015-04-30', '2015-05-01'],
          dtype='|S10')

### PANDAS IS AWESOME

```python
import pandas as pd
stk = pd.DataFrame(stkdata)
stk = stk.T
stk.head()
```

      0
      1
      2
      3
      4

      0
       497.59198
       11.29213
       24.94332
       25.61
       83.41148

      1
       495.88080
       11.54276
       25.59996
       26.85
       84.30331

      2
       493.26904
       11.46057
       25.60896
       27.74
       83.54011

      3
       495.79074
       11.51716
       25.75288
       27.92
       84.80925

      4
       491.46780
       12.47390
       25.60896
       27.58
       85.81255

```python
stk.columns=["C","AAPL","CSCO","YHOO","IBM"]
stk.index = rn
stk.head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2007-01-03
       497.59198
       11.29213
       24.94332
       25.61
       83.41148

      2007-01-04
       495.88080
       11.54276
       25.59996
       26.85
       84.30331

      2007-01-05
       493.26904
       11.46057
       25.60896
       27.74
       83.54011

      2007-01-08
       495.79074
       11.51716
       25.75288
       27.92
       84.80925

      2007-01-09
       491.46780
       12.47390
       25.60896
       27.58
       85.81255

```python
plot(stk["AAPL"])
```

```python
stk.ix['2007-01-03']
```

    C       497.59198
    AAPL     11.29213
    CSCO     24.94332
    YHOO     25.61000
    IBM      83.41148
    Name: 2007-01-03, dtype: float64

```python
stk.ix['2007-01-03']["AAPL"]
```

    11.29213

```python
stk["extra"] = 1.0
stk.head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM
      extra

      2007-01-03
       497.59198
       11.29213
       24.94332
       25.61
       83.41148
       1

      2007-01-04
       495.88080
       11.54276
       25.59996
       26.85
       84.30331
       1

      2007-01-05
       493.26904
       11.46057
       25.60896
       27.74
       83.54011
       1

      2007-01-08
       495.79074
       11.51716
       25.75288
       27.92
       84.80925
       1

      2007-01-09
       491.46780
       12.47390
       25.60896
       27.58
       85.81255
       1

```python
sort(stk["AAPL"])
```

    array([  10.53752,   10.8461 ,   11.09404, ...,  132.17   ,  132.64999,
            133.     ])

```python
stk.head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM
      extra

      2007-01-03
       497.59198
       11.29213
       24.94332
       25.61
       83.41148
       1

      2007-01-04
       495.88080
       11.54276
       25.59996
       26.85
       84.30331
       1

      2007-01-05
       493.26904
       11.46057
       25.60896
       27.74
       83.54011
       1

      2007-01-08
       495.79074
       11.51716
       25.75288
       27.92
       84.80925
       1

      2007-01-09
       491.46780
       12.47390
       25.60896
       27.58
       85.81255
       1

```python
stk = stk.drop("extra",axis=1)  #IF AXIS=0 (default), THEN ROW IS DROPPED
stk.head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2007-01-03
       497.59198
       11.29213
       24.94332
       25.61
       83.41148

      2007-01-04
       495.88080
       11.54276
       25.59996
       26.85
       84.30331

      2007-01-05
       493.26904
       11.46057
       25.60896
       27.74
       83.54011

      2007-01-08
       495.79074
       11.51716
       25.75288
       27.92
       84.80925

      2007-01-09
       491.46780
       12.47390
       25.60896
       27.58
       85.81255

```python
stk[["AAPL","IBM"]].head()
```

      AAPL
      IBM

      2007-01-03
       11.29213
       83.41148

      2007-01-04
       11.54276
       84.30331

      2007-01-05
       11.46057
       83.54011

      2007-01-08
       11.51716
       84.80925

      2007-01-09
       12.47390
       85.81255

```python
stk[stk["AAPL"]<11]
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2008-11-20
       46.80173
       10.84610
       13.01586
        8.95
       63.47624

      2009-01-20
       27.82269
       10.53752
       13.50160
       11.01
       72.53670

```python
stk[stk["AAPL"]<11]["IBM"]
```

    2008-11-20    63.48
    2009-01-20    72.54
    Name: IBM, dtype: float64

```python
(stk < 50).head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2007-01-03
       False
       True
       True
       True
       False

      2007-01-04
       False
       True
       True
       True
       False

      2007-01-05
       False
       True
       True
       True
       False

      2007-01-08
       False
       True
       True
       True
       False

      2007-01-09
       False
       True
       True
       True
       False

```python
sum(stk)
```

    C       221567.15
    AAPL    102168.36
    CSCO     42781.91
    YHOO     47775.87
    IBM     291151.26
    dtype: float64

```python
#USING FUNCTIONS ON DATA FRAMES
f = lambda x: x.max() - x.min()
stk.apply(f)
```

    C       496.94
    AAPL    122.46
    CSCO     18.41
    YHOO     43.42
    IBM     142.62
    dtype: float64

```python
def g(x):
    return pd.Series([x.mean(),x.std(),x.min(),x.max()], index=['mean','stdev','min','max'])
stk.apply(g)
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      mean
       106.522668
        49.119404
       20.568226
       22.969168
       139.976567

      stdev
       135.337468
        30.355509
        3.962029
        9.759319
        40.315380

      min
        10.160000
        10.540000
       12.250000
        8.950000
        63.480000

      max
       507.100000
       133.000000
       30.660000
       52.370000
       206.100000

```python
stk.sort_index(axis=1,ascending=False).head()
```

      YHOO
      IBM
      CSCO
      C
      AAPL

      2007-01-03
       25.61
       83.41
       24.94
       497.59
       11.29

      2007-01-04
       26.85
       84.30
       25.60
       495.88
       11.54

      2007-01-05
       27.74
       83.54
       25.61
       493.27
       11.46

      2007-01-08
       27.92
       84.81
       25.75
       495.79
       11.52

      2007-01-09
       27.58
       85.81
       25.61
       491.47
       12.47

```python
stk.sum()
```

    C       221567.15
    AAPL    102168.36
    CSCO     42781.91
    YHOO     47775.87
    IBM     291151.26
    dtype: float64

```python
stk.mean()
```

    C       106.522668
    AAPL     49.119404
    CSCO     20.568226
    YHOO     22.969168
    IBM     139.976567
    dtype: float64

```python
stk.describe()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      count
       2080.000000
       2080.000000
       2080.000000
       2080.000000
       2080.000000

      mean
        106.522668
         49.119404
         20.568226
         22.969168
        139.976567

      std
        135.337468
         30.355509
          3.962029
          9.759319
         40.315380

      min
         10.160000
         10.540000
         12.250000
          8.950000
         63.480000

      25%
         37.145000
         22.397500
         17.407500
         15.450000
        102.350000

      50%
         47.025000
         45.075000
         20.800000
         18.250000
        148.295000

      75%
         54.675000
         73.145000
         23.500000
         28.435000
        180.472500

      max
        507.100000
        133.000000
         30.660000
         52.370000
        206.100000

```python
stk.diff().head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2007-01-03
        NaN
        NaN
        NaN
        NaN
        NaN

      2007-01-04
      -1.71
       0.25
       0.66
       1.24
       0.89

      2007-01-05
      -2.61
      -0.08
       0.01
       0.89
      -0.76

      2007-01-08
       2.52
       0.06
       0.14
       0.18
       1.27

      2007-01-09
      -4.32
       0.95
      -0.14
      -0.34
       1.00

```python
stk.pct_change().head()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      2007-01-03
            NaN
            NaN
            NaN
            NaN
            NaN

      2007-01-04
      -0.003437
       0.022143
       0.026464
       0.048419
       0.010670

      2007-01-05
      -0.005263
      -0.006932
       0.000391
       0.033147
      -0.009015

      2007-01-08
       0.005109
       0.005236
       0.005467
       0.006489
       0.015202

      2007-01-09
      -0.008713
       0.082465
      -0.005437
      -0.012178
       0.011791

```python
stk.pct_change().mean()*252.0
```

    C      -0.052665
    AAPL    0.351436
    CSCO    0.062700
    YHOO    0.152730
    IBM     0.106559
    dtype: float64

```python
stk.pct_change().std()*sqrt(252.0)
```

    C       0.669590
    AAPL    0.343694
    CSCO    0.317718
    YHOO    0.416351
    IBM     0.228973
    dtype: float64

```python
rets = stk.pct_change()
rets.corr()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      C
       1.000000
       0.378732
       0.465752
       0.330307
       0.442698

      AAPL
       0.378732
       1.000000
       0.487141
       0.342638
       0.485708

      CSCO
       0.465752
       0.487141
       1.000000
       0.406206
       0.582500

      YHOO
       0.330307
       0.342638
       0.406206
       1.000000
       0.377730

      IBM
       0.442698
       0.485708
       0.582500
       0.377730
       1.000000

```python
rets.cov()
```

      C
      AAPL
      CSCO
      YHOO
      IBM

      C
       0.001779
       0.000346
       0.000393
       0.000365
       0.000269

      AAPL
       0.000346
       0.000469
       0.000211
       0.000195
       0.000152

      CSCO
       0.000393
       0.000211
       0.000401
       0.000213
       0.000168

      YHOO
       0.000365
       0.000195
       0.000213
       0.000688
       0.000143

      IBM
       0.000269
       0.000152
       0.000168
       0.000143
       0.000208

```python
sqrt(diag(rets.cov())*252.0)
```

    array([ 0.66958975,  0.3436938 ,  0.31771753,  0.41635095,  0.22897315])

```python
rets.corrwith(rets.AAPL)
```

    C       0.378732
    AAPL    1.000000
    CSCO    0.487141
    YHOO    0.342638
    IBM     0.485708
    dtype: float64

### STACKING UP DATA IN PANELS

```python
import pandas.io.data as pid
panel = pd.Panel(dict((stock, pid.get_data_yahoo(stock,'1/1/2014','2/28/2015')) for stock in ['C','AAPL','CSCO','YHOO','IBM']))
panel
```

    <class 'pandas.core.panel.Panel'>
    Dimensions: 5 (items) x 291 (major_axis) x 6 (minor_axis)
    Items axis: AAPL to YHOO
    Major_axis axis: 2014-01-02 00:00:00 to 2015-02-27 00:00:00
    Minor_axis axis: Open to Adj Close

```python
panel = panel.swapaxes('items','minor')
panel
```

    <class 'pandas.core.panel.Panel'>
    Dimensions: 6 (items) x 291 (major_axis) x 5 (minor_axis)
    Items axis: Open to Adj Close
    Major_axis axis: 2014-01-02 00:00:00 to 2015-02-27 00:00:00
    Minor_axis axis: AAPL to YHOO

```python
panel['Adj Close'].head()
```

      AAPL
      C
      CSCO
      IBM
      YHOO

      Date

      2014-01-02
       77.09
       52.22
       21.18
       179.87
       39.59

      2014-01-03
       75.39
       53.35
       21.16
       180.95
       40.12

      2014-01-06
       75.80
       53.76
       21.19
       180.33
       39.93

      2014-01-07
       75.26
       54.12
       21.47
       183.92
       40.92

      2014-01-08
       75.74
       54.75
       21.45
       182.24
       41.02

```python
panel.ix[:,'1/3/2014',:]
```

      Open
      High
      Low
      Close
      Volume
      Adj Close

      AAPL
       552.86
       553.70
       540.43
       540.98
       98116900
        75.39

      C
        52.39
        53.47
        52.31
        53.40
       26884900
        53.35

      CSCO
        22.09
        22.12
        21.83
        21.98
       36328200
        21.16

      IBM
       185.83
       187.35
       185.30
       186.64
        4063200
       180.95

      YHOO
        40.16
        40.44
        39.82
        40.12
       15755200
        40.12

### READING DATA FROM FILES

```python
import pandas as pd
data = pd.read_table("markowitzdata.txt")
```

```python
data.head()
```

      DATE
      SUNW
      MSFT
      IBM
      CSCO
      AMZN
      mktrf
      smb
      hml
      rf

      0
       20010102
      -0.087444
       0.000000
      -0.002206
      -0.129085
      -0.108434
      -0.0345
      -0.0037
       0.0209
       0.00026

      1
       20010103
       0.297297
       0.105187
       0.115696
       0.240150
       0.265766
       0.0527
       0.0097
      -0.0493
       0.00026

      2
       20010104
      -0.060606
       0.010430
      -0.015192
       0.013616
      -0.117438
      -0.0121
       0.0083
      -0.0015
       0.00026

      3
       20010105
      -0.096774
       0.014194
       0.008719
      -0.125373
      -0.060484
      -0.0291
       0.0027
       0.0242
       0.00026

      4
       20010108
       0.006696
      -0.003817
      -0.004654
      -0.002133
       0.025751
      -0.0037
      -0.0053
       0.0129
       0.00026

```python
gdata = pd.read_csv("goog.csv")
```

```python
gdata.head()
```

      Date
      Open
      High
      Low
      Close
      Volume
      Adj Close

      0
       2011-04-06
       572.18
       575.16
       568.00
       574.18
       2668300
       574.18

      1
       2011-04-05
       581.08
       581.49
       565.68
       569.09
       6047500
       569.09

      2
       2011-04-04
       593.00
       594.74
       583.10
       587.68
       2054500
       587.68

      3
       2011-04-01
       588.76
       595.19
       588.76
       591.80
       2613200
       591.80

      4
       2011-03-31
       583.00
       588.16
       581.74
       586.76
       2029400
       586.76

```python
scatter(data["mktrf"],data["IBM"])
xlabel("Market return")
ylabel("IBM return")
grid(True)
```

```python
from scipy import stats
y = data["IBM"]
x = data["mktrf"]
b, a, r_value, p_value, std_err = stats.linregress(x,y)
print "Intercept = ",a
print "slope (beta) = ",b
```

    Intercept =  0.00011890050765
    slope (beta) =  1.10915240511

### BRINGING IT ALL TOGETHER

```python
import pandas as pd
import pandas.io.data as web
aapl = web.DataReader('AAPL',data_source='google',start='1/1/2104',end='4/1/2015')
aapl.head()
```

      Open
      High
      Low
      Close
      Volume

      Date

      2014-04-02
       77.48
       77.64
       77.18
       77.51
       45104871

      2014-04-03
       77.34
       77.50
       76.81
       76.97
       40648111

      2014-04-04
       77.12
       77.14
       75.80
       75.97
       68812485

      2014-04-07
       75.43
       75.84
       74.56
       74.78
       72462530

      2014-04-08
       75.03
       75.16
       74.10
       74.78
       60971883

```python
aapl.tail()
```

      Open
      High
      Low
      Close
      Volume

      Date

      2015-03-26
       122.76
       124.88
       122.60
       124.24
       47572869

      2015-03-27
       124.57
       124.70
       122.91
       123.25
       39546151

      2015-03-30
       124.05
       126.40
       124.00
       126.37
       47099670

      2015-03-31
       126.09
       126.49
       124.36
       124.43
       42090553

      2015-04-01
       124.82
       125.12
       123.10
       124.25
       40621437

```python
aapl['cont_ret'] = log(aapl['Close']/aapl['Close'].shift(1))
aapl.head()
```

      Open
      High
      Low
      Close
      Volume
      cont_ret

      Date

      2014-04-02
       77.48
       77.64
       77.18
       77.51
       45104871
            NaN

      2014-04-03
       77.34
       77.50
       76.81
       76.97
       40648111
      -0.006991

      2014-04-04
       77.12
       77.14
       75.80
       75.97
       68812485
      -0.013077

      2014-04-07
       75.43
       75.84
       74.56
       74.78
       72462530
      -0.015788

      2014-04-08
       75.03
       75.16
       74.10
       74.78
       60971883
       0.000000

```python
aapl['Vols'] = pd.rolling_std(aapl['cont_ret'],window=5)*sqrt(252.0)
aapl.tail()
```

      Open
      High
      Low
      Close
      Volume
      cont_ret
      Vols

      Date

      2015-03-26
       122.76
       124.88
       122.60
       124.24
       47572869
       0.006946
       0.237901

      2015-03-27
       124.57
       124.70
       122.91
       123.25
       39546151
      -0.008000
       0.230939

      2015-03-30
       124.05
       126.40
       124.00
       126.37
       47099670
       0.024999
       0.301835

      2015-03-31
       126.09
       126.49
       124.36
       124.43
       42090553
      -0.015471
       0.318742

      2015-04-01
       124.82
       125.12
       123.10
       124.25
       40621437
      -0.001448
       0.247050

```python
aapl.head(10)
```

      Open
      High
      Low
      Close
      Volume
      cont_ret
      Vols

      Date

      2014-04-02
       77.48
       77.64
       77.18
       77.51
       45104871
            NaN
            NaN

      2014-04-03
       77.34
       77.50
       76.81
       76.97
       40648111
      -0.006991
            NaN

      2014-04-04
       77.12
       77.14
       75.80
       75.97
       68812485
      -0.013077
            NaN

      2014-04-07
       75.43
       75.84
       74.56
       74.78
       72462530
      -0.015788
            NaN

      2014-04-08
       75.03
       75.16
       74.10
       74.78
       60971883
       0.000000
            NaN

      2014-04-09
       74.66
       75.78
       74.57
       75.76
       51542722
       0.013020
       0.183481

      2014-04-10
       75.81
       76.03
       74.74
       74.78
       59912818
      -0.013020
       0.193232

      2014-04-11
       74.14
       74.69
       73.88
       74.23
       67975012
      -0.007382
       0.183662

      2014-04-14
       74.56
       74.59
       73.89
       74.53
       51445177
       0.004033
       0.160238

      2014-04-15
       74.32
       74.52
       73.05
       73.99
       66622577
      -0.007272
       0.166516

```python
aapl[['Close','Vols']].plot(subplots=True,color='blue',figsize=(8,6))
```

### DIVERSIFICATION

DIVERSIFICATION OF A PORTFOLIO

It is useful to examine the power of using vector algebra with an application. Here we use vector and summation math to understand how diversification in stock portfolios works. Diversification occurs when we increase the number of non-perfectly correlated stocks in a portfolio, thereby reducing portfolio variance. In order to compute the variance of the portfolio we need to use the portfolio weights ${\bf w}$ and the covariance matrix of stock returns ${\bf R}$, denoted ${\bf \Sigma}$. We first write down the formula for a portfolio's return variance:

\begin{equation}
Var(\boldsymbol{w'R}) = \boldsymbol{w'\Sigma w} = \sum_{i=1}^n \boldsymbol{w_i^2 \sigma_i^2}
+ \sum_{i=1}^n \sum_{j=1,i \neq j}^n \boldsymbol{w_i w_j \sigma_{ij}}
\end{equation}

Readers are strongly encouraged to implement this by hand for $n=2$ to convince themselves that the vector form of the expression for variance $\boldsymbol{w'\Sigma w}$ is the same thing as the long form on the right-hand side of the equation above.
If returns are independent, then the formula collapses to:

\begin{equation}
Var(\bf{w'R}) = \bf{w'\Sigma w} = \sum_{i=1}^n \boldsymbol{w_i^2 \sigma_i^2}
\end{equation}

If returns are dependent, and equal amounts are invested in
each asset ($w_i=1/n,\;\;\forall i$):

\begin{eqnarray*}
Var(\bf{w'R}) &=&  \frac{1}{n}\sum_{i=1}^n \frac{\sigma_i^2}{n}
+ \frac{n-1}{n}\sum_{i=1}^n \sum_{j=1,i \neq j}^n  \frac{\sigma_{ij}}{n(n-1)}\\
&=& \frac{1}{n} \bar{\sigma_i}^2 + \frac{n-1}{n} \bar{\sigma_{ij}}\\
&=& \frac{1}{n} \bar{\sigma_i}^2 + \left(1 - \frac{1}{n} \right) \bar{\sigma_{ij}}
\end{eqnarray*}

The first term is the average variance, denoted $\bar{\sigma_1}^2$ divided by $n$, and the second is the average covariance, denoted $\bar{\sigma_{ij}}$ multiplied by factor $(n-1)/n$. As $n \rightarrow \infty$,

\begin{equation}
Var({\bf w'R}) = \bar{\sigma_{ij}}
\end{equation}

This produces the remarkable result that in a well diversified portfolio, the variances of each stock's return does not matter at all for portfolio risk! Further the risk of the portfolio, i.e., its variance, is nothing but the average of off-diagonal terms in the covariance matrix.

```python
sd=0.20; cv=0.01; m=100
n = range(1,m+1)
sd_p = zeros(m)
for j in n:
    cv_mat = matrix(ones((j,j))*cv)
    fill_diagonal(cv_mat,sd**2)
    w = matrix(ones(j)*(1.0/j)).T
    sd_p[j-1] = sqrt((w.T).dot(cv_mat).dot(w))
```

```python
plot(n,sd_p)
xlabel('#stocks')
ylabel('stddev of portfolio')
grid()
```

### Arithmetic vs Geometric mean returns

The geometric mean is a good indicator of past performance, especially when we are interested in holding period returns. But if we are interested in expected future returns, the arithmetic mean is the relevant statistic.

Suppose a stock will rise by 30% or fall by 20% with equal probability. If it did one each in consecutive years then the geometric mean return is:

```python
g_ret = ((1+0.30)*(1-0.20))**0.5-1
print "Geometric mean return = ", g_ret
```

```python
a_ret = 0.5*(0.30+(-0.20))
print "Arithmetic mean return per year = ",a_ret
```

Which one is more realistic in predicting future expected returns over the next two years? Note that there are 4 cases to consider for outcomes, all with equal probability $1/4$.

```python
ret = zeros(4)
ret[0] = (1+0.3)*(1+0.3)
ret[1] = (1+0.3)*(1-0.2)
ret[2] = (1-0.2)*(1+0.3)
ret[3] = (1-0.2)*(1-0.2)
two_year_return = 0.25*sum(ret)
print "Expected two year return = ", two_year_return
print "Expected two year return (annualized) = ", two_year_return**0.5
```

```python

```


```python
# gradient descent
# derivatives = change in value with change in arguments (indicates min, max, saddle).
# Interested in direction of fastest decrease in cost function (-nablaC)
```

```python
import numpy as np

def gradient_descent(gradient, start, learn_rate, epoch):
    vector = start
    for _ in range(epoch):
        diff = -learn_rate * gradient(vector)
        vector += diff
    return vector
```

```python
gradient_descent(gradient=lambda v: 2 * v, start=10.0, learn_rate=0.2, epoch = 10)
```

    0.06046617599999997

```python
import matplotlib.pyplot as plt
import numpy as np

def graph(formula, x_range):
    x = np.array(x_range)
    #^ use x as range variable
    y = formula(x)
    #^          ^call the lambda expression with x
    #| use y as function result
    plt.plot(x,y)
    plt.show()

graph(lambda x : (2 * x), range(-10, 10))
#     ^use a lambda expression
```

```python
#concerned with the problem of finding the rate of change of a function with respect to the variable on which it depends
```

    Requirement already satisfied: sympy in /Users/masonkadem/opt/anaconda3/lib/python3.8/site-packages (1.6.2)
    Requirement already satisfied: mpmath>=0.19 in /Users/masonkadem/opt/anaconda3/lib/python3.8/site-packages (from sympy) (1.1.0)

```python
import numpy as np
from sklearn.metrics import mean_squared_error

def gradient_descent(X, y, learningrate = 0.05, epoch = 10):

    '''
   Gradient descent math
    '''

    m, b = 0.33, 0.48 # parameters
    log, mse = [], [] # lists to store learning process
    N = len(X) # number of samples

    for _ in range(epoch):

        f = y - (m*X + b)

        # Updating m and b
        m -= lr * (-2 * X.dot(f).sum() / N)
        b -= lr * (-2 * f.sum() / N)

        log.append((m, b))
        mse.append(mean_squared_error(y, (m*X + b)))

    return m, b, log, mse
```


```python
import math
import sympy as sp
from sympy.utilities.lambdify import lambdify
import numpy as np
import matplotlib.pyplot as plt

import openrtdynamics2.lang as dy
import openrtdynamics2.py_execute as dyexe
import openrtdynamics2.targets as tg

from vehicle_lib.vehicle_lib import *

```

# Kinematic bicycle model

The dynamic system equations are given by

$
\dot X
=
f(x,y,\psi)
=
\begin{pmatrix}
  \dot x \\
  \dot y \\
  \dot \psi
\end{pmatrix}
=
\begin{pmatrix}
  v \cos( \delta + \psi) \\
  v \sin( \delta + \psi) \\
  v / l_r \sin( \delta ), \\
\end{pmatrix}
$

with the state vector

$ X = [ x, y, \psi ]^T $.

Herein, $x$ and $y$ denote the coordinates of the vehicle front axle in cartesian space and $\psi$ the vehicle body orientation angle. The system inputs are the steering angle $\delta$ and the vehicle velocity $v$. Finally, the parameter $l_r$ denotes the wheelbase, which is the length in-between front and rear axle.

```python
x, y, v, delta, psi, l_r, T_s, n = sp.symbols('x y v delta psi l_r T_s n')

x_dot   = v * sp.cos( delta + psi )
y_dot   = v * sp.sin( delta + psi )
psi_dot = v / l_r * sp.sin( delta )

# system function f
f = sp.Matrix([ x_dot, y_dot, psi_dot ])

# state vector
X_bic = sp.Matrix( [ x, y, psi ])

# input vector
U_bic = sp.Matrix( [ delta, v ])
```

```python
f
```

$\displaystyle \left[\begin{matrix}v \cos{\left(\delta + \psi \right)}\\v \sin{\left(\delta + \psi \right)}\\\frac{v \sin{\left(\delta \right)}}{l_{r}}\end{matrix}\right]$

# Discretization of the continunous model

By applying Euler-forward discretization

$ {X}[k+1] = \underbrace{ {X}[k] + T_s  \dot{X} }_{f_{dscr}} $,

the continuous system is time-discretized with the sampling time $T_s$ yielding the discrete system funtion $f_{dscr}$.

```python
# apply Euler forward
f_dscr = sp.Matrix( [x,y,psi]) + T_s * f
```

```python
f_dscr
```

$\displaystyle \left[\begin{matrix}T_{s} v \cos{\left(\delta + \psi \right)} + x\\T_{s} v \sin{\left(\delta + \psi \right)} + y\\\frac{T_{s} v \sin{\left(\delta \right)}}{l_{r}} + \psi\end{matrix}\right]$

# Analytically compute the Jacobian matrices

A linearization of the non-linear system function around a dynamic set point is calculated by deriving the jacobian matrices w.r.t. the state vector $X$ and each system input $\delta$ and $v$:

continuous case
$
  A = \frac{ \partial f }{ \partial X},
  \qquad
  B = \frac{ \partial f }{ \partial [ \delta, v ]^T },
$

discrete-time case
$
  A_{dscr} = \frac{ \partial f_{dscr} }{ \partial X},
  \qquad
  B_{dscr} = \frac{ \partial f_{dscr} }{ \partial [ \delta, v ]^T },
$

```python
# continuous system matrices
A = f.jacobian(X_bic)
B = f.jacobian(U_bic)

# discrete system matrices
A_dscr = f_dscr.jacobian(X_bic)
B_dscr = f_dscr.jacobian(U_bic)
```

```python
A_dscr
```

$\displaystyle \left[\begin{matrix}1 & 0 & - T_{s} v \sin{\left(\delta + \psi \right)}\\0 & 1 & T_{s} v \cos{\left(\delta + \psi \right)}\\0 & 0 & 1\end{matrix}\right]$

```python
B_dscr
```

$\displaystyle \left[\begin{matrix}- T_{s} v \sin{\left(\delta + \psi \right)} & T_{s} \cos{\left(\delta + \psi \right)}\\T_{s} v \cos{\left(\delta + \psi \right)} & T_{s} \sin{\left(\delta + \psi \right)}\\\frac{T_{s} v \cos{\left(\delta \right)}}{l_{r}} & \frac{T_{s} \sin{\left(\delta \right)}}{l_{r}}\end{matrix}\right]$

# Create functions that generate the matrices A, B, and the system function f

Create python functions with which the symbolically dervied matrices and system function can be evaluated.

```python
variables = (T_s,l_r, x,y,psi,v,delta)

array2mat = [{'ImmutableDenseMatrix': np.matrix}, 'numpy']

A_dscr_fn = lambdify( variables, A_dscr, modules=array2mat)
B_dscr_fn = lambdify( variables, B_dscr, modules=array2mat)
f_dscr_fn = lambdify( variables, f_dscr, modules=array2mat)

```

```python
A_dscr_fn(0.01, 3.0, 0.1,0.2,0.4,10,0.1)
```

    matrix([[ 1.        ,  0.        , -0.04794255],
            [ 0.        ,  1.        ,  0.08775826],
            [ 0.        ,  0.        ,  1.        ]])

```python
B_dscr_fn(0.01, 3.0, 0.1,0.2,0.4,10,0.1)
```

    matrix([[-0.04794255,  0.00877583],
            [ 0.08775826,  0.00479426],
            [ 0.03316681,  0.00033278]])

```python
f_dscr_fn(0.01, 3.0, 0.1,0.2,0.4,10,0.1)
```

    matrix([[0.18775826],
            [0.24794255],
            [0.40332778]])

```python

```

# Run a simulation to generate test data

Set-up a simulation of a vehicle (bicycle model). The steering rate $\dot\delta$ is the input and integrated to obtain the steering angle $\delta$. A picewise constant (step signal) is applied to the input $\dot\delta$. The velocity $v$ is set to a constant value.

```python
system = dy.enter_system()

velocity               = dy.system_input( dy.DataTypeFloat64(1), name='velocity_',             default_value=5.0,    value_range=[0, 25],     title="vehicle velocity [m/s]")

s1                     = dy.system_input( dy.DataTypeFloat64(1), name='s1',                    default_value=0.0,    value_range=[-0.05, 0.05], title="steering rate 1 [rad/s]")
s2                     = dy.system_input( dy.DataTypeFloat64(1), name='s2',                    default_value=0.0,    value_range=[-0.05, 0.05], title="steering rate 2 [rad/s]")
s3                     = dy.system_input( dy.DataTypeFloat64(1), name='s3',                    default_value=0.0,    value_range=[-0.05, 0.05], title="steering rate 3 [rad/s]")
s4                     = dy.system_input( dy.DataTypeFloat64(1), name='s4',                    default_value=0.0,    value_range=[-0.05, 0.05], title="steering rate 4 [rad/s]")
s5                     = dy.system_input( dy.DataTypeFloat64(1), name='s5',                    default_value=0.0,    value_range=[-0.05, 0.05], title="steering rate 5 [rad/s]")

initial_steering       = dy.system_input( dy.DataTypeFloat64(1), name='initial_steering',      default_value=-0.0,   value_range=[-40, 40],   title="initial steering angle [degrees]")       * dy.float64(math.pi / 180.0)
initial_orientation    = dy.system_input( dy.DataTypeFloat64(1), name='initial_orientation',   default_value=0.0,    value_range=[-360, 360], title="initial orientation angle [degrees]")    * dy.float64(math.pi / 180.0)

# parameters
wheelbase = 3.0

# sampling time
Ts = 0.01

steering_rate = dy.float64(0)

cnt = dy.counter()

steering_rate = dy.conditional_overwrite(steering_rate, condition= cnt >= dy.int32(200), new_value=s1 )
steering_rate = dy.conditional_overwrite(steering_rate, condition= cnt >= dy.int32(400), new_value=s2 )
steering_rate = dy.conditional_overwrite(steering_rate, condition= cnt >= dy.int32(600), new_value=s3 )
steering_rate = dy.conditional_overwrite(steering_rate, condition= cnt >= dy.int32(800), new_value=s4 )
steering_rate = dy.conditional_overwrite(steering_rate, condition= cnt >= dy.int32(1000), new_value=s5 )

# linearly increasing steering angle
delta = dy.euler_integrator( steering_rate, Ts, initial_state=initial_steering )
delta = dy.saturate(u=delta, lower_limit=-math.pi/2.0, upper_limit=math.pi/2.0)

# the model of the vehicle
x, y, psi, x_dot, y_dot, psi_dot = discrete_time_bicycle_model(delta, velocity, Ts, wheelbase, psi0=initial_orientation)

#
# outputs: these are available for visualization in the html set-up
#

dy.append_output(x, 'x')
dy.append_output(y, 'y')
dy.append_output(psi, 'psi')
dy.append_output(psi_dot, 'psi_dot')
dy.append_output(delta, 'steering')
dy.append_output(velocity*dy.float64(1.0), 'velocity')

# generate code for Web Assembly (wasm), requires emcc (emscripten) to build
code_gen_results = dy.generate_code(template=tg.TargetCppWASM(), folder="generated/trajectory_generation", build=True)

#
dy.clear()

```

    compiling system simulation (level 0)...
    Generated code will be written to generated/trajectory_generation .
    writing file generated/trajectory_generation/simulation_manifest.json
    writing file generated/trajectory_generation/main.cpp
    Running compiler: emcc --bind -s MODULARIZE=1 -s EXPORT_NAME="ORTD_simulator" generated/trajectory_generation/main.cpp -O2 -s -o generated/trajectory_generation/main.js
    Compilation result:  0

```python
compiled_system = dyexe.CompiledCode(code_gen_results)
```

Run the simulation given parameters to define the steering rates

```python
testsim = dyexe.SystemInstance(compiled_system)

N=1000
input_data = {'initial_steering': 0.0, 's1' : 0.1, 's2' : -0.15, 's3' : 0.2 }

# sim_results = run_batch_simulation(testsim, input_data, N )
sim_results = dyexe.run_batch_simulation(testsim, input_data, N,  output_keys=['x', 'y', 'psi', 'psi_dot', 'steering', 'velocity'] )
```

# Add noise to the model (sensing noise)

Simulate measurement noise which is, e.g., introduced by GPS.

```python
# X/Y positioning noise (normal distribution)
eta_x = np.random.normal(0, 0.1, N)
eta_y = np.random.normal(0, 0.1, N)

x_meas = eta_x + sim_results['x']
y_meas = eta_y + sim_results['y']
psi_meas = sim_results['psi']
psi_dot_meas = sim_results['psi_dot']
v_meas = sim_results['velocity']

plt.figure()
plt.plot(x_meas, y_meas, '+')
plt.plot(sim_results['x'], sim_results['y'])
plt.show()

plt.figure()
plt.plot(psi_dot_meas)
plt.show()
```

# Extended Kalman filter

The extended Kalman filter is applied to the linearized model descibed by the matrices $A_{dscr}$ and $B_{dscr}$ and given the system function $f_{dscr}$. The simulated data serves as measured data and is the input to the filter.

```python
f_dscr
```

$\displaystyle \left[\begin{matrix}T_{s} v \cos{\left(\delta + \psi \right)} + x\\T_{s} v \sin{\left(\delta + \psi \right)} + y\\\frac{T_{s} v \sin{\left(\delta \right)}}{l_{r}} + \psi\end{matrix}\right]$

```python
A_dscr
```

$\displaystyle \left[\begin{matrix}1 & 0 & - T_{s} v \sin{\left(\delta + \psi \right)}\\0 & 1 & T_{s} v \cos{\left(\delta + \psi \right)}\\0 & 0 & 1\end{matrix}\right]$

```python
B_dscr
```

$\displaystyle \left[\begin{matrix}- T_{s} v \sin{\left(\delta + \psi \right)} & T_{s} \cos{\left(\delta + \psi \right)}\\T_{s} v \cos{\left(\delta + \psi \right)} & T_{s} \sin{\left(\delta + \psi \right)}\\\frac{T_{s} v \cos{\left(\delta \right)}}{l_{r}} & \frac{T_{s} \sin{\left(\delta \right)}}{l_{r}}\end{matrix}\right]$

The implemented filter in form of a loop

```python
l_r = wheelbase

# allocate space to store the filter results
results = {'delta' : np.zeros(N), 'x' : np.zeros(N), 'y' : np.zeros(N), 'psi' : np.zeros(N) }

# the guess/estimate of the initial states
X = np.matrix([ [0.5], [0.5], [0.1] ])

P = np.matrix([  [0.1, 0,   0     ],
                 [0,   0.1, 0     ],
                 [0,   0,   0.1   ]  ])

# covariance of the noise w addtitive to the states
Q = 0.00001*np.matrix([  [1, 0, 0 ],
                         [0, 1, 0 ],
                         [0, 0, 1 ]  ])

# covariance of the noise v in the measured system output signal
R = np.matrix([  [0.1, 0   ],
                 [0  , 0.1 ]  ])

for i in range(0,N):

    # measured input signals
    v       = v_meas[i]
    x       = x_meas[i]
    y       = y_meas[i]
    psi_dot = psi_dot_meas[i]

    # compute steering angle by the inverse for the vehicle orientation change
    delta = math.asin( psi_dot * l_r / v )

    # system output vector (x, y)
    z = np.matrix([ [x], [y] ])

    # pridiction step using the non-linear model (f_dscr)
    # x(k-1|k-1) --> x(k|k-1)

    X[0] = X[0] + Ts * ( v * math.cos( X[2] + delta ) )
    X[1] = X[1] + Ts * ( v * math.sin( X[2] + delta ) )
    X[2] = X[2] + Ts * ( v / l_r * math.sin(delta) )

    # optionally use the auto-generated python function for evaluation
    # X = f_dscr_fn( Ts, l_r, float(X[0]), float(X[1]), float(X[2]), v, delta )

    # evaluate jacobi matrices A_dscr and B_dscr
    F = np.matrix([  [1, 0, -Ts*v*math.sin(delta+X[2])  ],
                     [0, 1,  Ts*v*math.cos(delta+X[2])  ],
                     [0, 0, 1                           ]   ])

    # optionally use the auto-generated python function for evaluation
    # F = A_dscr_fn( Ts, l_r, float(X[0]), float(X[1]), float(X[2]), v, delta )

    B = np.matrix([  [-Ts*v*math.sin(delta+X[2]), Ts*math.cos(delta+X[2])      ],
                     [ Ts*v*math.cos(delta+X[2]), Ts*math.sin(delta+X[2])      ],
                     [Ts*v/l_r * math.cos(delta), Ts/l_r * math.sin(delta)     ]   ])

    # optionally use the auto-generated python function for evaluation
    # B = B_dscr_fn( Ts, l_r, float(X[0]), float(X[1]), float(X[2]), v, delta )

    # the system output matrix: returns X and Y when multiplied with the state vector X
    # which are compared to the measurements
    H = np.matrix([  [1,0,0],
                     [0,1,0] ])

    # prdicted state covariance P(k|k-1)
    P = F*P*F.transpose() + Q

    # estimation output residual vector
    e = z - H*X

    # Kalman gain
    S = H*P*H.transpose() + R
    K = P*H.transpose() * np.linalg.inv( S )

    # post priori state X(k|k)
    X = X + K*e

    # post priori covariance
    P = (np.eye(3) - K*H) * P

    # store results
    results['delta'][i] = delta
    results['x'][i] = X[0]
    results['y'][i] = X[1]
    results['psi'][i] = X[2]

# show results
plt.figure()
#plt.plot(x_meas, y_meas, '+')
plt.plot(sim_results['x'], sim_results['y'], 'g')
plt.plot(results['x'], results['y'])
plt.show()

plt.figure()
plt.plot(results['psi'])
plt.plot(sim_results['psi'], 'g')
plt.show()
```


```python
import sympy as sym
import numpy as np
```

```python
def rotationGlobalX(alpha):
    return np.array([[1,0,0],[0,np.cos(alpha),-np.sin(alpha)],[0,np.sin(alpha),np.cos(alpha)]])

def rotationGlobalY(beta):
    return np.array([[np.cos(beta),0,np.sin(beta)], [0,1,0],[-np.sin(beta),0,np.cos(beta)]])

def rotationGlobalZ(gamma):
    return np.array([[np.cos(gamma),-np.sin(gamma),0],[np.sin(gamma),np.cos(gamma),0],[0,0,1]])

def rotationLocalX(alpha):
    return np.array([[1,0,0],[0,np.cos(alpha),np.sin(alpha)],[0,-np.sin(alpha),np.cos(alpha)]])

def rotationLocalY(beta):
    return np.array([[np.cos(beta),0,-np.sin(beta)], [0,1,0],[np.sin(beta),0,np.cos(beta)]])

def rotationLocalZ(gamma):
    return np.array([[np.cos(gamma),np.sin(gamma),0],[-np.sin(gamma),np.cos(gamma),0],[0,0,1]])
```

```python
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
%matplotlib notebook
plt.rcParams['figure.figsize']=10,10

coefs = (1, 3, 15)  # Coefficients in a0/c x**2 + a1/c y**2 + a2/c z**2 = 1
# Radii corresponding to the coefficients:
rx, ry, rz = 1/np.sqrt(coefs)

# Set of all spherical angles:
u = np.linspace(0, 2 * np.pi, 30)
v = np.linspace(0, np.pi, 30)

# Cartesian coordinates that correspond to the spherical angles:
# (this is the equation of an ellipsoid):
x = rx * np.outer(np.cos(u), np.sin(v))
y = ry * np.outer(np.sin(u), np.sin(v))
z = rz * np.outer(np.ones_like(u), np.cos(v))

fig = plt.figure(figsize=plt.figaspect(1))  # Square figure
ax = fig.add_subplot(111, projection='3d')

xr = np.reshape(x, (1,-1))
yr = np.reshape(y, (1,-1))
zr = np.reshape(z, (1,-1))

RX = rotationGlobalX(np.pi/3)
RY = rotationGlobalY(np.pi/3)
RZ = rotationGlobalZ(np.pi/3)
Rx = rotationLocalX(np.pi/3)
Ry = rotationLocalY(np.pi/3)
Rz = rotationLocalZ(np.pi/3)

rRotx = RZ@RY@RX@np.vstack((xr,yr,zr))
print(np.shape(rRotx))
# Plot:
ax.plot_surface(np.reshape(rRotx[0,:],(30,30)), np.reshape(rRotx[1,:],(30,30)),
                np.reshape(rRotx[2,:],(30,30)), rstride=4, cstride=4, color='b')

# Adjustment of the axes, so that they all have the same span:
max_radius = max(rx, ry, rz)
for axis in 'xyz':
    getattr(ax, 'set_{}lim'.format(axis))((-max_radius, max_radius))

plt.show()
```

    <IPython.core.display.Javascript object>

    (3, 900)

```python

coefs = (1, 3, 15)  # Coefficients in a0/c x**2 + a1/c y**2 + a2/c z**2 = 1
# Radii corresponding to the coefficients:
rx, ry, rz = 1/np.sqrt(coefs)

# Set of all spherical angles:
u = np.linspace(0, 2 * np.pi, 30)
v = np.linspace(0, np.pi, 30)

# Cartesian coordinates that correspond to the spherical angles:
# (this is the equation of an ellipsoid):
x = rx * np.outer(np.cos(u), np.sin(v))
y = ry * np.outer(np.sin(u), np.sin(v))
z = rz * np.outer(np.ones_like(u), np.cos(v))

fig = plt.figure(figsize=plt.figaspect(1))  # Square figure
ax = fig.add_subplot(111, projection='3d')

xr = np.reshape(x, (1,-1))
yr = np.reshape(y, (1,-1))
zr = np.reshape(z, (1,-1))

RX = rotationGlobalX(np.pi/3)
RY = rotationGlobalY(np.pi/3)
RZ = rotationGlobalZ(np.pi/3)
Rx = rotationLocalX(np.pi/3)
Ry = rotationLocalY(np.pi/3)
Rz = rotationLocalZ(np.pi/3)

rRotx = RY@RX@np.vstack((xr,yr,zr))
print(np.shape(rRotx))
# Plot:
ax.plot_surface(np.reshape(rRotx[0,:],(30,30)), np.reshape(rRotx[1,:],(30,30)),
                np.reshape(rRotx[2,:],(30,30)), rstride=4, cstride=4, color='b')

# Adjustment of the axes, so that they all have the same span:
max_radius = max(rx, ry, rz)
for axis in 'xyz':
    getattr(ax, 'set_{}lim'.format(axis))((-max_radius, max_radius))

plt.show()
```

    <IPython.core.display.Javascript object>

    (3, 900)

```python
np.sin(np.arccos(0.7))
```

    0.71414284285428498

```python
print(RZ@RY@RX)
```

    [[ 0.25       -0.0580127   0.96650635]
     [ 0.4330127   0.89951905 -0.0580127 ]
     [-0.8660254   0.4330127   0.25      ]]

```python
import sympy as sym
sym.init_printing()
```

```python
a,b,g = sym.symbols('alpha, beta, gamma')
```

```python
RX = sym.Matrix([[1,0,0],[0,sym.cos(a),-sym.sin(a)],[0,sym.sin(a),sym.cos(a)]])
RY = sym.Matrix([[sym.cos(b),0,sym.sin(b)],[0,1,0],[-sym.sin(b),0,sym.cos(b)]])
RZ = sym.Matrix([[sym.cos(g),-sym.sin(g),0],[sym.sin(g),sym.cos(g),0],[0,0,1]])
RX,RY,RZ
```

```python
R = RZ@RY@RX
R
```

```python
mm = np.array([2.71, 10.22, 26.52])
lm = np.array([2.92, 10.10, 18.85])
fh = np.array([5.05, 41.90, 15.41])
mc = np.array([8.29, 41.88, 26.52])
ajc = (mm + lm)/2
kjc = (fh + mc)/2
```

```python
i = np.array([1,0,0])
j = np.array([0,1,0])
k = np.array([0,0,1])
v1 = kjc - ajc
v1 = v1 / np.sqrt(v1[0]**2+v1[1]**2+v1[2]**2)
v2 = (mm-lm) - ((mm-lm)@v1)*v1
v2 = v2/ np.sqrt(v2[0]**2+v2[1]**2+v2[2]**2)
v3 = k - (k@v1)*v1 - (k@v2)*v2
v3 = v3/ np.sqrt(v3[0]**2+v3[1]**2+v3[2]**2)
```

```python
v1
```

    array([ 0.12043275,  0.99126617, -0.05373394])

```python
R = np.array([v1,v2,v3])
RGlobal = R.T
RGlobal
```

    array([[ 0.12043275, -0.02238689,  0.99246903],
           [ 0.99126617,  0.05682604, -0.11900497],
           [-0.05373394,  0.99813307,  0.02903508]])

```python
alpha = np.arctan2(RGlobal[2,1],RGlobal[2,2])*180/np.pi
alpha
```

```python
beta = np.arctan2(-RGlobal[2,0],np.sqrt(RGlobal[2,1]**2+RGlobal[2,2]**2))*180/np.pi
beta
```

```python
gamma = np.arctan2(RGlobal[1,0],RGlobal[0,0])*180/np.pi
gamma
```

```python
R2 = np.array([[0, 0.71, 0.7],[0,0.7,-0.71],[-1,0,0]])
R2
```

    array([[ 0.  ,  0.71,  0.7 ],
           [ 0.  ,  0.7 , -0.71],
           [-1.  ,  0.  ,  0.  ]])

```python
alpha = np.arctan2(R[2,1],R[2,2])*180/np.pi
alpha
```

```python
gamma = np.arctan2(R[1,0],R[0,0])*180/np.pi
gamma
```

```python
beta = np.arctan2(-R[2,0],np.sqrt(R[2,1]**2+R[2,2]**2))*180/np.pi
beta
```

```python
R = RY@RZ@RX
R
```

```python
alpha = np.arctan2(-R2[1,2],R2[1,1])*180/np.pi
alpha
```

```python
gamma = 0
```

```python
beta = 90
```

```python
import sympy as sym

```

```python
sym.init_printing()
```

```python
a,b,g = sym.symbols('alpha, beta, gamma')
```

```python
RX = sym.Matrix([[1,0,0],[0,sym.cos(a), -sym.sin(a)],[0,sym.sin(a), sym.cos(a)]])
RY = sym.Matrix([[sym.cos(b),0, sym.sin(b)],[0,1,0],[-sym.sin(b),0, sym.cos(b)]])
RZ = sym.Matrix([[sym.cos(g), -sym.sin(g), 0],[sym.sin(g), sym.cos(g),0],[0,0,1]])
RX,RY,RZ
```

```python

```

```python
RXYZ = RZ*RY*RX
RXYZ
```

```python
RZXY = RZ*RX*RY
RZXY
```

```python

```


```python
import numpy as np
import sympy as sp
import matplotlib.pyplot as plt
import pandas as pd
from scipy.stats import f, t #use for f and t distribution

from IPython.display import display, Markdown, Math
```

```python
#sympy quicker variable naming
def var(v):
    return sp.symbols(v)

#display math latex
def print_lat(s):
    display(Math(s))
```

```python
class LeastSq:
    def __init__(self, pred, expl, df):
        '''
        Takes pandas dataframe, and makes x, Y based on pred Y,
        and explanatory variables x.
        '''
        self.df = df
        self.pred = pred
        self.expl = expl

        self.x, self.Y = self.gen_x_Y()

        self.dim = [self.x.shape, self.Y.shape]
        self.check_dim()

    def gen_x_Y(self):
        '''
        Generates x, Y from pd dataframe
        '''
        res_Y = np.matrix(self.df[self.pred].to_numpy())
        res_x = np.matrix([self.df[i].to_numpy() for i in self.expl])

        return res_x, res_Y

    #check dimension errors
    def check_dim(self):
        if self.dim[0][1] != self.dim[1][1] or self.dim[1][0] != 1:
            raise ValueError(f"Dimensions are incorrect, x: {self.dim[0]} Y: {self.dim[1]}")

    def linear(self):
        '''
        Given x, Y: solve for linear equation

        f(x1 ... xn) = A_1*x1 ... An * xn + B
                     = [a1, ... an, 1]x
                     = Ax

                     Ax = f

        for b = f
        Solved via AtAx = Atb
        '''

        A = self.x.T
        A = np.hstack((A, np.ones(self.dim[1]).T))

        b = self.Y.T

        AtA = np.matmul(A.T, A)
        Atb = np.matmul(A.T, b)

        ans = np.linalg.solve(AtA, Atb)

        return LQ_out(self.pred, self.expl, df, ans, A)

class LQ_out(LeastSq):
    def __init__(self, pred, expl, df, ans, A):
        '''
        Subclass of LeastSq, handles output results
        '''
        super().__init__(pred, expl, df)
        self.A = A

        #df = n - k - 1, s.t k = # explanatory variables
        self.df = self.dim[0][1]-self.dim[0][0]-1

        #ans vector
        self.ans = ans
        self.ans_dim = ans.shape

        #coefficients
        self.coefs = self.get_coefs()

        #latex/sympy reporting
        self.func = self.f_sympy()
        self.latex = sp.latex(self.func)

        #residual info
        self.resid = self.residuals()
        self.SSE = np.linalg.norm(self.resid.T)**2
        self.RSE = self.get_RSE()
        self.resid_quartiles = self.quartiles(self.resid)
        self.sigma = np.sqrt(self.SSE/self.df)

        #SST = SSE + SSM
        self.SST = np.linalg.norm(self.get_SST().T)**2
        self.SSM = self.SST - self.SSE

        #R-squared
        self.R_SQ, self.R_SQ_adj = self.get_R_SQ()

        #f-statistic
        self.F = self.get_F_stats()
        self.p_value = self.get_p_value()

        #t-statistic
        self.SE = self.get_SE()
        self.T = self.get_T_stats()
        self.coef_p_value = self.get_coef_p_value()
        self.signifs = self.signif_codes()

    def residuals(self):
        '''
        Given A, x, y, where x is the result of solving least squares, return error
        e = b - Ax
        '''

        b = self.Y.T

        Ax = np.matmul(self.A, self.ans)

        return b-Ax

    def dup_rows(self, a, indx, num_dups=1):
        '''
        Duplicates a row in matrix num_dups times
        '''
        return np.insert(a,[indx+1]*num_dups,a[[indx],:],axis=0)

    def signif(self, p):
        '''
        Finds significance code of single p-value
        '''
        dic = {"***": (0, 0.001),
               "**": (0.001, 0.01),
               "*": (0.01, 0.05),
               ".": (0.05, 0.1),
               " ": (0.1, 1)}

        for key, val in dic.items():
            if val[0] <= p < val[1]:
                return key

        return "not a p-value"

    def signif_codes(self):
        '''
        Generate significance code for all cexplanatory variable coefficients

        range based on values in above method dictionary.
        '''
        return [self.signif(i) for i in self.coef_p_value]

    def get_coef_p_value(self):
        '''
        Gets p-value of explanatory variable coefficients s.t:

        p_i = 2*(1-T^-1(|t|, df)), i = 1 ... k
        '''

        ps = 2*(1-t.cdf(np.abs(self.T), self.df ))

        return ps

    def get_T_stats(self):
        '''
        Gets t statistic s.t:

        t_i = beta_i/SE_i, i = 1 ... k
        '''

        return self.ans/self.SE.T

    def get_SE(self):
        '''
        Gets Standard Error of explanatory variable coefficients s.t:

        SE_i = sqrt((AtA)^-1 * S^2), for SEi_i, i = 1 ... k
        '''

        AtA_inv = np.linalg.inv(np.matmul(self.A.T, self.A))

        AtA_inv_diags = AtA_inv.diagonal()

        SE = np.sqrt(AtA_inv_diags*self.sigma**2)

        return SE

    def get_p_value(self):
        '''
        Gets p-value of f-statistic s.t:

        p = 1 - F^-1(f, k, df)
        '''
        x_dim = self.dim[0]

        p = 1 - f.cdf(self.F, x_dim[0], self.df)

        return float("%.4g" % p)

    def get_F_stats(self):
        '''
        Gets F-statistic s.t:

        f = (SSM/k)/(SSE/df)
        '''
        x_dim = self.dim[0]
        F = (self.SSM/x_dim[0])/(self.SSE/(self.df))

        return round(F, 2)

    def get_R_SQ(self):
        '''
        Gets both R-squared, and Adjusted R-squared s.t:

        R-squared = SSM/SST

        Adj. R-squared: (n-1)/df * R-squared - k/df
        '''
        rsq = self.SSM/self.SST

        x_dim = self.dim[0]
        rsq_adj = ((x_dim[1]-1)/(self.df))*rsq - (x_dim[0])/(self.df)

        return np.round([rsq, rsq_adj], 4)

    def get_SST(self):
        '''
        Gets SST, the square difference between each predictor variable
        and the mean of the predictor s.t:

        SST = sum (y_i - ybar)^2
        '''
        b = self.Y.T

        y_bar = np.array([[np.mean(self.Y)]]*self.dim[0][1])

        return b-y_bar

    def quartiles(self, dat):
        '''
        Finds quantiles, and min/max based on sorted dat, (meant for residuals)
        '''
        sorted_dat = np.sort(dat.A1)

        Qs_tags = ["Min", "1Q", "Median", "3Q", "Max"]

        Qs = {i : round(np.quantile(sorted_dat, indx*.25), 3) for indx, i in enumerate(Qs_tags)}

        return Qs

    def get_RSE(self):
        '''
        Gets Residual standard error, s.t:

        RSE = sqrt(SSE/df)
        '''

        RSE = np.sqrt(self.SSE/self.df)

        return round(RSE, 1)

    def get_coefs(self):
        '''
        Gets coefficients as a dictionary based on explantory variables,
        and intercept

        s.t linear equation is: tags["Intercept"] + sum (tags[expl] * x_i)
        '''
        tags = self.expl + ["Intercept"]
        return {i : round(float(self.ans[indx]), 3) for indx, i in enumerate(tags)}

    def f_sympy(self):
        '''
        Generate sympy function based on result of regression model.

        variable names based on chosen explanatory variables, in paranthesis.
        '''
        eq = 0

        for i in self.expl:
            eq += self.coefs[i] * var(f"({i})")

        eq += self.coefs["Intercept"]

        return eq

    def summary(self):
        '''
        Does summary function using computed values, based on the
        R function summary() for regression models.
        '''

        #Do Call
        sum_expls = " + ".join(x for x in self.expl)
        print(f"Call:\nlm(formula = {self.pred} ~ {sum_expls}, data = df)")

        #Do Residuals
        resid_tags = "\tMin \t 1Q\t Median\t 3Q \t Max"
        resid_qs = "\t".join(str(x) for x in list(self.resid_quartiles.values()))
        print("Residuals:")
        print(resid_tags.expandtabs(4))
        print(resid_qs, "\n")

        #Do Coefficients
        expl_tags = ["(Intercept)"] + self.expl
        #print(self.signifs)

        dats = [self.ans, self.SE.T, self.T, self.coef_p_value, ["{:<4}".format(i) for i in self.signifs]]
        dats = [np.asarray(i).reshape(-1) for i in dats]

        df = pd.DataFrame()
        df["Estimate"] = dats[0]
        df["Std. Error"] = dats[1]
        df["t value"] = dats[2]
        df["Pr(>|t|)"] = dats[3]
        df[""] = dats[4]

        df = df.apply(np.roll, shift=1)

        df.index = expl_tags

        print("Coefficients:")
        print(df)

        #Do Signif
        print("---")
        print("Signif. codes: 0 ’***’ 0.001 ’**’ 0.01 ’*’ 0.05 ’.’ 0.1 ’ ’ 1\n")

        #Do Resid SE
        print("Residual standard error:", self.RSE, "on", self.df,  "degrees of freedom")

        #Do Multiple R-Sq
        print(f"Multiple R-squared: {self.R_SQ}, Adjusted R-squared: {self.R_SQ_adj}")

        #Do F-stat
        print(f"F-statistic: {self.F}, on {self.dim[0][0]} and {self.df} DF, p-value: {self.p_value}")

```

```python
#Example (requires GPA.csv to work)
df = pd.read_csv("data/GPA.csv")
df.head()
```

      obs
      GPA
      HSM
      HSS
      HSE
      SATM
      SATCR
      SATW
      sex

      0
      1
      3.84
      10
      10
      10
      630
      570
      590
      2

      1
      2
      3.97
      10
      10
      10
      750
      700
      630
      1

      2
      3
      3.49
      8
      10
      9
      570
      510
      490
      2

      3
      4
      1.95
      6
      4
      8
      640
      600
      610
      1

      4
      5
      2.59
      8
      10
      9
      510
      490
      490
      2

```python
model = LeastSq("GPA", ["HSM", "HSS", "HSE"], df)
```

```python
res = model.linear()
```

```python
res.summary()
```

    Call:
    lm(formula = GPA ~ HSM + HSS + HSE, data = df)
    Residuals:
        Min      1Q  Median  3Q      Max
    -2.203	-0.393	0.142	0.545	1.496

    Coefficients:
                 Estimate  Std. Error   t value  Pr(>|t|)
    (Intercept)  0.069304    0.453656  0.152767  0.878793
    HSM          0.123246    0.054855  2.246774  0.026155  *
    HSS          0.136137    0.069951  1.946177  0.053554  .
    HSE          0.058478    0.065417  0.893916  0.372838
    ---
    Signif. codes: 0 ’***’ 0.001 ’**’ 0.01 ’*’ 0.05 ’.’ 0.1 ’ ’ 1

    Residual standard error: 0.7 on 146 degrees of freedom
    Multiple R-squared: 0.2277, Adjusted R-squared: 0.2119
    F-statistic: 14.35, on 3 and 146 DF, p-value: 3.036e-08

```python

```


### Considered model: four-bar linkage (= two link manipulator + one link manipulator + rigid coupling)

```python
%load_ext ipydex.displaytools
%matplotlib inline

import os
import sympy as sp
import numpy as npy  # we need `np` later
from sympy import sin, cos, pi
from sympy.interactive import printing
import symbtools as st
import symbtools.modeltools as mt
import symbtools.visualisation as vt
from symbtools.modeltools import Rz # Rotationsmatrix in the xy plane (c, -s; s, c)
import scipy.integrate as sc_integrate
import matplotlib.pyplot as pl
from assimulo.solvers import IDA # Imports the solver IDA from Assimulo
from assimulo.problem import Implicit_Problem # Imports the problem formulation from Assimulo
import ipydex
plt = pl

printing.init_printing(1)
```

    Could not find GLIMDA.

```python
t = sp.Symbol('t')  # time variable

# coordinates
np = 2
nq = 1
n = np + nq
pp = st.symb_vector("p1:{}".format(np+1)) ##:T
qq = st.symb_vector("q1:{}".format(nq+1)) ##:T

aa = st.symb_vector("a1:{}".format(nq+1)) ##:T

ttheta = st.row_stack(pp, qq) ##:T
tthetad = st.time_deriv(ttheta, ttheta) ##:T
tthetadd = st.time_deriv(ttheta, ttheta, order=2) ##:T
st.make_global(ttheta, tthetad, tthetadd)
```

```python
params = sp.symbols('s1, s2, s3, m1, m2, m3, J1, J2, J3, l1, l2, l3, l4, kappa1, kappa2, g')
parameter_values = list(dict(s1=1/2, s2=1/2, s3=1/2, m1=1, m2=1, m3=3, J1=1/12 , J2=1/12, J3=1/12,
                             l1=0.8, l2=1.5, l3=1.5, l4=2, kappa1=3/2 , kappa2=14.715, g=9.81).items())
st.make_global(params)

# ttau = sp.symbols('tau')
tau1, tau2 = ttau = st.symb_vector("tau1, tau2")

```

```python
Rz(q1) # Rotationsmatirx
```

Specify the geometry (joints G and centers of masses S)

```python
# uni vectors

ex = sp.Matrix([1, 0])
ey = sp.Matrix([0, 1])

# Basis 1 und 2
B1 = sp.Matrix([0, 0])
B2 = sp.Matrix([l4, 0])

# Coordinates two link manipulator
S1 = Rz(q1)*ex*s1 ##:

G1 = Rz(q1)*ex*l1 ##:

S2 = G1 + Rz(q1 + p1)*ex*s2 ##:

G2 = G1 + Rz(q1 + p1)*ex*l2 ##:

# one link manipulator
G2b = B2 + Rz(p2)*ex*l3 ##:

S3 = B2 + Rz(p2)*ex*s3 ##:

constraints = sp.Matrix([G2 - G2b]) ##:

# Time derivative
Sd1, Sd2, Sd3 = st.col_split(st.time_deriv(st.col_stack(S1, S2, S3), ttheta))
```

```python
# kinetic energy

T_rot = (J1*qdot1**2 + J2*(qdot1 + pdot1)**2 + J3*(pdot2)**2)/2
T_trans = ( m1*Sd1.T*Sd1  +  m2*Sd2.T*Sd2 +  m3*Sd3.T*Sd3 )/2

T = T_rot + T_trans[0] ##:

# potential energy
V = m1*g*S1[1] + m2*g*S2[1] + m3*g*S3[1] ##:
```

---

The next cell draws the fourbar and allows to interactively test the kinematics of the mechanism.

Use the sliders to adjust the coordinates. Activate the checkbox to respect the linkage constraints. Note that you can choose the keyword argument `free_vars` to be one of `q1`, `p1`, `p2`.

---

```python
# useful to test recent development of the lib

import importlib as il
il.reload(vt)

```

    <module 'symbtools.visualisation' from '/media/workcard/workstickdir/projekte/rst_python/symbtools-TUD-RST-Account/symbtools/visualisation.py'>

```python
vis = vt.Visualiser(ttheta, xlim=(-2, 4), ylim=(-3, 3))
vis.add_linkage(st.col_stack(B1, G1, G2,).subs(parameter_values))
vis.add_linkage(st.col_stack(G2b, B2).subs(parameter_values))

cnstrs = constraints.subs(parameter_values)

# Note: There are two possibilities for the kinematic chain to close (two solutions to the algebraic constraints)

# prefer "upward solution"
vis.interact(p1=(-4, 4, .1, pi/2), free_vars=q1, constraints=cnstrs)

# prefer "downward solution"
# vis.interact(p1=(-4, 4, .1, -pi/2), p2=(-4, 4, .1, -2.1), free_vars=q1, constraints=cnstrs)
```

    interactive(children=(Checkbox(value=False, description='solve constraints (fmin)'), FloatSlider(value=1.57079…

```python
print(list(npy.arange(3)))
```

    [0, 1, 2]

```python
external_forces = [0 , 0, tau1]

%time mod = mt.generate_symbolic_model(T, V, ttheta, external_forces, constraints=constraints)

```

    CPU times: user 5.32 s, sys: 0 ns, total: 5.32 s
    Wall time: 5.33 s

```python
# condition that endeffectors of the two manipulators are at the same place (x and y)

mod.constraints
```

```python
# ODE-part oft the equation

mod.eqns
```

### Creation of DAE System

```python
mod.constraints
```

```python
# generate a dae object

dae = mod.calc_dae_eq(parameter_values)
dae.generate_eqns_funcs()
```

```python
# show dae variables

dae.yy ##:T
dae.yyd ##:T
```

### Calculate consistent initial values by optimization (with given hints)

```python
yy0, yyd0 = dae.calc_consistent_init_vals(p1=0.3) ##:

t0 = 0

# check if all values are almost zero (-> initial values fulfill the model)
assert npy.allclose(dae.model_func(t0, yy0, yyd0), 0)
```

    Optimization terminated successfully.
             Current function value: 0.000000
             Iterations: 91
             Function evaluations: 176
    Optimization terminated successfully.
             Current function value: 0.000000
             Iterations: 33
             Function evaluations: 67

    (yy0, yyd0) := (array([ 0.3       ,  1.74961317,  0.50948621,  0.        ,  0.        ,  0.        , -0.27535424,  0.5455313 ]),
     array([  0.        ,   0.        ,   0.        ,  23.53968609,   2.82766884, -14.48960943,  -0.        ,   0.        ]))

    ---

```python
model = Implicit_Problem(dae.model_func, yy0, yyd0, t0)
model.name = 'fourbar linkage'

# indicate which components of y are differential (1) and which are purely algebraic (0)
# model.algvar = dae.diff_alg_vars ##:

sim = IDA(model)
# sim.suppress_alg = True #Necessary to set to True for >1 index problem

# values <= 30 produce lots of output
sim.verbosity = 0
```

```python
tfinal = 10.0        # Specify the final time
ncp = 500            # Number of communication points (number of return points)

# takes about 48 seconds
tt_sol, yy_sol, yyd_sol = sim.simulate(tfinal, ncp)
```

    No initialization defined for the problem.
    No finalization defined for the problem.
    Final Run Statistics: fourbar linkage

     Number of steps                                 : 3504
     Number of function evaluations                  : 5621
     Number of Jacobian evaluations                  : 199
     Number of function eval. due to Jacobian eval.  : 1592
     Number of error test failures                   : 100
     Number of nonlinear iterations                  : 5621
     Number of nonlinear convergence failures        : 0

    Solver options:

     Solver                       : IDA (BDF)
     Maximal order                : 5
     Suppressed algebr. variables : False
     Tolerances (absolute)        : 1e-06
     Tolerances (relative)        : 1e-06

    Simulation interval    : 0.0 - 10.0 seconds.
    Elapsed simulation time: 9.14327409299949 seconds.

```python
ttheta_sol = yy_sol[:, :mod.dae.ntt]
ttheta_d_sol = yy_sol[:, mod.dae.ntt:mod.dae.ntt*2]
```

```python
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7)); plt.sca(ax1)

ax1.plot(tt_sol, ttheta_sol)
ax1.set_title("angles")

ax2.plot(tt_sol, ttheta_d_sol)
ax2.set_title("angular velocities")
```

####  Visualization and Animation

```python
joint_pos_expr = st.col_stack(B1, G1, G2, B2).subs(parameter_values) ##:
joint_pos_func = st.expr_to_func(mod.tt, joint_pos_expr, keep_shape=True)
```

```python
# Create object for Animation

simanim = vt.SimAnimation(mod.xx[:3], tt_sol, yy_sol[:, :3], figsize=(14, 7))
simanim.add_visualiser(vis)

# plot first frame
simanim.plot_frame(0)
```

```python
# this might need to be adapted on other systems

plt.rcParams["animation.codec"] = "libvpx-vp9"  # codec for webm
plt.rcParams['animation.ffmpeg_path'] = os.path.join(os.getenv("CONDA_PREFIX"), "bin", "ffmpeg")
# plt.rcParams['animation.ffmpeg_path'] = '/usr/bin/ffmpeg'
# writer = animation.FFMpegWriter(fps=30)

fname="fourbar_linkage_free_movement2.webm"

# this might take some 1e1 seconds
# simanim.save(fname, dpi=30)
```

```python
vt.display_video_file(fname)
```


# SymPy Modules Reference

*Arthur Ryman* <br/>
*Last Updated: 2020-04-18*

This notebook contains the examples from the
[SymPy Modules Reference](https://docs.sympy.org/latest/modules/index.html).

## [Scalar and Vector Field Functionality](https://docs.sympy.org/latest/modules/vector/fields.html)

### Implementation in sympy.vector

#### Scalar and vector fields

```python
from sympy.vector import CoordSys3D
R = CoordSys3D('R')
v = 3*R.i + 4*R.j + 5*R.k
v
```

$\displaystyle (3)\mathbf{\hat{i}_{R}} + (4)\mathbf{\hat{j}_{R}} + (5)\mathbf{\hat{k}_{R}}$

```python
from sympy.vector import CoordSys3D
R = CoordSys3D('R')
electric_potential = 2*R.x**2*R.y
electric_potential
```

$\displaystyle 2 \mathbf{{x}_{R}}^{2} \mathbf{{y}_{R}}$

```python
from sympy.vector import CoordSys3D
R = CoordSys3D('R')
electric_potential = 2*R.x**2*R.y
from sympy import diff
diff(electric_potential, R.x)
```

$\displaystyle 4 \mathbf{{x}_{R}} \mathbf{{y}_{R}}$

#### The Del operator

```python
from sympy.vector import CoordSys3D, Del
C = CoordSys3D('C')
delop = Del()
gradient_field = delop(C.x*C.y*C.z)
gradient_field
```

$\displaystyle (\frac{\partial}{\partial \mathbf{{x}_{C}}} \mathbf{{x}_{C}} \mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}} + (\frac{\partial}{\partial \mathbf{{y}_{C}}} \mathbf{{x}_{C}} \mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{j}_{C}} + (\frac{\partial}{\partial \mathbf{{z}_{C}}} \mathbf{{x}_{C}} \mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{k}_{C}}$

```python
gradient_field.doit()
```

$\displaystyle (\mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}} + (\mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{j}_{C}} + (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{k}_{C}}$

### Field operators and related functions

#### Curl

```python
from sympy.vector import CoordSys3D, Del
C = CoordSys3D('C')
delop = Del()
delop.cross(C.x*C.y*C.z*C.i).doit()
```

$\displaystyle (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{j}_{C}} + (- \mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{k}_{C}}$

```python
(delop ^ C.x*C.y*C.z*C.i).doit()
```

$\displaystyle (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{j}_{C}} + (- \mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{k}_{C}}$

```python
from sympy.vector import curl
curl(C.x*C.y*C.z*C.i)
```

$\displaystyle (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{j}_{C}} + (- \mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{k}_{C}}$

#### Divergence

```python
from sympy.vector import CoordSys3D, Del
C = CoordSys3D('C')
delop = Del()
delop.dot(C.x*C.y*C.z*(C.i + C.j + C.k)).doit()
```

$\displaystyle \mathbf{{x}_{C}} \mathbf{{y}_{C}} + \mathbf{{x}_{C}} \mathbf{{z}_{C}} + \mathbf{{y}_{C}} \mathbf{{z}_{C}}$

```python
(delop & C.x*C.y*C.z*(C.i + C.j + C.k)).doit()
```

$\displaystyle \mathbf{{x}_{C}} \mathbf{{y}_{C}} + \mathbf{{x}_{C}} \mathbf{{z}_{C}} + \mathbf{{y}_{C}} \mathbf{{z}_{C}}$

```python
from sympy.vector import divergence
divergence(C.x*C.y*C.z*(C.i + C.j + C.k))
```

$\displaystyle \mathbf{{x}_{C}} \mathbf{{y}_{C}} + \mathbf{{x}_{C}} \mathbf{{z}_{C}} + \mathbf{{y}_{C}} \mathbf{{z}_{C}}$

#### Gradient

```python
from sympy.vector import CoordSys3D, Del
C = CoordSys3D('C')
delop = Del()
delop.gradient(C.x*C.y*C.z).doit()
```

$\displaystyle (\mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}} + (\mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{j}_{C}} + (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{k}_{C}}$

```python
delop(C.x*C.y*C.z).doit()
```

$\displaystyle (\mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}} + (\mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{j}_{C}} + (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{k}_{C}}$

```python
from sympy.vector import gradient
gradient(C.x*C.y*C.z)
```

$\displaystyle (\mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}} + (\mathbf{{x}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{j}_{C}} + (\mathbf{{x}_{C}} \mathbf{{y}_{C}})\mathbf{\hat{k}_{C}}$

#### Directional Derivative

```python
from sympy.vector import CoordSys3D, Del
C = CoordSys3D('C')
delop = Del()
vel = C.i + C.j + C.k
scalar_field = C.x*C.y*C.z
vector_field = C.x*C.y*C.z*C.i
(vel.dot(delop))(scalar_field)
```

$\displaystyle \mathbf{{x}_{C}} \mathbf{{y}_{C}} + \mathbf{{x}_{C}} \mathbf{{z}_{C}} + \mathbf{{y}_{C}} \mathbf{{z}_{C}}$

```python
(vel & delop)(vector_field)
```

$\displaystyle (\mathbf{{x}_{C}} \mathbf{{y}_{C}} + \mathbf{{x}_{C}} \mathbf{{z}_{C}} + \mathbf{{y}_{C}} \mathbf{{z}_{C}})\mathbf{\hat{i}_{C}}$

```python
from sympy.vector import directional_derivative
directional_derivative(C.x*C.y*C.z, 3*C.i + 4*C.j + C.k)
```

$\displaystyle \mathbf{{x}_{C}} \mathbf{{y}_{C}} + 4 \mathbf{{x}_{C}} \mathbf{{z}_{C}} + 3 \mathbf{{y}_{C}} \mathbf{{z}_{C}}$

### Field operator in orthogonal curvilinear coordinate system

```python
from sympy.vector import CoordSys3D
c = CoordSys3D('c', transformation='cylindrical', variable_names=('r', 'theta', 'z'))
gradient(c.r*c.theta*c.z)
```

$\displaystyle (\mathbf{{theta}_{c}} \mathbf{{z}_{c}})\mathbf{\hat{i}_{c}} + (\mathbf{{z}_{c}})\mathbf{\hat{j}_{c}} + (\mathbf{{r}_{c}} \mathbf{{theta}_{c}})\mathbf{\hat{k}_{c}}$

### Conservative and Solenoidal Fields

```python
from sympy.vector import CoordSys3D, is_conservative
R = CoordSys3D('R')
field = R.y*R.z*R.i + R.x*R.z*R.j + R.x*R.y*R.k
is_conservative(field)
```

    True

```python
curl(field)
```

$\displaystyle \mathbf{\hat{0}}$

```python
from sympy.vector import CoordSys3D, is_solenoidal
R = CoordSys3D('R')
field = R.y*R.z*R.i + R.x*R.z*R.j + R.x*R.y*R.k
is_solenoidal(field)
```

    True

```python
divergence(field)
```

$\displaystyle 0$

### Scalar potential functions

```python
from sympy.vector import CoordSys3D, scalar_potential
R = CoordSys3D('R')
conservative_field = 4*R.x*R.y*R.z*R.i + 2*R.x**2*R.z*R.j + 2*R.x**2*R.y*R.k
scalar_potential(conservative_field, R)
```

$\displaystyle 2 \mathbf{{x}_{R}}^{2} \mathbf{{y}_{R}} \mathbf{{z}_{R}}$

```python
from sympy.vector import CoordSys3D, Point
from sympy.vector import scalar_potential_difference
R = CoordSys3D('R')
P = R.origin.locate_new('P', 1*R.i + 2*R.j + 3*R.k)
vectfield = 4*R.x*R.y*R.i + 2*R.x**2*R.j
is_conservative(vectfield)
```

    True

```python
scalar_potential_difference(vectfield, R, R.origin, P)
```

$\displaystyle 4$

```python
scalarpot = scalar_potential(vectfield, R)
scalarpot
```

$\displaystyle 2 \mathbf{{x}_{R}}^{2} \mathbf{{y}_{R}}$

```python
scalarpot.subs({R.x: 0})
```

$\displaystyle 0$

```python
P.express_coordinates(R)
```

    (1, 2, 3)

```python
R.origin.express_coordinates(R)
```

    (0, 0, 0)

```python
P_coord = P.express_coordinates(R)
scalarpot.subs({R.x: P_coord[0], R.y: P_coord[1], R.z: P_coord[2]})
```

$\displaystyle 4$


# 演習課題（３）－２の計算例

ビットレートの計算式

\begin{equation}
r = \frac{1}{\Delta_t}\times N_1 \times N_0 \times (\ell_Y + \frac{\ell_{C_b}}{4}+\frac{\ell_{C_r}}{4})
\end{equation}

SMTPE296M(1280$\times$720, 4:2:0, 30P)のパラメータ

*   $\Delta_t=1/30$ [1/s]
*   $N_1 = 720$
*   $N_2 = 1280$
*   $\ell_Y=\ell_{C_b}=\ell_{C_r}=8$ [bits]

```
delta_t = 1/30
N1 = 720
N0 = 1280
ell = 8

r = (1/delta_t)*N1*N0*(ell + ell/4 + ell/4)

print('ビットレート r = {:4.2f}'.format(r))
```

    ビットレート r = 331776000.00