```python
import numpy as np
import control
import matplotlib.pyplot as plt # plotting library
from sympy import symbols
from sympy.physics.control.lti import TransferFunction, Feedback, Series
from sympy.physics.control.control_plots import pole_zero_plot, step_response_plot
```

# Equations of motion

The EOM for $m_1$ is:

$m_1 \ddot{x}_1+c_s \dot{x}_1+k_s x_1 = c_s \dot{x}_2 + k_s x_2$

The EOM for $m_2$ is:

$m_2 \ddot{x}_2+c_s \dot{x}_2+(k_s +k_t) x_2 = c_s \dot{x}_1 + k_s x_1 + k_t y$

# Block diagram

# Transfer Function

## Sympy Implementation

```python
m1, m2, kt, ks, cs, s = symbols('m_1 m_2 k_t k_s c_s s')

# have python perform the block diagram operations symbolically
num1 = cs*s + ks
den1 = m1*s**2 + cs*s + ks
G1 = TransferFunction(num1, den1, s)

num2 = cs*s + ks
den2 = m2*s**2 + cs*s + (ks+kt)
G2 = TransferFunction(num2,den2, s)

num3 = kt
den3 = m2*s**2 + cs*s + (ks+kt)
G3 = TransferFunction(num3,den3, s)

G4 = Feedback(G1,G2,sign=1).doit()
TF = Series(G3,G4).doit()

TF=TF.simplify()
TF
```

$\displaystyle \frac{- k_{t} \left(c_{s} s + k_{s}\right)}{\left(c_{s} s + k_{s}\right)^{2} - \left(c_{s} s + k_{s} + m_{1} s^{2}\right) \left(c_{s} s + k_{s} + k_{t} + m_{2} s^{2}\right)}$

Expanding this TF gives us

```python
TF.expand()
```

$\displaystyle \frac{- c_{s} k_{t} s - k_{s} k_{t}}{- c_{s} k_{t} s - c_{s} m_{1} s^{3} - c_{s} m_{2} s^{3} - k_{s} k_{t} - k_{s} m_{1} s^{2} - k_{s} m_{2} s^{2} - k_{t} m_{1} s^{2} - m_{1} m_{2} s^{4}}$

```python
# sub in values for the variables
TF = TF.subs([(kt,190000),(cs,5000),(ks,20000),(m1,290),(m2,59)])
```

```python
# Plot the poles and zeros of the TF
pole_zero_plot(TF)
```

```python
step_response_plot(TF,grid=False,color='r')
```

## Controls Package Implementation

```python
# cs = 5000
# ks = 20000
cs = 10
ks = 5
kt = 190000
m1 = 290
m2 = 59

num1 = [cs, ks]
den1 = [m1, cs, ks]
G1 = control.tf(num1,den1)

num2 = [cs, ks]
den2 = [m2, cs, (ks+kt)]
G2 = control.tf(num2,den2)

num3 = [kt]
den3 = [m2, cs, (ks+kt)]
G3 = control.tf(num3,den3)

G4 = control.feedback(G1,G2,sign=1)

G = control.series(G4,G3)
G = control.minreal(G)
print(G)
```

    2 states have been removed from the model

                  111 s + 55.52
    ------------------------------------------
    s^4 + 0.204 s^3 + 3220 s^2 + 111 s + 55.52

```python
poles, zeros = control.pzmap(G)
```

```python
time = np.arange(0,100,0.1) # time samples from 0 to 20 seconds with time spacing of 0.05 seconds

# t, xout = control.step_response(G,T=time)
t, xout = control.impulse_response(G,T=time)
```

```python
#############################################################
fig = plt.figure(figsize=(6,4))
ax = plt.gca()
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)

# Change the axis units font
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
plt.setp(ax.get_xmajorticklabels(),fontsize=18)

ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')

ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')

# Turn on the plot grid and set appropriate linestyle and color
ax.grid(True,linestyle=':', color='0.75')
ax.set_axisbelow(True)

# Define the X and Y axis labels
plt.xlabel('Time (s)', fontsize=22, weight='bold', labelpad=5)
plt.ylabel('Displacement', fontsize=22, weight='bold', labelpad=10)

plt.plot(time, xout, linewidth=2, linestyle='-', label=r'Response')
# uncomment below and set limits if needed
# plt.xlim(0,5)
# plt.ylim(-0.01,2.5)

# # Create the legend, then fix the fontsize
# leg = plt.legend(loc='upper right', ncol = 1, fancybox=True, )
# ltext  = leg.get_texts()
# plt.setp(ltext,fontsize=16)

# Adjust the page layout filling the page using the new tight_layout command
plt.tight_layout(pad=0.5)

# save the figure as a high-res pdf in the current folder
# plt.savefig('plot_filename.pdf')

# plt.show()
```

```python

```


# Using Indexed Expressions Efficiently: Examples and Exercises

## Author: Patrick Nelson
### Formatting improvements courtesy Brandon Clark

### NRPy+ Source Code for this module:
* TODO?

## Introduction:
NRPy+ provides several methods to declare variables and lists to represent the tensors, vectors, and scalars of general relativity. Each function has its use case, but when starting out, it is not always trivial to determine which is appropriate at any given time. This is further complicated by the fact that these use cases can change depending on whether or not the notebook at hand is going to output C code or not. So, this module will provide some pedagogy to help new users with these functions.

We will first explore the case in which one does not intend to immediately output C code. This is done in tutorials that solely generate symbolic expressions and their corresponding modules. For example, consider the tutorial [Tutorial-GRHD_Equations-Cartesian.ipynb](Tutorial-GRHD_Equations-Cartesian.ipynb) and the corresponding [module](../edit/GRHD/equations.py). The tutorial notebook gives in-depth, $\latex$ documentation about the GRHD expressions interspersed with the python code that generates the corresponding sympy expressions. The module provides the same python code, but in a format that makes it easy to `import` it into other Jupyter notebooks and modules. So, the tutorial also performs a self-validation check to guarantee that the notebook and module are generating identical sympy expressions.

Then, we will explore the case in which one is outputting C code. In this case, one will import the python modules described above to generate any symbolic expressions needed. These expressions can then be passed to `FD_outputC()`. But this function requires that *every* symbol in the expressions passed to it *must* be either a gridfunction or C parameter to help make sure that the resulting C code compiles correctly.

This tutorial assumes that the reader has previously looked over the previous module about indexed expressions in NRPy+, available [here](Tutorial-Indexed_Expressions.ipynb). If you have not read this, please do so before continuing to ensure that you are familiar with the basic syntax for the functions provided by `indexedexp.py`. This module is focused more on *how* and *when* to use each of those functions.

# Table of Contents
$$\label{toc}$$

This notebook is organized as follows

1. [Step 1](#no_c): Zero-rank, Declare-rank, Symbols
    1. [Step 1.a](#no_c_ex) Exercises
1. [Step 2](#gf_c): Gridfunctions and C parameters
    1. [Step 2.a](#gf_c_ex) Exercises
1. [Step 3](#applications): Applications
1. [Step 4](#key) Solutions for Exercises in Steps 2 & 3

# Step 1: Zero-rank, Declare-rank, Symbols \[Back to [top](#toc)\]
$$\label{no_c}$$

If we do not intend for a notebook to output C code, we only need to consider two families. In the first, we declare something symbolically; in the second, we set it to zero. For example, consider a simple index-lowering operation of a three-vector $A^i$ using the three-metric $\gamma_{ij}$:
$$
A_i = \gamma_{ij} A^j.
$$
We've written this in a way that suggests we will define new quantities, $A_i$, in terms of a known quantities, $A^i$ and $\gamma_{ij}$. The known quantities must be declared; depending on the rank of the quantity in question, this will use one of the following functions:
* `sympy.symbols()`
* `indexedexp.declarerank1()`
* `indexedexp.declarerank2()`
* `indexedexp.declarerank3()`
* `indexedexp.declarerank4()`

The parameters that these take are detailed in [a previous tutorial](Tutorial-Indexed_Expressions.ipynb). The new quantities belong to the other family. These must be zeroed before we can add things to them; this is analogous to initializing a variable in C. This is done with one of the following functions, depending on the rank of the quantities:
* `sympy.sympify(0)`
* `indexedexp.zerorank1()`
* `indexedexp.zerorank2()`
* `indexedexp.zerorank3()`
* `indexedexp.zerorank4()`

These all generate either zero or lists of zero that are compatible with sympify.

Let us again consider the example
$$
A_i = \gamma_{ij} A^j.
$$

```python
import indexedexp as ixp         # NRPy+: Symbolic indexed expression (e.g., tensors, vectors, etc.) support
import sympy as sp

# First we'll declare the knowns:
AU = ixp.declarerank1("AU",DIM=3)
gammaDD = ixp.declarerank2("gammaDD","sym01",DIM=3) # The metric tensor is symmetric, so we set it as such.

# Now, let's initialize the new quantity:
AD = ixp.zerorank1()

# Finally, we can loop over all the indices in the implied sum to express the new
# quantity in terms of the knowns:
for i in range(3):
    for j in range(3):
        AD[i] += gammaDD[i][j] * AU[j]
```

## Step 1.a: Exercises \[Back to [top](#toc)\]
$$\label{no_c_ex}$$

Code the following expressions, paying particular attention to the initial declarations and zeroings of tensors and scalars.

$$
a = \gamma_{ij} A^i A^j
$$

```python
# What are we trying to compute? What known quantities is that in terms of?
# Declare the knowns:

# And initialize the new quantity you want to compute to zero:

# Then loop through the indices to build the quantity:

```

$$
T^\xi_{\ \ \mu\nu} = k L^\xi M_\mu N_\nu
$$

```python
# Remember that Greek letters denote four-dimensional spacetime quantities!

```

# Step 2: Gridfunctions and C parameters \[Back to [top](#toc)\]
$$\label{gf_c}$$

Once we are ready to write a start-to-finish module, in which we generate, compile, and then run C code, we will also need to start registering gridfunctions and C parameters. Any expression passed to `FD_outputC()` must be entirely in terms of gridfunctions and C parameters; this requirement was put in place to help make sure that the automatically generated files cover everything that they need to do in order to minimize the amount of hand-coding in C that will need to be done. So, we will need to use the following functions:
* NRPy_param_funcs.Cparameters()
* grid.register_gridfunctions()
* indexedexp.register_gridfunctions_for_single_rank1()
* indexedexp.register_gridfunctions_for_single_rank2()

The zero-rank and declare-rank functions still have uses here, though. We will use `declarerank` whenever we want to take a finite-difference derivative; by appropriately naming it, NRPy+ will automatically generate code to differentiate a specified gridfunction as detailed in [its tutorial](Tutorial-Finite_Difference_Derivatives.ipynb). The `zerorank` functions find a use for intermediate expressions and for expressions whose variable name does not match the gridfunction's name (e.g. the right-hand side of an evolution equation. We will demonstrate these points with a mock PDE. Suppose that there is some vector $J_i$ that evolves according to the following equation:
$$
\partial_t J_i = \partial_j J^j K_i - G \Lambda_{i},
$$

where $G$ is Newton's gravitational constant and $\Lambda_{i} = J_i K_j K^j$. Suppose that, in this simulation, the quantities we want to evolve (and thus store) are $J_i$ and $K^i$

(N.B. This equation is not intended to correspond to any particular physics, merely to demonstrate many different use cases of NRPy+'s indexed expressions utilities.)

```python
import NRPy_param_funcs as par   # NRPy+: Parameter interface
import grid as gri               # NRPy+: Functions having to do with numerical grids

```

## Step 2.a: Exercises \[Back to [top](#toc)\]
$$\label{gf_c_ex}$$

```python

```

# Step 3: Applications \[Back to [top](#toc)\]
$$\label{applications}$$

# Step 4: Solutions for Exercises in Steps 2 & 3 \[Back to [top](#toc)\]
$$\label{key}$$

$$
a = \gamma_{ij} A^i A^j
$$

```python
gammaDD = ixp.declarerank2("gammaDD","sym01",DIM=3)
AU = ixp.declarerank1("AU",DIM=3)

a = sp.sympify(0)

for i in range(3):
    for j in range(3):
        a += gammaDD[i][j] * AU[i] * AU[j]
```

$$
T^\xi_{\ \ \mu\nu} = k L^\xi M_\mu N_\nu
$$

```python
k = sp.symbols("k",real=True)
LU = ixp.declarerank1("LU",DIM=4)
MD = ixp.declarerank1("MD",DIM=4)
ND = ixp.declarerank1("ND",DIM=4)

TUDD = ixp.zerorank3()

for xi in range(4):
    for mu in range(4):
        for nu in range(4):
            TUDD[xi][mu][nu] += k * LU[xi] * MD[mu] * ND[nu]

```


| |Pierre Proulx, ing, professeur|
|:---|:---|
|Département de génie chimique et de génie biotechnologique |** GCH200-Phénomènes d'échanges I **|

## Exemple 7-6.1

#### Solution

##### Bilan de masse
$\begin{equation*}
  \boxed{\rho v_1 S_1 - \rho v_2 S_2 = 0}
  \end{equation*}$ (1)

##### Bilan de quantité de mouvement linéaire dans la direction z

$\begin{equation*}
  \boxed{ ( v_1w_1 + p_1 S_1)-( v_2w_2 + p_2 S_2)  =
          \vec F_{fluide \rightarrow surface}}
  \end{equation*}$ (2)

mais la force que le fluide exerce sur la conduite peut être évalué en reconnaissant que la force exercée par le fluide sur la conduite dans la direction z est principalement la pression exercée par le fluide sur la surface annulaire juste après l'expansion. Donc $F =-p_1  (S_2-S_1)$ En substituant dans (2):

$\begin{equation*}
  \boxed{ ( v_1w + p_1 S_1)-( v_2w + p_2 S_2)  =
          -p_1  (S_2-S_1)}
  \end{equation*}$

Souvenons-nous que $w_1=w_2=w$ et que $\rho v_1S_1=\rho v_2 S_2$

$\begin{equation*}
  \boxed{ S_1( \rho v_1^2 + p_1) - S_2( \rho v_2^2 + p_2)  =
          -p_1  (S_2-S_1)}
  \end{equation*}$

ou

$\begin{equation*}
  \boxed{ S_1 \rho v_1^2  - S_2 \rho v_2^2 = -S_2(p_1-p_2 )}
  \end{equation*}$ (3)

## 4. Bilan d'énergie mécanique

$\begin{equation*}
  \boxed{  \bigg ( \frac {1}{2} v_1^2 +  \frac {p_1}{\rho_1} \bigg)-
           \bigg ( \frac {1}{2} v_2^2 +  \frac {p_2}{\rho_2} \bigg )=
          E_v}
  \end{equation*}$

```python
# Préparation de l'affichage et des outils de calcul symbolique
#
import sympy as sp
from IPython.display import *
sp.init_printing(use_latex=True)
%matplotlib inline
#
v_1,v_2,p_1,p_2,S_1,S_2,rho,E_v=sp.symbols('v_1,v_2,p_1,p_2,S_1,S_2,rho,E_v')
eq1=sp.Eq(v_1*S_1-v_2*S_2)
eq2=sp.Eq(v_1**2*S_1*rho-v_2**2*S_2*rho+S_2*(p_1-p_2))
eq3=sp.Eq(1/2*v_1**2+p_1/rho-1/2*v_2**2+p_2/rho -E_v)
solution=sp.solve((eq1,eq2,eq3),v_2,p_2,E_v)
display(solution[0][2].simplify())
```

```python

```


[](https://colab.research.google.com/github/Rishit-dagli/TFUG-Mysuru-2020/blob/master/TFQuantum_starter.ipynb)

# Getting started with [TensorFlow Quantum](https://www.tensorflow.org/quantum)

In this notebook you will build your first hybrid quantum classical model with
[Cirq](https://cirq.readthedocs.io/en/stable/) and TensorFlow Quantum (TFQ). We will build a very simple model to do
binary classification in this notebook. You will then use Keras to create a wrapper for the model and simulate it to
train and evluate the model.

> Note: This notebook is designed to be run in Google Colab if you want to run it locally or on a Jupyter notebook you
would skip the code cells with the `Colab only` comment.

## Setup

### Install TensorFlow 2.x (Colab only)

```python
# Colab only
pip install -q tensorflow==2.1.0
```

### Install TensorFlow Quantum (Colab only)

```python
# Colab only
pip install -q tensorflow-quantum
```

### Imports

Now import TensorFlow and the module dependencies:

```python
import cirq
import random
import numpy as np
import sympy
import tensorflow as tf
import tensorflow_quantum as tfq

from matplotlib import pyplot as plt
from cirq.contrib.svg import SVGCircuit
```

### Place a qubit on the grid

You will then place a qubit on thee grid

```python
qubit = cirq.GridQubit(0, 0)
```

## Prepare quantum data

The first thing you would do is set up the labels and parameters for preparation of the quantum data. For simplicity
here we have included just 2 data points `a` and `b`.

```python
expected_labels = np.array([[1, 0], [0, 1]])
```

Randonly rotate `x` and `z` axis

```python
angle = np.random.uniform(0, 2 * np.pi)
```

## Building the quantum Circuit

You will now build the quantum circuit and also convert it into a tensor

```python
a = cirq.Circuit(cirq.ry(angle)(qubit))
b = cirq.Circuit(cirq.ry(angle + np.pi / 2)(qubit))
quantum_data = tfq.convert_to_tensor([a, b])
```

```python
SVGCircuit(a)
```

    findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.

```python
SVGCircuit(b)
```

## Building the hybrid model

This section also shows the interoperatability between TensorFlow and Cirq. With the TFQ PQC layer you can easily
embed your quantum part of the model within a standard classical Keras model.

```python
q_data_input = tf.keras.Input(shape = (), dtype = tf.dtypes.string)
theta = sympy.Symbol("theta")
q_model = cirq.Circuit(cirq.ry(theta)(qubit))
```

```python
expectation = tfq.layers.PQC(q_model, cirq.Z(qubit))
expectation_output = expectation(q_data_input)
```

```python
classifier = tf.keras.layers.Dense(2, activation = tf.keras.activations.softmax)
classifier_output = classifier(expectation_output)
```

You will now define the optimizer and loss functions for your model

```python
model = tf.keras.Model(inputs = q_data_input,
                       outputs = classifier_output)
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = 0.1),
              loss = tf.keras.losses.CategoricalCrossentropy())
```

## Traaining the model

Training the model is just like training any other Keras model and is made easy.

```python
history = model.fit(x = quantum_data,
                    y = expected_labels,
                    epochs = 250)
```

    Train on 2 samples
    Epoch 1/250
    2/2 [==============================] - 2s 1s/sample - loss: 0.5722
    Epoch 2/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.5098
    Epoch 3/250
    2/2 [==============================] - 0s 2ms/sample - loss: 0.4543
    Epoch 4/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.4016
    Epoch 5/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.3534
    Epoch 6/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.3114
    Epoch 7/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.2756
    Epoch 8/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.2450
    Epoch 9/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.2185
    Epoch 10/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.1952
    Epoch 11/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.1745
    Epoch 12/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.1560
    Epoch 13/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.1395
    Epoch 14/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.1248
    Epoch 15/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.1118
    Epoch 16/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.1004
    Epoch 17/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0904
    Epoch 18/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0817
    Epoch 19/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0741
    Epoch 20/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0675
    Epoch 21/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0617
    Epoch 22/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0566
    Epoch 23/250
    2/2 [==============================] - 0s 2ms/sample - loss: 0.0522
    Epoch 24/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0483
    Epoch 25/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0449
    Epoch 26/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0418
    Epoch 27/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0391
    Epoch 28/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0367
    Epoch 29/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0345
    Epoch 30/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0325
    Epoch 31/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0308
    Epoch 32/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0292
    Epoch 33/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0277
    Epoch 34/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0264
    Epoch 35/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0252
    Epoch 36/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0241
    Epoch 37/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0231
    Epoch 38/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0221
    Epoch 39/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0213
    Epoch 40/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0205
    Epoch 41/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0197
    Epoch 42/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0191
    Epoch 43/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0184
    Epoch 44/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0178
    Epoch 45/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0173
    Epoch 46/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0168
    Epoch 47/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0163
    Epoch 48/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0158
    Epoch 49/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0154
    Epoch 50/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0150
    Epoch 51/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0146
    Epoch 52/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0143
    Epoch 53/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0140
    Epoch 54/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0136
    Epoch 55/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0133
    Epoch 56/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0130
    Epoch 57/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0128
    Epoch 58/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0125
    Epoch 59/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0123
    Epoch 60/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0120
    Epoch 61/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0118
    Epoch 62/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0116
    Epoch 63/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0114
    Epoch 64/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0112
    Epoch 65/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0110
    Epoch 66/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0108
    Epoch 67/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0106
    Epoch 68/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0104
    Epoch 69/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0103
    Epoch 70/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0101
    Epoch 71/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0100
    Epoch 72/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0098
    Epoch 73/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0097
    Epoch 74/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0095
    Epoch 75/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0094
    Epoch 76/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0092
    Epoch 77/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0091
    Epoch 78/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0090
    Epoch 79/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0089
    Epoch 80/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0087
    Epoch 81/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0086
    Epoch 82/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0085
    Epoch 83/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0084
    Epoch 84/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0083
    Epoch 85/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0082
    Epoch 86/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0081
    Epoch 87/250
    2/2 [==============================] - 0s 6ms/sample - loss: 0.0080
    Epoch 88/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0079
    Epoch 89/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0078
    Epoch 90/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0077
    Epoch 91/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0076
    Epoch 92/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0075
    Epoch 93/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0074
    Epoch 94/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0073
    Epoch 95/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0072
    Epoch 96/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0072
    Epoch 97/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0071
    Epoch 98/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0070
    Epoch 99/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0069
    Epoch 100/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0068
    Epoch 101/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0068
    Epoch 102/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0067
    Epoch 103/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0066
    Epoch 104/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0065
    Epoch 105/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0065
    Epoch 106/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0064
    Epoch 107/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0063
    Epoch 108/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0063
    Epoch 109/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0062
    Epoch 110/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0061
    Epoch 111/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0061
    Epoch 112/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0060
    Epoch 113/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0059
    Epoch 114/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0059
    Epoch 115/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0058
    Epoch 116/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0058
    Epoch 117/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0057
    Epoch 118/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0057
    Epoch 119/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0056
    Epoch 120/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0055
    Epoch 121/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0055
    Epoch 122/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0054
    Epoch 123/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0054
    Epoch 124/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0053
    Epoch 125/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0053
    Epoch 126/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0052
    Epoch 127/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0052
    Epoch 128/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0051
    Epoch 129/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0051
    Epoch 130/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0050
    Epoch 131/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0050
    Epoch 132/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0049
    Epoch 133/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0049
    Epoch 134/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0049
    Epoch 135/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0048
    Epoch 136/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0048
    Epoch 137/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0047
    Epoch 138/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0047
    Epoch 139/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0046
    Epoch 140/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0046
    Epoch 141/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0046
    Epoch 142/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0045
    Epoch 143/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0045
    Epoch 144/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0045
    Epoch 145/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0044
    Epoch 146/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0044
    Epoch 147/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0043
    Epoch 148/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0043
    Epoch 149/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0043
    Epoch 150/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0042
    Epoch 151/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0042
    Epoch 152/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0042
    Epoch 153/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0041
    Epoch 154/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0041
    Epoch 155/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0041
    Epoch 156/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0040
    Epoch 157/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0040
    Epoch 158/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0040
    Epoch 159/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0039
    Epoch 160/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0039
    Epoch 161/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0039
    Epoch 162/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0038
    Epoch 163/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0038
    Epoch 164/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0038
    Epoch 165/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0038
    Epoch 166/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0037
    Epoch 167/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0037
    Epoch 168/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0037
    Epoch 169/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0036
    Epoch 170/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0036
    Epoch 171/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0036
    Epoch 172/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0036
    Epoch 173/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0035
    Epoch 174/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0035
    Epoch 175/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0035
    Epoch 176/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0035
    Epoch 177/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0034
    Epoch 178/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0034
    Epoch 179/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0034
    Epoch 180/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0034
    Epoch 181/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0033
    Epoch 182/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0033
    Epoch 183/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0033
    Epoch 184/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0033
    Epoch 185/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0032
    Epoch 186/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0032
    Epoch 187/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0032
    Epoch 188/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0032
    Epoch 189/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0032
    Epoch 190/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0031
    Epoch 191/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0031
    Epoch 192/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0031
    Epoch 193/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0031
    Epoch 194/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0030
    Epoch 195/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0030
    Epoch 196/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0030
    Epoch 197/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0030
    Epoch 198/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0030
    Epoch 199/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0029
    Epoch 200/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0029
    Epoch 201/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0029
    Epoch 202/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0029
    Epoch 203/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0029
    Epoch 204/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0029
    Epoch 205/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0028
    Epoch 206/250
    2/2 [==============================] - 0s 5ms/sample - loss: 0.0028
    Epoch 207/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0028
    Epoch 208/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0028
    Epoch 209/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0028
    Epoch 210/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0027
    Epoch 211/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0027
    Epoch 212/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0027
    Epoch 213/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0027
    Epoch 214/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0027
    Epoch 215/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0027
    Epoch 216/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0026
    Epoch 217/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0026
    Epoch 218/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0026
    Epoch 219/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0026
    Epoch 220/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0026
    Epoch 221/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0026
    Epoch 222/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 223/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 224/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 225/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 226/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 227/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 228/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0025
    Epoch 229/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 230/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 231/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 232/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 233/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 234/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0024
    Epoch 235/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0024
    Epoch 236/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 237/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 238/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 239/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 240/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 241/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 242/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0023
    Epoch 243/250
    2/2 [==============================] - 0s 4ms/sample - loss: 0.0023
    Epoch 244/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 245/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 246/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 247/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 248/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 249/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022
    Epoch 250/250
    2/2 [==============================] - 0s 3ms/sample - loss: 0.0022

## Evaluating the model

```python
plt.plot(history.history['loss'])
plt.title('model loss')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train'], loc='upper left')
plt.show()
```

## Performing inference

```python
noise = np.random.uniform(-0.25, 0.25, 2)
test_data = tfq.convert_to_tensor([
    cirq.Circuit(cirq.ry(noise[0])(qubit)),
    cirq.Circuit(cirq.ry(noise[1] + np.pi/2)(qubit)),
])
```

You can see in the below cell that our model does a good job with this data though it was very easy.

```python
predictions = model.predict(test_data)
predictions
```

    array([[9.0111643e-01, 9.8883577e-02],
           [1.7436201e-04, 9.9982566e-01]], dtype=float32)

```python

```


# Model: Log Firing rates with ADVI

- spike counts are poisson distributed
- firing rates are log-Normally distributed
- firing rates are covariant across units
- noise is also covariant

That is, we want to make the model

$$\begin{align}
N_{tu} &\sim \mathrm{Poisson}(e^{\eta_{tu}}) \\
\eta_{t\cdot} &\sim \mathcal{N}(a_\cdot, S_\eta) \\
a_u &\sim \mathcal{N}(\mu_a, \Sigma_a) \\
S_\eta &\sim \text{Inv-Wishart}(\nu_\eta, \Psi_\eta)
\end{align}$$

```julia
using Distributions
using PyPlot
using ForwardDiff
using VinDsl
srand(12345);
```

    /Users/jmxp/anaconda/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
      warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')
    WARNING: Method definition logpdf(Distributions.Gamma, Real) in module Distributions at /Users/jmxp/.julia/v0.4/Distributions/src/univariates.jl:321 overwritten in module VinDsl at /Users/jmxp/.julia/v0.4/VinDsl/src/distributions/expfam.jl:42.
    WARNING: Method definition logpdf(Distributions.Poisson, Int64) in module Distributions at /Users/jmxp/.julia/v0.4/Distributions/src/univariates.jl:321 overwritten in module VinDsl at /Users/jmxp/.julia/v0.4/VinDsl/src/distributions/expfam.jl:64.

Set up some parameters

```julia
T = 1000  # number of time steps
U = 10  # units
```

    10

# Make firing rates

```julia
baseline = 10.
baseline_sd = 0.10
log_bl = log(baseline)
unit_bl = log_bl + baseline_sd * randn(U);
```

```julia
PyPlot.plt[:hist](unit_bl, 20);
```

```julia
fr_log = unit_bl' .+ zeros(T, U)  # firing rate

# make a tridiagonal covariance matrix
Ση = MvNormal(full(SymTridiagonal(ones(U), 0.2 * ones(U - 1))))

eta = fr_log + 0.1 * rand(Ση, T)'
fr = exp(eta);
```

```julia
PyPlot.plt[:hist](fr[:], 100);
```

# Make spikes

```julia
spikes = Array{Int}(size(fr)...)
for i in eachindex(fr)
    spikes[i] = rand(Poisson(fr[i]))
end
```

```julia
matshow(spikes', aspect="auto", cmap="gray");
```

# Fleshing out the model:

## Data:
- $N$

## Parameters:
- $\eta$
- $a$
- $S$

## Priors:
- $a_u \sim \mathcal{N}(\log(15), 0.1^2)$ (but we want posterior over $a$ to be MvNormal
- $S \sim \text{Inv-Wishart}(U, \mathbb{1}_{U \times U})$
@advi begin
    for u in 1:U
        a[u] ~ Normal(log(15.), 0.1)
    end
    S ~ InverseWishart(U, eye(U))
    for t in 1:T
        η[t] ~ MvNormal(a, S)
    end
    for t in 1:T
        for u in 1:U
            N[t, u] ~ Poisson(exp(η[t, u])
        end
    end
end

```julia
function ELBO(x::Vector)
    ctr = 1
    L = zero(eltype(x))

    @advi_declarations begin
        a::Real()[U]
        S::CovMat(U)
        η::RealVec(U)[T]
    end

    ########## E[log p]
    for u in 1:U
        L += logpdf(Normal(log(15.), 0.1), a[u])
    end
    L += logpdf(InverseWishart(U, eye(U)), S)
    for t in 1:T
        for u in 1:U
            L += logpdf(Poisson(exp(η[t][u])), spikes[t, u])
        end
    end

    L
end
```

    ELBO (generic function with 1 method)

```julia
npars = U * VinDsl.num_pars_advi(RReal()) + VinDsl.num_pars_advi(RCovMat(U)) +
T * VinDsl.num_pars_advi(RRealVec(U))
xx = 0.1 * randn(npars)
ELBO(xx)
```

    -172796.1518362259

```julia
∇L = (storage, x) -> ForwardDiff.gradient!(storage, ELBO, x)
```

    (anonymous function)

```julia
stor = similar(xx)
∇L(stor, xx)
```

    20130-element Array{Float64,1}:
     302.684
     -39.0358
     221.276
     101.422
     284.617
     -40.9675
     156.693
     187.883
     261.503
      59.0891
      83.1857
     165.997
     319.943
       ⋮
      10.9585
       7.64507
       5.96391
       2.86289
      -1.81848
      -3.33408
      -1.97074
      -0.327695
      -6.20405
      14.7903
       0.700619
      -2.32479

```julia
stor = similar(xx)
yy = copy(xx)
@time @profile ∇L(stor, yy)
#Profile.print()
```

     50.368151 seconds (182.11 M allocations: 27.367 GB, 5.55% gc time)

    20130-element Array{Float64,1}:
      250.737
       97.9996
      204.179
      128.533
      178.075
      164.28
      139.292
      191.312
      390.156
     -414.037
      482.816
     -968.425
      196.844
        ⋮
       10.6792
        7.65086
      -16.0406
      -16.5243
       12.1565
        5.6099
        5.92308
       -0.42114
       -0.418602
       -7.63236
       -8.26516
        2.40584

```julia
stor = similar(xx)
yy = copy(xx)
@time ∇L(stor, yy)
#Profile.print()
```

     48.528979 seconds (211.82 M allocations: 35.263 GB, 6.61% gc time)

    20130-element Array{Float64,1}:
      112.689
      148.616
      214.532
      106.602
      356.776
     -319.006
      330.402
     -241.958
       94.8451
      161.058
      287.778
      -77.8113
      228.628
        ⋮
        6.17156
        7.90104
        4.48699
       19.2151
       10.3063
        4.47824
       -6.23559
        3.31081
      -18.533
       -3.91601
      -13.1217
        0.916936

```julia
Profile.print(format=:flat, sortedby=:count)
```

     Count File                       Function                                 Line
         1 ./array.jl                 fill!                                     197
         1 ./essentials.jl            append_any                                128
         1 ./essentials.jl            append_any                                133
         1 ./essentials.jl            append_any                                136
         1 ./grisu.jl                 grisu                                      46
         1 ./grisu/fastfixed.jl       fastfixedtoa                              233
         1 ./grisu/fastfixed.jl       filldigits32                               44
         1 ./hashing.jl               hash_64_64                                 18
         1 ./hashing.jl               hash_64_64                                 19
         1 ./hashing.jl               hash_64_64                                 21
         1 ./range.jl                 steprange_last                             54
         1 ./unicode/utf8.jl          string                                    166
         1 ./unicode/utf8.jl          string                                    171
         1 ...atrix/inversewishart.jl _logpdf                                    71
         1 ...s/src/matrixvariates.jl logpdf                                     14
         1 ...ions/src/univariates.jl logpdf                                    321
         1 ...orwardDiff/src/cache.jl seed!                                     137
         1 ...ForwardDiff/src/dual.jl zero                                       90
         1 ...ardDiff/src/gradient.jl chunk_mode_gradient!                      133
         1 ...0.4/PDMats/src/pdmat.jl call                                       10
         1 ...0.4/PDMats/src/pdmat.jl call                                       13
         1 ...Funs/src/distrs/norm.jl zval                                        4
         1 ...4/StatsFuns/src/misc.jl logmvgamma                                 11
         1 ...ibutions/constraints.jl num_pars_advi                             123
         1 ...l/src/inference/advi.jl advi_initialize!                           95
         1 ...l/src/inference/advi.jl advi_initialize!                          105
         1 ...l/src/inference/advi.jl advi_rand                                  23
         1 ...l/src/inference/advi.jl advi_rand                                  28
         1 ...l/src/inference/advi.jl advi_variable                             126
         1 ...0.4/VinDsl/src/utils.jl call                                       82
         1 array.jl                   copy!                                      91
         1 array.jl                   fill!                                     193
         1 array.jl                   fill!                                     197
         1 broadcast.jl               _F_                                        88
         1 broadcast.jl               longer_size                                24
         1 cartesian.jl               _F_                                        31
         1 dSFMT.jl                   dsfmt_fill_array_close1_open2!             65
         1 dict.jl                    ht_keyindex2                              591
         1 dict.jl                    ht_keyindex2                              599
         1 dict.jl                    ht_keyindex2                              606
         1 linalg/bidiag.jl           A_ldiv_B!                                 217
         1 linalg/bidiag.jl           Ac_ldiv_B!                                233
         1 linalg/cholesky.jl         A_ldiv_B!                                 194
         1 linalg/cholesky.jl         logdet                                    244
         1 linalg/dense.jl            tril!                                      90
         1 linalg/generic.jl          vecdot                                    255
         1 linalg/triangular.jl       A_mul_Bc!                                 737
         1 linalg/triangular.jl       A_mul_Bc!                                 738
         1 linalg/triangular.jl       A_mul_Bc!                                 741
         1 linalg/triangular.jl       convert                                    59
         1 linalg/triangular.jl       naivesub!                                 770
         1 linalg/triangular.jl       naivesub!                                 771
         1 linalg/triangular.jl       naivesub!                                 774
         1 linalg/triangular.jl       naivesub!                                 803
         1 linalg/triangular.jl       setindex!                                 144
         1 printf.jl                  fix_dec                                   930
         1 reduce.jl                  _mapreduce                                140
         1 reduce.jl                  _mapreduce                                141
         1 reduce.jl                  _mapreduce                                142
         1 reduce.jl                  _mapreduce                                153
         1 reduce.jl                  mapreduce_pairwise_impl                   108
         1 reduce.jl                  mapreduce_seq_impl                        230
         1 util.jl                    anonymous                                 158
         1 util.jl                    time_print                                112
         2 ./array.jl                 fill!                                     196
         2 ./essentials.jl            append_any                                126
         2 ./essentials.jl            append_any                                127
         2 ./range.jl                 colon                                      82
         2 ./range.jl                 start                                     346
         2 ...atrix/inversewishart.jl call                                       17
         2 ...orwardDiff/src/cache.jl seed!                                     127
         2 ...ForwardDiff/src/dual.jl convert                                   155
         2 ...ardDiff/src/gradient.jl chunk_mode_gradient!                      135
         2 ...l/src/inference/advi.jl advi_initialize!                          107
         2 In[13]                     ELBO                                        2
         2 array.jl                   copy!                                      90
         2 array.jl                   copy!                                      94
         2 array.jl                   eye                                       218
         2 array.jl                   eye                                       226
         2 array.jl                   getindex                                  309
         2 broadcast.jl               broadcast!                                239
         2 dict.jl                    ht_keyindex2                              594
         2 linalg/bidiag.jl           Ac_ldiv_B!                                235
         2 linalg/cholesky.jl         chol!                                      41
         2 linalg/triangular.jl       convert                                    58
         2 linalg/triangular.jl       naivesub!                                 778
         2 random.jl                  randn_unlikely                           1122
         2 random.jl                  randn_unlikely                           1129
         3 ./essentials.jl            append_any                                137
         3 ./expr.jl                  symbol                                     10
         3 ./hashing.jl               hash_64_64                                 16
         3 ./promotion.jl             ==                                        225
         3 ...l/src/inference/advi.jl advi_rand                                  31
         3 array.jl                   unsafe_copy!                               86
         3 array.jl                   unsafe_getindex                           290
         3 broadcast.jl               broadcast!                                247
         3 broadcast.jl               broadcast_shape                            34
         3 linalg/cholesky.jl         getindex                                  168
         3 linalg/dense.jl            tril!                                      98
         3 operators.jl               Ac_rdiv_Bc                                173
         3 reduce.jl                  _mapreduce                                151
         4 ...ForwardDiff/src/dual.jl +                                         172
         4 ...ibutions/constraints.jl constrain                                  97
         4 ...l/src/inference/advi.jl advi_variable                             122
         4 abstractarray.jl           copy!                                     310
         4 array.jl                   copy                                      100
         4 broadcast.jl               check_broadcast_shape                      51
         4 cartesian.jl               _F_                                        34
         4 dict.jl                    broadcast!                                717
         4 dict.jl                    ht_keyindex2                              609
         4 essentials.jl              call                                      190
         4 linalg/triangular.jl       convert                                    60
         5 ...l/src/inference/advi.jl advi_variable                             132
         5 array.jl                   unsafe_getindex                           295
         5 linalg/triangular.jl       getindex                                  118
         5 linalg/triangular.jl       naivesub!                                 811
         5 random.jl                  randn_unlikely                           1120
         5 reduce.jl                  _mapreduce                                137
         5 tuple.jl                   getindex                                    8
         6 ./range.jl                 next                                      347
         6 ...l/src/inference/advi.jl advi_variable                             141
         6 dict.jl                    broadcast!                                715
         6 linalg/triangular.jl       getindex                                  116
         6 operators.jl               A_mul_Bc                                  164
         7 broadcast.jl               _F_                                        96
         7 linalg/triangular.jl       naivesub!                                 772
         8 ./essentials.jl            append_any                                132
         8 abstractarray.jl           ndims                                      58
         8 array.jl                   unsafe_copy!                               80
         8 random.jl                  randn!                                   1134
         9 array.jl                   size                                       48
        10 ...ibutions/constraints.jl logdetjac                                 112
        10 ...l/src/inference/advi.jl advi_variable                             139
        10 In[13]                     ELBO                                       13
        10 linalg/cholesky.jl         chol!                                      48
        11 ...l/src/inference/advi.jl advi_variable                             136
        12 linalg/bidiag.jl           A_ldiv_B!                                 218
        12 random.jl                  randn_unlikely                           1126
        13 broadcast.jl               broadcast_shape                            32
        13 linalg/triangular.jl       naivesub!                                 805
        13 reduce.jl                  _mapreduce                                145
        15 ./range.jl                 done                                      348
        15 ...ardDiff/src/gradient.jl chunk_mode_gradient!                      141
        15 ...l/src/inference/advi.jl advi_variable                             121
        15 linalg/triangular.jl       A_mul_Bc!                                 739
        17 array.jl                   unsafe_getindex                           293
        19 ...ardDiff/src/gradient.jl chunk_mode_gradient!                      125
        19 linalg/triangular.jl       A_mul_Bc                                 1017
        20 arraymath.jl               transpose!                                318
        21 dSFMT.jl                   dsfmt_fill_array_close1_open2!             67
        23 linalg/triangular.jl       A_mul_Bc                                  980
        23 random.jl                  gen_rand                                   88
        24 ...atrix/inversewishart.jl _logpdf                                    69
        24 ...l/src/inference/advi.jl advi_rand                                  22
        24 dict.jl                    ht_keyindex2                              597
        25 reduce.jl                  _mapreduce                                144
        26 ...0.4/PDMats/src/pdmat.jl call                                       18
        28 ./bool.jl                  !                                          16
        28 dict.jl                    ht_keyindex2                              592
        29 ...ibutions/constraints.jl constrain                                  99
        29 ...l/src/inference/advi.jl H                                          77
        30 ...ate/discrete/poisson.jl call                                       24
        30 ...tributions/src/utils.jl call                                        6
        35 ...ibutions/constraints.jl constrain                                  95
        35 ...l/src/inference/advi.jl H                                          61
        35 dict.jl                    ht_keyindex2                              589
        40 ./array.jl                 zeros                                     211
        40 ...l/src/inference/advi.jl advi_rand                                  25
        40 ...0.4/VinDsl/src/utils.jl call                                       72
        41 ...distributions/expfam.jl logpdf                                     64
        41 reduce.jl                  _mapreduce                                148
        45 ./linalg/lapack.jl         potrf!                                   2485
        45 linalg/cholesky.jl         chol!                                      27
        48 ...ForwardDiff/src/dual.jl +                                         170
        60 functors.jl                call                                       41
        62 broadcast.jl               longer_size                                25
        67 ./essentials.jl            append_any                                123
        69 ...l/src/inference/advi.jl advi_variable                             138
        69 broadcast.jl               broadcast_shape                            39
        70 ...atrix/inversewishart.jl call                                       23
        71 ...l/src/inference/advi.jl advi_initialize!                          101
        72 broadcast.jl               broadcast_shape                            40
        74 ...ForwardDiff/src/dual.jl exp                                       300
        74 reduce.jl                  r_promote                                  23
        76 reduce.jl                  _mapreduce                                143
        84 reduce.jl                  _mapreduce                                149
        94 dict.jl                    ht_keyindex2                              608
        96 array.jl                   getindex                                  283
        98 broadcast.jl               _F_                                        97
       101 ...l/src/inference/advi.jl ELBO                                      166
       106 ...l/src/inference/advi.jl advi_rand                                  29
       109 broadcast.jl               broadcast_shape                            33
       118 broadcast.jl               check_broadcast_shape                      57
       124 broadcast.jl               check_broadcast_shape                      58
       134 random.jl                  randn!                                   1135
       136 broadcast.jl               broadcast_shape                            37
       141 broadcast.jl               check_broadcast_shape                      52
       142 broadcast.jl               broadcast_shape                            31
       142 broadcast.jl               broadcast_shape                            41
       146 broadcast.jl               broadcast_shape                            38
       147 In[13]                     ELBO                                       16
       152 broadcast.jl               broadcast_shape                            36
       156 ...ForwardDiff/src/dual.jl exp                                       299
       163 ...l/src/inference/advi.jl H                                          80
       167 linalg/bidiag.jl           Ac_ldiv_B!                                234
       183 broadcast.jl               check_broadcast_shape                      56
       183 linalg/cholesky.jl         A_ldiv_B!                                 197
       190 ...atrix/inversewishart.jl _logpdf                                    72
       190 linalg/factorization.jl    \                                          25
       194 linalg/cholesky.jl         cholfact                                  126
       197 ...ate/discrete/poisson.jl call                                       27
       198 dict.jl                    broadcast!                                709
       205 ./essentials.jl            call                                      200
       215 ...s/src/matrixvariates.jl logpdf                                     16
       216 ...distributions/expfam.jl logpdf                                    115
       307 In[13]                     ELBO                                       15
       410 ...l/src/inference/advi.jl H                                          78
       415 array.jl                   getindex                                  282
       557 broadcast.jl               check_broadcast_shape                      55
       571 operators.jl               exp                                       380
       678 ...distributions/expfam.jl logpdf                                     65
       685 broadcast.jl               broadcast_shape                            35
       760 broadcast.jl               broadcast_shape                            46
      1147 broadcast.jl               _F_                                        89
      1244 ...l/src/inference/advi.jl advi_initialize!                           99
      1276 array.jl                   unsafe_getindex                           291
      1319 array.jl                   getindex                                  288
      1330 broadcast.jl               broadcast!                                246
      1367 In[13]                     ELBO                                       17
      5288 ...l/src/inference/advi.jl advi_rand                                  26
      5981 ...l/src/inference/advi.jl advi_initialize!                          103
      7302 ...l/src/inference/advi.jl advi_variable                             124
      7329 ...l/src/inference/advi.jl ELBO                                      158
     25581 In[13]                     ELBO                                       18
     34841 ...ardDiff/src/gradient.jl chunk_mode_gradient!                      134
     34878 ...ardDiff/src/gradient.jl gradient!                                  30
     34878 ...ardDiff/src/gradient.jl gradient!                                  35
     34878 In[15]                     anonymous                                   1
     34878 profile.jl                 anonymous                                  16
     34879 ....4/IJulia/src/IJulia.jl eventloop                                 143
     34879 .../src/execute_request.jl execute_request_0x535c5df2                183
     34879 loading.jl                 include_string                            282
     34879 task.jl                    anonymous                                 447

```julia
yy = copy(xx)
gg = similar(yy)
avg_sq_grad = similar(gg)
firstpass = true
elbo = Float64[]
```

```julia
step_size = 0.1
decay = 0.9
eps = 1e-8
nsteps = 200

for jj in 1:nsteps
    ∇L(gg, yy)
    if firstpass == false
        avg_sq_grad = avg_sq_grad * decay + gg.^2 * (1 - decay)
    else
        avg_sq_grad = gg.^2
        firstpass = false
    end
    yy += step_size * gg ./(sqrt(avg_sq_grad) + eps)
    push!(elbo, ELBO(yy))
    display("Iteration $(length(elbo)) $(elbo[end])")
end
```

```julia
plot(elbo)
ylim(-170000, 0)
```

```julia
unit_bl
```

    10-element Array{Float64,1}:
     2.41982
     2.38786
     2.34414
     2.35423
     2.37116
     2.33081
     2.15488
     2.31526
     2.44923
     2.28677

```julia
yy[1:2:20]
```

    10-element Array{Float64,1}:
     2.91538
     2.67399
     2.72302
     2.67405
     2.73146
     2.68792
     2.74393
     2.76845
     2.73165
     2.76666

```julia
exp(yy[2:2:20])
```

    10-element Array{Float64,1}:
     0.2226
     0.168821
     0.188767
     0.133385
     0.213639
     0.145906
     0.165139
     0.161354
     0.140502
     0.170247

```julia
ex = :(begin
a::RealVec(K)[P, Q]
b::Real()[5, H]
c::Foo(5)
d::Bar
end
)
```

    quote  # In[13], line 2:
        a::(RealVec(K))[P,Q] # In[13], line 3:
        b::(Real())[5,H] # In[13], line 4:
        c::Foo(5) # In[13], line 5:
        d::Bar
    end

```julia
xdump(ex)
```

    Expr
      head: Symbol block
      args: Array(Any,(8,))
        1: LineNumberNode
          file: Symbol In[13]
          line: Int64 2
        2: Expr
          head: Symbol ::
          args: Array(Any,(2,))
            1: Symbol a
            2: Expr
              head: Symbol ref
              args: Array(Any,(3,))
              typ: Any::DataType  <: Any
          typ: Any::DataType  <: Any
        3: LineNumberNode
          file: Symbol In[13]
          line: Int64 3
        4: Expr
          head: Symbol ::
          args: Array(Any,(2,))
            1: Symbol b
            2: Expr
              head: Symbol ref
              args: Array(Any,(3,))
              typ: Any::DataType  <: Any
          typ: Any::DataType  <: Any
        5: LineNumberNode
          file: Symbol In[13]
          line: Int64 4
        6: Expr
          head: Symbol ::
          args: Array(Any,(2,))
            1: Symbol c
            2: Expr
              head: Symbol call
              args: Array(Any,(2,))
              typ: Any::DataType  <: Any
          typ: Any::DataType  <: Any
        7: LineNumberNode
          file: Symbol In[13]
          line: Int64 5
        8: Expr
          head: Symbol ::
          args: Array(Any,(2,))
            1: Symbol d
            2: Symbol Bar
          typ: Any::DataType  <: Any
      typ: Any::DataType  <: Any

```julia
ex.args[2].args[2].args[1].args[1]
```

    :RealVec

```julia
ex.args[6].args[2]
```

    :(Foo(5))

```julia
function _declarations(ex::Expr, vars)
    if ex.head == :(::)
        vname = ex.args[1]
        ζ_name = Symbol("ζ_", vname)
        T_name = Symbol("T_", vname)
        typearg = ex.args[2]
        if isa(typearg, Expr)
            if typearg.head == :ref   # array of variables
                T = Symbol("R", typearg.args[1])
                dims = typearg.args[2:end]
                d_expr = :(
                begin
                    ΔL, ctr, $vname = VinDsl.advi_variable(x, ctr, $T, Tuple($(dims...)))
                    L += ΔL
                end
                    )
                else  # single variable
                T = Symbol("R", typearg)
                d_expr = :(
                begin
                    ΔL, ctr, $vname = VinDsl.advi_variable(x, ctr, $T)
                    L += ΔL
                end
                    )
            end
            push!(vars, d_expr)
        end

    else
        for a in filter(x -> isa(x, Expr), ex.args)
            _declarations(a, vars)
        end
    end
    vars
end

_declarations(ex::Expr) = _declarations(ex, [])
```

    _declarations (generic function with 2 methods)

```julia
_declarations(ex)
```

    3-element Array{Any,1}:
     quote  # In[40], line 13:
        (ΔL,ctr,a) = VinDsl.advi_variable(x,ctr,RRealVec(K),Tuple(P,Q)) # In[40], line 14:
        L += ΔL
    end
     quote  # In[40], line 13:
        (ΔL,ctr,b) = VinDsl.advi_variable(x,ctr,RReal(),Tuple(5,H)) # In[40], line 14:
        L += ΔL
    end
     quote  # In[40], line 21:
        (ΔL,ctr,c) = VinDsl.advi_variable(x,ctr,RFoo(5)) # In[40], line 22:
        L += ΔL
    end

```julia

```