## Perturbed DG Experiments - Exponential Toy Game
#### On Duality Gap as a Measure for Monitoring GAN Training
---

This Notebook Contains the Code for the experiments and visualization pertaining perturbed DG over the exponential toy game.

---

### Imports

```python
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras import Model
from tensorflow.keras.layers import Dense, Flatten
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import LogNorm
from sympy import symbols, Matrix, Function, simplify, exp, hessian, solve, init_printing
from sympy import Symbol, solve, Derivative,pprint
from math import exp,pow
from sympy import *
import sympy
from sympy.solvers.solveset import nonlinsolve
```

### Function Definition

```python
def exp_minmax_loss_fxn(x,y):
    a = -0.01
    b = 0.3
    c = 0.5
    f = tf.math.exp(a*(x**2 + y**2))*((b*(x**2) + y)**2 + (c*(y**2) + x)**2)
    return f
```

### Function Visualization

```python
def show_exp_minmax_loss_fxn():

    fig = plt.figure(figsize=(10, 10))

    w1_min, w1_max, w1_step = -25.0, 25.0, 0.5
    w2_min, w2_max, w2_step = -25.0, 25.0, 0.5
    W1, W2 = np.meshgrid(np.arange(w1_min, w1_max+ w1_step, w1_step), np.arange(w2_min, w2_max+ w2_step, w2_step))
    Z = exp_minmax_loss_fxn(W1, W2 )

    ax = plt.axes(projection='3d', elev=80, azim=-50)
    ax.set_xlim((w1_min, w1_max))
    ax.set_ylim((w2_min, w2_max))

    ax.plot_surface(W1, W2, Z, rstride=1, cstride=1, edgecolor='none', alpha=.7, cmap=plt.cm.jet)
    ax.set_xlabel('$W1$')
    ax.set_ylabel('$W2$')
    ax.set_zlabel('$Z$')
    plt.savefig('exp_minmax_fxn.png')
    plt.show()

show_exp_minmax_loss_fxn()
```

### Gradient and Hessian Computation

```python
init_printing()

a = -0.01
b = 0.3
c = 0.5

x1, x2 = symbols('x1 x2')
f, g, h = symbols('f g h', cls=Function)

X = Matrix([x1,x2])
f = Matrix([exp(-0.01*(x1**2 + x2**2))*((0.3*(x1**2)+x2)**2 + (0.5*(x2**2)+x1)**2)])

```

```python

gradf = simplify(f.jacobian(X))
gradf
```

$\displaystyle \left[\begin{matrix}\left(1.2 x_{1} \left(0.3 x_{1}^{2} + x_{2}\right) - 0.02 x_{1} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) + 2 x_{1} + 1.0 x_{2}^{2}\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}} & \left(0.6 x_{1}^{2} + 2.0 x_{2} \left(x_{1} + 0.5 x_{2}^{2}\right) - 0.02 x_{2} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) + 2 x_{2}\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}}\end{matrix}\right]$

```python

hessianf = simplify(hessian(f, X))
hessianf
```

$\displaystyle \left[\begin{matrix}\left(0.0004 x_{1}^{2} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) + 1.08 x_{1}^{2} - 0.04 x_{1} \left(1.2 x_{1} \left(0.3 x_{1}^{2} + x_{2}\right) + 2 x_{1} + 1.0 x_{2}^{2}\right) + 1.2 x_{2} - 0.02 \left(x_{1} + 0.5 x_{2}^{2}\right)^{2} - 0.02 \left(0.3 x_{1}^{2} + x_{2}\right)^{2} + 2\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}} & \left(0.0004 x_{1} x_{2} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) - 0.02 x_{1} \left(0.6 x_{1}^{2} + 2.0 x_{2} \left(x_{1} + 0.5 x_{2}^{2}\right) + 2 x_{2}\right) + 1.2 x_{1} - 0.02 x_{2} \left(1.2 x_{1} \left(0.3 x_{1}^{2} + x_{2}\right) + 2 x_{1} + 1.0 x_{2}^{2}\right) + 2.0 x_{2}\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}}\\\left(0.0004 x_{1} x_{2} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) - 0.02 x_{1} \left(0.6 x_{1}^{2} + 2.0 x_{2} \left(x_{1} + 0.5 x_{2}^{2}\right) + 2 x_{2}\right) + 1.2 x_{1} - 0.02 x_{2} \left(1.2 x_{1} \left(0.3 x_{1}^{2} + x_{2}\right) + 2 x_{1} + 1.0 x_{2}^{2}\right) + 2.0 x_{2}\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}} & \left(2.0 x_{1} + 0.0004 x_{2}^{2} \left(\left(x_{1} + 0.5 x_{2}^{2}\right)^{2} + \left(0.3 x_{1}^{2} + x_{2}\right)^{2}\right) + 3.0 x_{2}^{2} - 0.04 x_{2} \left(0.6 x_{1}^{2} + 2.0 x_{2} \left(x_{1} + 0.5 x_{2}^{2}\right) + 2 x_{2}\right) - 0.02 \left(x_{1} + 0.5 x_{2}^{2}\right)^{2} - 0.02 \left(0.3 x_{1}^{2} + x_{2}\right)^{2} + 2\right) e^{- 0.01 x_{1}^{2} - 0.01 x_{2}^{2}}\end{matrix}\right]$

```python
def get_jacobian(val_x=0,val_y=0):
  x = Symbol('x')
  y = Symbol('y')
  f1 = -sympy.exp(a*(x**2 + y**2))*((b*(x**2) + y)**2 + (c*(y**2) + x)**2)
  f2 = sympy.exp(a*(x**2 + y**2))*((b*(x**2) + y)**2 + (c*(y**2) + x)**2)

  d1 = Derivative(f1,x).doit()
  d2 = Derivative(f2,y).doit()

  print( ' Gradients : ',d1.subs(x,val_x).subs(y,val_y),' \t ',d2.subs(x,val_x).subs(y,val_y))
  d11 = Derivative(d1,x).doit()
  d22 = Derivative(d2,y).doit()

  d12 = Derivative(d1,y).doit()
  d21 = Derivative(d2,x).doit()

  Jacobian = Matrix([[d11.subs(x,val_x).subs(y,val_y),d12.subs(x,val_x).subs(y,val_y)],[d21.subs(x,val_x).subs(y,val_y),d22.subs(x,val_x).subs(y,val_y)]])

  print( ' Jacobian : \n')
  pprint(Jacobian)
  eigenVals = Jacobian.eigenvals()

  expanded_eigenvals = [complex(key) for key in eigenVals.keys() for i in range(eigenVals[key]) ]
  print('\n\n EigenValues : \n\n {}'.format(expanded_eigenvals))
  return expanded_eigenvals

```

```python

init_points = [(-12.467547,-8.67366),(0.0,0.0)]

for x,y in init_points:
    print('--'*50 +'\n \t Init Point : X:{} Y:{} \n'.format(x,y)+'--'*50)
    eigenvals = get_jacobian(x,y)
    eigenvals = [np.array([complex(item) for item in eigenvals])]
```

    ----------------------------------------------------------------------------------------------------
     	 Init Point : X:-12.467547 Y:-8.67366
    ----------------------------------------------------------------------------------------------------
     Gradients :  0.0620581635165252  	  -0.0683054521266797
     Jacobian :

    ⎡1.11718768601266   12.1764191162642⎤
    ⎢                                   ⎥
    ⎣-12.1764191162642  9.82599396907192⎦

     EigenValues :

     [(5.47159082754229-11.371207313911825j), (5.47159082754229+11.371207313911825j)]
    ----------------------------------------------------------------------------------------------------
     	 Init Point : X:0.0 Y:0.0
    ----------------------------------------------------------------------------------------------------
     Gradients :  0  	  0
     Jacobian :

    ⎡-2  0⎤
    ⎢     ⎥
    ⎣0   2⎦

     EigenValues :

     [(-2+0j), (2+0j)]

```python

import tqdm
import matplotlib.pyplot as plt
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers,models
import time
import math
from math import ceil
from math import floor
from numpy import ones
from numpy import expand_dims
from numpy import log
from numpy import mean
from numpy import std
from numpy import exp
from numpy.random import shuffle
from numpy import asarray

EPOCHS = 15
alpha=0.95
decay=0.95
B=tf.Variable(0.0)
duality_gap_batch=tf.Variable(0.0)
acc_Y_cost=tf.Variable(0.0)
acc_X_cost=tf.Variable(0.0)
num_batches=1

class TAU(object):
    def __init__(self, x=-5.0,y=-10.0,optimizer='adam'):
        self.X = tf.Variable(x)
        self.Y = tf.Variable(y)

        print( self.X, self.Y)

        self.lr = 5e-3

        self.gradients_of_X=-1
        self.gradients_of_Y=-1

        self.Y_loss=-1
        self.X_loss=-1

        if(optimizer=='adam'):
            self.optimizer='adam'
            self.X_optimizer=tf.keras.optimizers.Adam(self.lr)
            self.Y_optimizer=tf.keras.optimizers.Adam(self.lr)
        elif(optimizer=='sgd'):
            self.optimizer='sgd'
            self.X_optimizer=tf.keras.optimizers.SGD(self.lr)
            self.Y_optimizer=tf.keras.optimizers.SGD(self.lr)

        self.update_X_list = []
        self.update_Y_list = []

    def show_contour(self,epoch=0,reward=0):
      fig, ax = plt.subplots(figsize=(10, 6))
      w1_min, w1_max, w1_step = -26.0, 26.0, 0.2
      w2_min, w2_max, w2_step = -26.0, 26.0, 0.2
      W1, W2 = np.meshgrid(np.arange(w1_min, w1_max+ w1_step, w1_step), np.arange(w2_min, w2_max+ w2_step, w2_step))
      Z = exp_minmax_loss_fxn(W1, W2 )
      cs = ax.contourf(W1, W2, Z, cmap=plt.cm.Greys ,alpha=0.6)

      ax.plot( self.update_X_list, self.update_Y_list, color='g',linewidth=2.0,label='Autoloss')
      ax.scatter( [-12,-11,12], [-9,7,-7], color='b',linewidth=2.0,label='Nash',marker='x')
      ax.scatter( [-.5], [-1], color='r',linewidth=2.0,label='Non Nash',marker='*')

      cbar = fig.colorbar(cs)
      leg = plt.legend()
      ax.set_xlabel('$X$')
      ax.set_ylabel('$Y$')
      plt.title('Iteration : {} Reward : {} DualityGAP :{}'.format(epoch,reward,duality_gap_batch.numpy()))
      plt.xticks(np.arange(w1_min, w1_max+5, 5.0))
      plt.yticks(np.arange(w2_min, w2_max+5, 5.0))
      ax.set_xlim((w1_min, w1_max))
      ax.set_ylim((w2_min, w2_max))
      plt.grid()
      plt.show()

    def calculate_duality_gap(self,random=False):

      stddev = 0.01
      self.update_X_list.append(self.X.numpy())
      self.update_Y_list.append(self.Y.numpy())

      self.update_X_list_dg = []
      self.update_Y_list_dg = []

      self.update_X_list_dg.append(self.X.numpy())
      self.update_Y_list_dg.append(self.Y.numpy())

      if(random==True):
        X = tf.Variable(self.X.numpy())
        Y = tf.Variable(self.Y.numpy()+ abs(tf.random.normal(mean=1.0, stddev=stddev,shape=self.Y.numpy().shape)))
      else:
        X = tf.Variable(self.X.numpy())
        Y = tf.Variable(self.Y.numpy())

      iterations = 500
      lr = 5e-4

      if(self.optimizer=='adam'):
            X_optimizer=tf.keras.optimizers.Adam(self.lr)
            Y_optimizer=tf.keras.optimizers.Adam(self.lr)
      elif(self.optimizer=='sgd'):
            X_optimizer=tf.keras.optimizers.SGD(self.lr)
            Y_optimizer=tf.keras.optimizers.SGD(self.lr)

      for iteration in range(iterations):
        with tf.GradientTape() as Y_tape:
          Y_tape.watch(Y)
          Y_loss = exp_minmax_loss_fxn(X,Y)
        gradients_of_Y = Y_tape.gradient(Y_loss, Y)
      #         Y = Y - lr*gradients_of_Y
        Y_optimizer.apply_gradients(zip([gradients_of_Y],[Y]))

        if(iteration%5==0):
            self.update_Y_list_dg.append(Y.numpy())
            self.update_X_list.append(X.numpy())

      fmin = exp_minmax_loss_fxn(X,Y).numpy()

      if(random==True):
        X = tf.Variable(self.X.numpy()+ abs(tf.random.normal(mean=0.0,stddev=stddev,shape=self.X.numpy().shape)))
        Y = tf.Variable(self.Y.numpy())
      else:
        X = tf.Variable(self.X.numpy())
        Y = tf.Variable(self.Y.numpy())

      for iteration in range(iterations):
        with tf.GradientTape() as X_tape:
          X_tape.watch(X)
          X_loss = -1*exp_minmax_loss_fxn(X,Y)
        gradients_of_X = X_tape.gradient(X_loss, X)
      #         X = X - lr*gradients_of_X
        X_optimizer.apply_gradients(zip([gradients_of_X],[X]))

        if(iteration%5==0):
          self.update_X_list_dg.append(X.numpy())
          self.update_Y_list.append(Y.numpy())

      fmax = exp_minmax_loss_fxn(X,Y).numpy()

      print('Duality Gap Random : ',random,' : ',fmax - fmin)
      return fmax - fmin

    def train_step_Y(self):
        with tf.GradientTape() as Y_tape:
            Y_tape.watch(self.Y)
            self.Y_loss = exp_minmax_loss_fxn(self.X,self.Y)

        self.gradients_of_Y = Y_tape.gradient(self.Y_loss,self.Y)
        self.Y_optimizer.apply_gradients(zip([self.gradients_of_Y],[self.Y]))

        return self.gradients_of_Y,self.Y_loss

    def train_step_X(self):
        with tf.GradientTape() as X_tape:
            X_tape.watch(self.X)
            self.X_loss = -1*exp_minmax_loss_fxn(self.X,self.Y)

        self.gradients_of_X = X_tape.gradient(self.X_loss, self.X)
        self.X_optimizer.apply_gradients(zip([self.gradients_of_X],[self.X]))

        return self.gradients_of_X,self.X_loss

    def train(self,action,epoch,i):

        if action[0]==1:
            self.gradients_of_Y,self.Y_loss=self.train_step_Y()

        else:
            self.gradients_of_X,self.X_loss=self.train_step_X()

        if(epoch%5==0):
            self.update_X_list.append(self.X.numpy())
            self.update_Y_list.append(self.Y.numpy())

class Controller():
      def __init__(self,epoch=500):
          self.epoch = epoch

      def compute_dg(self,x,y,opt):
          tau = TAU(x,y,opt)

          data = {}
          vanilla_dg = tau.calculate_duality_gap(random=False)
          updates = {
                'X':tau.update_X_list,
                'Y':tau.update_Y_list,
                'X_DG':tau.update_X_list_dg,
                'Y_DG':tau.update_Y_list_dg,
                'DG':vanilla_dg
            }
          data['vanilla']=updates

          tau = TAU(x,y,opt)
          random_dg = tau.calculate_duality_gap(random=True)
          updates = {
                'X':tau.update_X_list,
                'Y':tau.update_Y_list,
                'X_DG':tau.update_X_list_dg,
                'Y_DG':tau.update_Y_list_dg,
                'DG':random_dg
            }
          data['random']=updates

          print('Vanilla DG : {} \nLocal Random DG  : {}'.format(vanilla_dg,random_dg))
          print('Final Coordinates : ' ,tau.update_X_list_dg[-1],tau.update_Y_list_dg[-1])
          updates = {
                'X':tau.update_X_list,
                'Y':tau.update_Y_list,
                'X_DG':tau.update_X_list_dg,
                'Y_DG':tau.update_Y_list_dg,
                'vanilla_DG':vanilla_dg,
                'local_random_DG':random_dg
            }
          return data

      def train_on_ratio(self,x=-12.87,y=-7.31,nx=1,ny=1,k=1,opt='adam'):
            tau = TAU(x,y,opt)

            for epoch in tqdm.tqdm(range(k*self.epoch)):
              for i in range(nx):
                  tau.train([0,1],epoch,i)

              for i in range(ny):
                  tau.train([1,0],epoch,i)

            updates = {
                'X':tau.update_X_list,
                'Y':tau.update_Y_list,
                'DG':tau.calculate_duality_gap()
            }
            return updates

      def train_on_random_ratio(self,x=-5.1,y=-10.1,opt='adam'):
            tau = TAU(x,y,opt)
            max_iter = 20
            min_iter = 1

            nx = np.random.random_integers(min_iter,max_iter+1)
            ny = np.random.random_integers(min_iter,max_iter+1)

            for epoch in tqdm.tqdm(range(self.epoch)):
              for i in range(nx):
                  tau.train([0,1],epoch,i)

              for i in range(ny):
                  tau.train([1,0],epoch,i)

            updates = {
                'X' :tau.update_X_list,
                'Y' :tau.update_Y_list
            }
            return updates,nx+ny

controller=Controller(5000)
controller.train_on_ratio()
optimizers = ['adam']
final_dat = {}

init_points = [(0.0,0.0),( -12.467547,-8.67366)]

ntrials = 5

for opt in optimizers:
    print(opt)
    data={
        'single':{},
        'random':{}
    }
    for x,y in init_points:
        print(x,y)
        trials = []
        for i in range(ntrials):
          trials.append(controller.compute_dg(x,y,opt))
        final_dat['X{}Y{}'.format(x,y)] = trials

```

      0%|          | 0/5000 [00:00<?, ?it/s]

    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.87> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-7.31>

    100%|██████████| 5000/5000 [00:29<00:00, 172.40it/s]

    Duality Gap Random :  False  :  41.495758
    adam
    0.0 0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  False  :  0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  True  :  49.21852
    Vanilla DG : 0.0
    Local Random DG  : 49.21852111816406
    Final Coordinates :  4.5419307 0.017989157
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  False  :  0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  True  :  49.216076
    Vanilla DG : 0.0
    Local Random DG  : 49.2160758972168
    Final Coordinates :  4.541852 0.02073147
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  False  :  0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  True  :  49.220108
    Vanilla DG : 0.0
    Local Random DG  : 49.22010803222656
    Final Coordinates :  4.541986 0.020360613
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  False  :  0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  True  :  49.205074
    Vanilla DG : 0.0
    Local Random DG  : 49.205074310302734
    Final Coordinates :  4.5414867 0.019537272
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  False  :  0.0
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.0>
    Duality Gap Random :  True  :  49.216587
    Vanilla DG : 0.0
    Local Random DG  : 49.21658706665039
    Final Coordinates :  4.5418653 0.017021596
    -12.467547 -8.67366
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  False  :  0.001953125
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  True  :  0.001663208
    Vanilla DG : 0.001953125
    Local Random DG  : 0.0016632080078125
    Final Coordinates :  -12.519718 -8.659502
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  False  :  0.001953125
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  True  :  0.0016479492
    Vanilla DG : 0.001953125
    Local Random DG  : 0.00164794921875
    Final Coordinates :  -12.519718 -8.658344
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  False  :  0.001953125
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  True  :  0.0016784668
    Vanilla DG : 0.001953125
    Local Random DG  : 0.001678466796875
    Final Coordinates :  -12.519718 -8.658691
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  False  :  0.001953125
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  True  :  0.0015716553
    Vanilla DG : 0.001953125
    Local Random DG  : 0.0015716552734375
    Final Coordinates :  -12.519717 -8.65773
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  False  :  0.001953125
    <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-12.467547> <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-8.67366>
    Duality Gap Random :  True  :  0.001663208
    Vanilla DG : 0.001953125
    Local Random DG  : 0.0016632080078125
    Final Coordinates :  -12.519718 -8.659255

```python
from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes
from mpl_toolkits.axes_grid1.inset_locator import mark_inset
from matplotlib.patches import Patch
from matplotlib.lines import Line2D

def show_contour(data,init_points):

      fig, ax = plt.subplots(1,1,figsize=(12, 8))
      inp = -1
      w1_min, w1_max, w1_step = -20.0, 20.0, 0.2
      w2_min, w2_max, w2_step = -20.0, 20.0, 0.2
      W1, W2 = np.meshgrid(np.arange(w1_min, w1_max+ w1_step, w1_step), np.arange(w2_min, w2_max+ w2_step, w2_step))
      Z = exp_minmax_loss_fxn(W1, W2 )

      cmap_ = 'viridis'
      cs = ax.contourf(W1, W2, Z ,cmap=plt.get_cmap(cmap_),alpha=0.999)

      cbar = fig.colorbar(cs)
      plt.rc('xtick',labelsize=15)
      plt.rc('ytick',labelsize=15)
      legend_elements = [Line2D([], [], color='cyan', lw=3, label='Vanilla DG'),
                         Line2D([], [], color='darksalmon', lw=3, label='Perturbed DG'),
                         Line2D([], [], marker='o',color='k', label='Start Point',markeredgecolor='w', markerfacecolor=(0, 0, 0, 0.01), markersize=20,lw=0,mew=2),
                         Line2D([], [], marker='D',color='k', label='End Point',markeredgecolor='w', markerfacecolor=(0, 0, 0, 0.01), markersize=15,lw=0,mew=2),
                         Line2D([], [], marker='x', label='Nash Point (A)',markeredgecolor='lawngreen', markersize=15,mew=3,lw=0),
                         Line2D([], [], marker='*', label='Non Nash Point (B)',markerfacecolor='r',markeredgecolor='r', markersize=20,mew=3,lw=0)]

      plt.legend(handles=legend_elements,loc=4,fontsize=15, ncol=3,facecolor='k',framealpha=0.5,labelcolor='w')

      scale = 4.5
      axins = zoomed_inset_axes(ax, scale, loc=1)
      axins.contourf(W1, W2, Z ,cmap=plt.get_cmap(cmap_),alpha=0.95)

      axins1 = zoomed_inset_axes(ax,scale, loc=2)
      axins1.contourf(W1, W2, Z ,cmap=plt.get_cmap(cmap_),alpha=0.95)

      for x,y in init_points:

          inp = inp+1
#           print(x,y, data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'])
          start_marker='o'
          end_marker='D'

          color = 'cyan'

          ax.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'], final_dat['X{}Y{}'.format(x,y)][0]['vanilla']['Y'],color=color)
          ax.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][0], color=color,s=550.0,facecolors='none',marker=start_marker)
          ax.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][-1], color=color,s=180.0, facecolors='none',marker=end_marker)

          ax.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'], data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'],color=color)
          ax.scatter(data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][0], color=color,facecolors='none',s=550.0,marker=start_marker)
          ax.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][-1], color=color,facecolors='none',s=180.0,marker=end_marker)

          color = 'cyan'
          axins.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'], final_dat['X{}Y{}'.format(x,y)][0]['vanilla']['Y'],color=color,label='X Worst - Vanilla ')
          axins.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][0], color=color,s=scale*550.0,facecolors='none',marker=start_marker)
          axins.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][-1], color=color,s=scale*180.0, facecolors='none',marker=end_marker)

          axins.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'], data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'],color=color,label='Y Worst - Vanilla ')
          axins.scatter(data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][0], color=color,facecolors='none',s=scale*550.0,marker=start_marker)
          axins.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][-1], color=color,facecolors='none',s=scale*180.0,marker=end_marker)

          color = 'darksalmon'
          lwdth = 8
          for i in range(ntrials):
            axins.plot( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'], final_dat['X{}Y{}'.format(x,y)][i]['random']['Y'],color=color,alpha=0.1*(i+1))
            axins.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y'][0], color=color,facecolors='none',s=scale*900.0,marker=start_marker)
            axins.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y'][-1], color=color,facecolors='none',s=scale*300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          lwdth = 8
          for i in range(ntrials):
            axins.plot( data['X{}Y{}'.format(x,y)][i]['random']['X'], data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'],color=color,alpha=0.1*(i+1))
            axins.scatter(data['X{}Y{}'.format(x,y)][i]['random']['X'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][1], color=color,facecolors='none',s=scale*900.0,marker=start_marker)
            axins.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][-1], color=color,facecolors='none',s=scale*300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          axins.scatter( [0.0], [0.0], color='r',linewidth=3.0,label='Non Nash',marker='*',s=100)

          axins.set_ylim(-1, 1)
          axins.set_xlim(-0.75, 0.75)
          plt.yticks(visible=False)
          plt.xticks(visible=False)
          mark_inset(ax, axins, loc1=3, loc2=4, fc="none", ec="w",alpha=0.5,linestyle='--')

          color = 'cyan'
          axins1.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'], final_dat['X{}Y{}'.format(x,y)][0]['vanilla']['Y'],color=color)
          axins1.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][0], color=color,s=scale*550.0,facecolors='none',marker=start_marker,label='Start point')
          axins1.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X_DG'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y'][-1], color=color,s=scale*180.0, facecolors='none',marker=end_marker,label='End point')

          axins1.plot( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'], data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'],color=color)
          axins1.scatter(data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][0],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][0], color=color,facecolors='none',s=scale*550.0,marker=start_marker)
          axins1.scatter( data['X{}Y{}'.format(x,y)][0]['vanilla']['X'][-1],data['X{}Y{}'.format(x,y)][0]['vanilla']['Y_DG'][-1], color=color,facecolors='none',s=scale*180.0,marker=end_marker)

          color = 'darksalmon'
          lwdth = 8
          for i in range(ntrials):
            axins1.plot( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'], final_dat['X{}Y{}'.format(x,y)][i]['random']['Y'],color=color,alpha=0.1*(i+1))
            axins1.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y'][0], color=color,facecolors='none',s=scale*900.0,marker=start_marker)
            axins1.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y'][-1], color=color,facecolors='none',s=scale*300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          lwdth = 8
          for i in range(ntrials):
            axins1.plot( data['X{}Y{}'.format(x,y)][i]['random']['X'], data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'],color=color,alpha=0.1*(i+1))
            axins1.scatter(data['X{}Y{}'.format(x,y)][i]['random']['X'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][1], color=color,facecolors='none',s=scale*900.0,marker=start_marker)
            axins1.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][-1], color=color,facecolors='none',s=scale*300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          axins1.scatter( [0.0], [0.0], color='r',linewidth=3.0,marker='*',s=100)
          axins1.scatter( [-12.467547], [-8.67366],linewidth=3.0, color='lawngreen',marker='x',s=100)

          axins.set_xticks([])
          axins1.set_xticks([])
          axins.set_yticks([])
          axins1.set_yticks([])

          axins1.set_xlim(-13.2, -11.8)
          axins1.set_ylim(-9.6, -7.6)
          plt.yticks(visible=False)
          plt.xticks(visible=False)
          mark_inset(ax, axins1, loc1=3, loc2=4, fc="none", ec="w",alpha=0.5,linestyle='--')

          color = 'darksalmon'

          lwdth = 8
          for i in range(ntrials):
            ax.plot( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'], final_dat['X{}Y{}'.format(x,y)][i]['random']['Y'],color=color,alpha=0.5*(i+1))
            ax.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y'][0], color=color,facecolors='none',s=900.0,marker=start_marker)
            ax.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X_DG'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y'][-1], color=color,facecolors='none',s=300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          lwdth = 8
          for i in range(ntrials):
            ax.plot( data['X{}Y{}'.format(x,y)][i]['random']['X'], data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'],color=color,alpha=0.5*(i+1))
            ax.scatter(data['X{}Y{}'.format(x,y)][i]['random']['X'][0],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][1], color=color,facecolors='none',s=900.0,marker=start_marker)
            ax.scatter( data['X{}Y{}'.format(x,y)][i]['random']['X'][-1],data['X{}Y{}'.format(x,y)][i]['random']['Y_DG'][-1], color=color,facecolors='none',s=300.0,marker=end_marker)
            lwdth = lwdth - 1.5

          if(inp==0):
            ax.scatter(4.5, 0, marker='>',color=color)

          ax.scatter( [0.0], [0.0], color='r',linewidth=3.0,marker='*',s=100)
          ax.scatter( [-12.467547], [-8.67366],linewidth=3.0, color='lawngreen',marker='x',s=100)
          ax.set_xlabel('$x$',fontsize=20)
          ax.set_ylabel('$y$',fontsize=20)
          ax.set_xlim((w1_min, w1_max))
          ax.set_ylim((w2_min, w2_max))
          plt.grid()

      plt.savefig('./comparison_vanilla_random.png')
      plt.show()

```

```python
import seaborn as sns
sns.set_style("ticks", {"xtick.major.size": 1, "ytick.major.size": 1})
sns.set_style('whitegrid')
show_contour(final_dat,init_points)
plt.show
```

```python

```

```python

```

```python

```


```
import numpy as np
import matplotlib.pyplot as plt
import scipy.sparse as sps
```

# Dataset, binary data and continuous data

```
def digit_basis(geometry):
    num_bit = np.prod(geometry)
    M = 2**num_bit
    x = np.arange(M)
    return x

def binary_basis(geometry):
    num_bit = np.prod(geometry)
    M = 2**num_bit
    x = np.arange(M)
    return unpacknbits(x[:,None], num_bit).reshape((-1,)+geometry)

def unpacknbits(arr, nbit, axis=-1):
    '''unpack numbers to bitstrings.'''
    nd = np.ndim(arr)
    if axis < 0:
        axis = nd + axis
    return (((arr & (1 << np.arange(nbit - 1, -1, -1)).reshape([-1] + [1] * (nd - axis - 1)))) > 0).astype('int8')

def packnbits(arr, axis=-1):
    '''pack bitstrings to numbers.'''
    nd = np.ndim(arr)
    nbit = np.shape(arr)[axis]
    if axis < 0:
        axis = nd + axis
    return (arr * (1 << np.arange(nbit - 1, -1, -1)).reshape([-1] + [1] * (nd - axis - 1))\
           ).sum(axis=axis, keepdims=True).astype('int')
```

```
def gaussian_pdf(geometry, mu, sigma):
    '''get gaussian distribution function'''
    x = digit_basis(geometry)
    pl = 1. / np.sqrt(2 * np.pi * sigma**2) * \
        np.exp(-(x - mu)**2 / (2. * sigma**2))
    return pl/pl.sum()
```

```
def barstripe_pdf(geometry):
    '''get bar and stripes PDF'''
    x = binary_basis(geometry)
    pl = is_bs(x)
    return pl/pl.sum()

def is_bs(samples):
    '''a sample is a bar or a stripe.'''
    return (np.abs(np.diff(samples,axis=-1)).sum(axis=(1,2))==0\
           )|((np.abs(np.diff(samples, axis=1)).sum(axis=(1,2)))==0)
```

```
num_bit = 9
pl1 = gaussian_pdf((num_bit,), mu=2**(num_bit-1)-0.5, sigma=2**(num_bit-2))
plt.plot(pl1)
plt.show()
```

```
geometry = (3,3)
pl2 = barstripe_pdf(geometry)
plt.plot(pl2)

# show ticks
basis = binary_basis(geometry)
plt.xticks([0, 511], [basis[0], basis[511]])
plt.show()

# generate samples
samples = basis[pl2>1e-2]

# show bar and stripes
def plot_bs(samples, size):
    plt.figure(facecolor='#777777')
    gs = plt.GridSpec(*size)

    for i in range(size[0]):
        for j in range(size[1]):
            if i*size[1]+j == len(samples): break
            plt.subplot(gs[i,j]).imshow(samples[i*size[1]+j], vmin=0, vmax=1)
            plt.axis('equal')
            plt.axis('off')
    plt.show()

size = (4, 4)
plot_bs(samples, size)
```

# Kernel Two Sample Test
## Kernel Method
Given a vector $x$ in $n$-dimensional space, we perform the mapping to reproducing kernel Hilbert space (RKHS) $x\rightarrow \phi(x)$.
Define the inner product as $\phi(x)^T\phi = K(x,x')$, this is the kernel function.
Good let's see a simple example of polynomial kernel function $K(x,y)=(x^T y)^2=\sum_{ij} x_i y_i x_j y_j$
The mapping should be $\phi(x)=x\otimes x$.

#### Ref
Hofmann, T., Scholkopf, B., & Smola, A. J. (2008). Kernel methods in machine learning. Annals of Statistics, 36(3), 1171–1220. https://doi.org/10.1214/009053607000000677

```
# define the kernel and mapping
K = lambda x, y: x.dot(y)**2
phi = lambda x: (x[:,None]*x).ravel()

# randomly generate some vectors
num_sample = 10
num_dim = 100
x = np.random.randn(num_sample, num_dim)

print("Size of x = %d"%x[0].size)
print("Size of phi(x) = %d"%phi(x[0]).size)

kmat = np.zeros([num_sample, num_sample])

for i in range(num_sample):
    for j in range(num_sample):
        kmat[i, j] = K(x[i], x[j])
        np.testing.assert_almost_equal(kmat[i, j], phi(x[i]).dot(phi(x[j])))
```

    Size of x = 100
    Size of phi(x) = 10000

## Properties
* K is ususally chosen as **positive definite**, the Moore–Aronszajn theorem states that, for every positive definite kernel, there exists a unique RKHS and vice versa.
* sometimes RKHS can be $\infty$-dimensional, e.g. RBF kernel $K(x,y) = \exp(\frac{\|x-y\|}{2\sigma})$

```
# get all eigen values
print(np.linalg.eigvalsh(kmat), "are all positive!")
```

    [ 6705.47440995  8237.20293944  9835.08782136 10220.3915002
     11378.71166636 11835.40403696 13443.38452008 13744.29091067
     15806.48534292 16826.94894898] are all positive!

## Kernel Two Sample Test (MMD)
Defined as the distance in RKHS between two mean embeddings
\begin{align}
    \mathcal{L} =&  \left\|\sum_{x} p_\theta(x) \phi(x)- \sum_{x} \pi(x) \phi(x)  \right\|^2 \\
    =&\langle K(x,y)\rangle_{x\sim p_\theta, y\sim p_\theta}-2\langle K(x,y)\rangle_{x\sim p_\theta,y\sim \pi}+\langle K(x, y)\rangle_{x\sim \pi,y\sim \pi}
\end{align}

#### Ref
Gretton, A. (2012). A Kernel Two-Sample Test. Journal of Machine Learning Research, 13, 723–773. Retrieved from http://kyb.tuebingen.mpg.de/publications/attachments/NIPS2009-Gretton_[0].pdf

## Probability embedding

We next extend the notion of feature map to the embedding of a probability distribution: we will define an element $\mu(p)\in \mathcal H$ such that $E_x(f) = f^T \mu_p$ for all $f \in \mathcal H$.

**Theorem 5**
Let $\mathcal F$ be a unit ball in a **universal** RKHS $\mathcal{H}$, defined on the compact metric space $\mathcal X$, with associated continuous kernel $k(·, ·)$. Then MMD$[\mathcal F, p,q]=0$ if and only if $p=q$.

* universal RKHS: By the universality of $\mathcal H$, for any given $\epsilon > 0$ and $f \in C(X)$ (bounded continuous function) there exists a $g \in \mathcal H$ such that $\|f-g\|_\infty\leq \epsilon$. e.g. Gaussian and Laplace kernels (I. Steinwart. On the influence of the kernel on the consistency of support vector machines. Journal of Machine Learning Research, 2:67–93, 2001.).

#### Ref
A.Berlinet and C. Thomas-Agnan. Reproducing KernelHilbert Spaces in Probability and Statistics. Kluwer, 2004. Chapter 4.

```
class RBFMMD2(object):
    def __init__(self, sigma_list, basis):
        self.sigma_list = sigma_list
        self.basis = basis
        self.K = mix_rbf_kernel(self.basis, self.basis, self.sigma_list)

    def __call__(self, px, py):
        '''
        Args:
            px (1darray, default=None): probability for data set x, used only when self.is_exact==True.
            py (1darray, default=None): same as px, but for data set y.

        Returns:
            float, loss.
        '''
        pxy = px-py
        return self.kernel_expect(pxy, pxy)

    def kernel_expect(self, px, py):
        return px.dot(self.K.dot(py))

    def witness(self, px, py):
        '''witness function of this kernel.'''
        return self.K.dot(px-py)

def mix_rbf_kernel(x, y, sigma_list):
    ndim = x.ndim
    if ndim == 1:
        exponent = np.abs(x[:, None] - y[None, :])**2
    elif ndim == 2:
        exponent = ((x[:, None, :] - y[None, :, :])**2).sum(axis=2)
    else:
        raise
    K = 0.0
    for sigma in sigma_list:
        gamma = 1.0 / (2 * sigma)
        K = K + np.exp(-gamma * exponent)
    return K
```

### Witness Function
To see how sensitive the MMD loss to the probability difference defined as $\delta (x) = p(x)-\pi(x)$. We define
$$W_\pi(x)\propto \langle \phi(x), \mu_p-\mu_\pi \rangle_{\rm RKHS}=E_y[K(y\sim p,x)]-E_y[K(y\sim\pi, x)]$$
So that $\mathcal{L}=\delta (x)^TW_\pi(x)$.

Give a target empirical distribution $\hat{\pi}(x) = \sum\limits_{d_i \in \mathcal{D}} \delta(x-d_i)$, see witness function.

```
basis = digit_basis(num_bit)
mmd = RBFMMD2([20], basis)
print("If p = q, then mmd = %.4f"%mmd(pl1, pl1))
print("If p != q, then mmd = %.4f"%mmd(pl1, pl2))

def show_witness(p, pi):
    wit = mmd.witness(p, pi)
    plt.plot(basis, np.transpose([pi, p, wit/20]))
    plt.legend(['$\pi$', 'p', 'witness'])
    plt.ylim(-0.008,0.008)
    plt.show()

def _empirical(pi, num_sample):
    '''turn a distribution function to empirical distribution function.'''
    samples = np.random.choice(np.arange(2**num_bit), (num_sample,), p=pl1)
    px = np.bincount(samples, minlength=2**num_bit)
    px = px / float(np.sum(px))
    return px

# witness function using band-width 20
show_witness(pl1, pl2)

# witness function using empirical (spiky) distribution
pl1_ = _empirical(pl1, 100)
show_witness(pl2, pl1_)

# witness function using narrower bandwidth
mmd = RBFMMD2([1], basis)
show_witness(pl1, pl2)
```

# Building Differentiable Circuits

$$
\newcommand{\dataset}{{\mathcal{D}}}
\newcommand{\wfunc}{{\psi}}
\newcommand{\thetav}{{\boldsymbol{\theta}}}
\newcommand{\gammav}{{\boldsymbol{\gamma}}}
\newcommand{\thetai}{{\theta^\alpha_l}}
\newcommand{\Expect}{{\mathbb{E}}}
\newcommand{\etc}{{\it etc~}}
\newcommand{\etal}{{\it etal~}}
\newcommand{\xset}{\mathbf{X}}
\newcommand{\gammaset}{\boldsymbol{\Gamma}}
\newcommand{\ei}{\mathbf{e}_l^\alpha}
\newcommand{\sigmag}{{\nu}}
\newcommand{\BAS}{Bars-and-Stripes}
\newcommand{\qexpect}[1]{{\left\langle #1\right\rangle}}
\newcommand{\expect}[2]{{\mathop{\mathbb{E}}\limits_{\substack{#2}}\left[#1\right]}}
\newcommand{\pshift}[1]{{p_{\thetav+#1}}}
$$

### For an Obserable

Consider the expect of $B$ on state $\vert\psi_N\rangle = U_{N:k+1} U_k(\eta)U_{k-1:1}\vert\psi_0\rangle$ with $U_k(\eta)=e^{i\Xi(\eta)}$,
The first task is to get the derivation of the graident of an expectation value that appear in Ref. [2-4] briefly, the gradient

$$\begin{align}\frac{\partial \langle B\rangle_\eta}{\partial \eta} &=i\langle \psi_0\vert U_{N:1}^\dagger BU_{N:k+1} \frac{\partial \Xi(\eta)}{\partial \eta} U_{k:1}\vert \psi_0\rangle-i\langle \psi_0\vert U_{k:1}^\dagger \frac{\partial \Xi(\eta)}{\partial \eta} U_{N:k+1}^\dagger BU_{N:1}\vert \psi_0\rangle\end{align}$$

Here, we have used the fact that $\Xi(\eta)$ is Hermitian. Define $O_{k+1}\equiv U_{N:k+1}^\dagger BU_{N:k+1}$ and $\vert \psi_{k}\rangle\equiv U_{k:1}\vert \psi_0\rangle$, we have

$$\begin{equation}\frac{\partial \langle B\rangle_\eta}{\partial \eta} =\langle\psi_{k}\vert  i\left[O_{k+1}, \frac{\partial \Xi(\eta)}{\partial \eta}\right]\vert \psi_{k}\rangle.\end{equation}$$

Define $A_\pm\equiv\frac{1}{\sqrt{2}} (1\pm i\frac{\partial \Xi(\eta)}{\partial \eta})$, we can easily verify that $ i\left[O_{k+1}, \frac{\partial \Xi(\eta)}{\partial \eta}\right]= A^\dagger_+ O_{k+1}A_+-A_-^\dagger O_{k+1}A_-$,
which can be estimated unbiasedly by constructing $\vert \psi_N\rangle_\pm = U_{N:k+1}A_\pm U_{k:1}\vert \psi_0\rangle$.
Noticing for a non-dissipative system, we further require $A$ being unitary, which means $\frac{\partial \Xi(\eta)}{\partial \eta}^2=1$ (e.g. Pauli operators, CNOT and SWAP).

```
######  Pauli Matrices  ########

I2 = sps.eye(2).tocsr()
sx = sps.csr_matrix([[0,1],[1,0.]])
sy = sps.csr_matrix([[0,-1j],[1j,0.]])
sz = sps.csr_matrix([[1,0],[0,-1.]])

p0 = (sz + I2) / 2
p1 = (-sz + I2) / 2
h = (sx + sz) / np.sqrt(2.)
sxyz = [I2, sx, sy, sz]

# single bit rotation matrices

def _ri(si, theta):
    return np.cos(theta/2.)*I2 - 1j*np.sin(theta/2.)*si

def rx(theta):
    return _ri(sx, theta)

def ry(theta):
    return _ri(sy, theta)

def rz(theta):
    return _ri(sz, theta)

def rot(t1, t2, t3):
    '''
    a general rotation gate rz(t3)rx(r2)rz(t1).

    Args:
        t1, t2, t3 (float): three angles.

    Returns:
        2x2 csr_matrix: rotation matrix.
    '''
    return rz(t3).dot(rx(t2)).dot(rz(t1))

# multiple bit construction

def CNOT(ibit, jbit, n):
    res = _([p0, I2], [ibit, jbit], n)
    res = res + _([p1, sx], [ibit, jbit], n)
    return res

def _(ops, locs, n):
    '''
    Put operators in a circuit and compile them.

    notice the big end are high loc bits!

    Args:
        ops (list): list of single bit operators.
        locs (list): list of positions.
        n (int): total number of bits.

    Returns:
        csr_matrix: resulting matrix.
    '''
    if np.ndim(locs) == 0:
        locs = [locs]
    if not isinstance(ops, (list, tuple)):
        ops = [ops]
    locs = np.asarray(locs)
    locs = n - locs
    order = np.argsort(locs)
    locs = np.concatenate([[0], locs[order], [n + 1]])
    return _wrap_identity([ops[i] for i in order], np.diff(locs) - 1)

def _wrap_identity(data_list, num_bit_list):
    if len(num_bit_list) != len(data_list) + 1:
        raise Exception()

    res = sps.eye(2**num_bit_list[0])
    for data, nbit in zip(data_list, num_bit_list[1:]):
        res = sps.kron(res, data)
        res = sps.kron(res, sps.eye(2**nbit, dtype='complex128'))
    return res

def initial_wf(num_bit):
    wf = np.zeros(2**num_bit, dtype='complex128')
    wf[0] = 1.
    return wf
```

```
class ArbitraryRotation(object):
    def __init__(self, num_bit):
        self.num_bit = num_bit
        self.mask = np.array([True] * (3*num_bit), dtype='bool')

    def __str__(self):
        return 'Rotate[%d]'%(self.num_param)

    @property
    def num_param(self):
        return self.mask.sum()

    def tocsr(self, theta_list):
        '''transform this block to csr_matrix.'''
        theta_list_ = np.zeros(3*self.num_bit)
        theta_list_[self.mask] = theta_list
        rots = [rot(*ths) for ths in theta_list_.reshape([self.num_bit,3])]
        res = [_([r], [i], self.num_bit) for i,r in enumerate(rots)]
        return res

class CNOTEntangler(object):
    def __init__(self, num_bit, pairs):
        self.num_bit = num_bit
        self.pairs = pairs

    def __str__(self):
        pair_str = ','.join(['%d-%d'%(i,j) for i,j in self.pairs])
        return 'CNOT(%s)'%(pair_str)

    @property
    def num_param(self):
        return 0

    def tocsr(self, theta_list):
        '''transform this block to csr_matrix.'''
        i, j = self.pairs[0]
        res = CNOT(i, j, self.num_bit)
        for i, j in self.pairs[1:]:
            res = CNOT(i,j,self.num_bit).dot(res)
        res.eliminate_zeros()
        return [res]
```

```
class BlockQueue(list):
    '''
    Block Queue that keep track of theta_list changing history, for fast update.
    '''
    def __init__(self, *args):
        list.__init__(self, *args)
        self.theta_last = None
        self.memo = None

    def __call__(self, qureg, theta_list):
        # cache? if theta_list change <= 1 parameters, then don't touch memory.
        remember = self.theta_last is None or (abs(self.theta_last-theta_list)>1e-12).sum() > 1

        mats = []
        theta_last = self.theta_last
        if remember:
            self.theta_last = theta_list.copy()

        qureg_ = qureg
        for iblock, block in enumerate(self):
            # generate or use a block matrix
            num_param = block.num_param
            theta_i, theta_list = np.split(theta_list, [num_param])
            if theta_last is not None:
                theta_o, theta_last = np.split(theta_last, [num_param])
            if self.memo is not None and (num_param==0 or np.abs(theta_i-theta_o).max()<1e-12):
                # use data cached in memory
                mat = self.memo[iblock]
            else:
                if self.memo is not None and not remember:
                    # update the changed gate, but not touching memory.
                    mat = _rot_tocsr_update1(block, self.memo[iblock], theta_o, theta_i)
                else:
                    # regenerate one
                    mat = block.tocsr(theta_i)
            for mat_i in mat:
                qureg_ = mat_i.dot(qureg_)
            mats.append(mat)

        if remember:
            # cache data
            self.memo = mats
        # update register
        qureg[...] = qureg_

    @property
    def num_bit(self):
        return self[0].num_bit

    @property
    def num_param(self):
        return sum([b.num_param for b in self])

    def __str__(self):
        return '\n'.join([str(b) for b in self])

def _rot_tocsr_update1(layer, old, theta_old, theta_new):
    '''
    rotation layer csr_matrix update method.

    Args:
        layer (ArbitraryRotation): rotatio layer.
        old (csr_matrix): old matrices.
        theta_old (1darray): old parameters.
        theta_new (1darray): new parameters.

    Returns:
        csr_matrix: new rotation matrices after the theta changed.
    '''
    idiff_param = np.where(abs(theta_old-theta_new)>1e-12)[0].item()
    idiff = np.where(layer.mask)[0][idiff_param]

    # get rotation parameters
    isite = idiff//3
    theta_list_ = np.zeros(3*layer.num_bit)
    theta_list_[layer.mask] = theta_new

    new = old[:]
    new[isite] = _(rot(*theta_list_[isite*3:isite*3+3]), isite, layer.num_bit)
    return new
```

```
def get_nn_pairs(geometry):
    '''define pairs that cnot gates will apply.'''
    num_bit = np.prod(geometry)
    if len(geometry) == 2:
        nrow, ncol = geometry
        res = []
        for ij in range(num_bit):
            i, j = ij // ncol, ij % ncol
            res.extend([(ij, i_ * ncol + j_)
                        for i_, j_ in [((i + 1) % nrow, j), (i, (j + 1) % ncol)]])
        return res
    elif len(geometry) == 1:
        res = []
        for inth in range(2):
            for i in range(inth, num_bit, 2):
                res = res + [(i, i_ % num_bit) for i_ in range(i + 1, i + 2)]
        return res
    else:
       raise NotImplementedError('')
```

```
def get_diff_circuit(num_bit, depth, pairs):
    '''
    build a differentiable circuit

    Args:
        num_bit (int): number of qubit.
        depth (int): depth of circuit.
        pairs (list): list of tuples that represent where CNOT gates applied.
    '''
    blocks = []
    # build circuit
    for idepth in range(depth+1):
        blocks.append(ArbitraryRotation(num_bit))
        if idepth!=depth:
            blocks.append(CNOTEntangler(num_bit, pairs))

    # set leading and trailing Rz to disabled
    blocks[0].mask[::3] = False
    blocks[-1].mask[2::3] = False
    return BlockQueue(blocks)
```

```
depth = 2
geometry = (4,)
num_bit = np.prod(geometry)
pairs = get_nn_pairs(geometry)

circuit = get_diff_circuit(num_bit, depth, pairs)
print(circuit)
```

    Rotate[8]
    CNOT(0-1,2-3,1-2,3-0)
    Rotate[12]
    CNOT(0-1,2-3,1-2,3-0)
    Rotate[8]

```
theta_list = np.random.random(circuit.num_param)*np.pi*2
wf = initial_wf(num_bit)
circuit(wf, theta_list)

print(wf)
```

    [ 0.00056575+0.09888234j  0.19134579-0.30111828j -0.1689071 +0.31456431j
     -0.03176943+0.12489008j -0.0963301 +0.2707479j  -0.10215699-0.2329753j
     -0.25322989+0.02847686j -0.21620301-0.08289819j  0.1910678 -0.08112196j
     -0.12659273-0.319008j    0.16075672+0.14750737j -0.18150813+0.20632003j
     -0.17513398-0.16878919j -0.04635411-0.27225991j  0.14554747-0.00530493j
     -0.06474349+0.0903669j ]

```
# construct an observable
observable = np.random.randn(*[2**num_bit]*2)+1j*np.random.randn(*[2**num_bit]*2)
observable += observable.T.conj()

def expect_val(theta_list):
    wf = initial_wf(num_bit)
    circuit(wf, theta_list)
    return wf.conj().dot(observable.dot(wf))

def gradient_numerical(theta_list, delta=1e-2):
    '''
    numerical differenciation.
    '''
    grad = []
    for i in range(len(theta_list)):
        theta_list[i] += delta/2.
        loss_pos = expect_val(theta_list)
        theta_list[i] -= delta
        loss_neg = expect_val(theta_list)
        theta_list[i] += delta/2.

        grad_i = (loss_pos - loss_neg)/delta
        grad.append(grad_i)
    return np.array(grad)

def gradient(theta_list):
    '''
    cheat and get gradient.
    '''
    grad = []
    for i in range(len(theta_list)):
        # pi/2 phase
        theta_list[i] += np.pi/2.
        mean_pos = expect_val(theta_list)
        # -pi/2 phase
        theta_list[i] -= np.pi
        mean_neg = expect_val(theta_list)
        # recover
        theta_list[i] += np.pi/2.

        grad.append((mean_pos - mean_neg)/2.)
    return np.array(grad)

g1 = gradient(theta_list)
print("Exact = ", g1)
g2 = gradient_numerical(theta_list)
print("Diff = ", g1 - g2)
```

    Exact =  [-0.89463303-1.38777878e-16j -0.11217499-1.11022302e-16j
     -1.79280291-5.55111512e-17j -0.5864455 -1.11022302e-16j
      1.26719505-1.52655666e-16j -0.09476827-3.88578059e-16j
      0.72094707+1.45716772e-16j  1.93379526+5.55111512e-17j
     -2.4973808 -2.77555756e-17j  0.35375861+2.49800181e-16j
     -1.48451599+5.55111512e-17j -0.68485904-1.11022302e-16j
      0.690804  -5.55111512e-17j -0.59058008-5.55111512e-17j
      0.44002793-1.11022302e-16j -2.35894687+4.16333634e-17j
      0.59581721+4.16333634e-17j -2.59120551-1.66533454e-16j
      0.05863847+5.55111512e-17j -2.62490017+0.00000000e+00j
     -0.22833212+4.16333634e-17j  0.56197748-1.11022302e-16j
     -1.62767971-5.55111512e-17j -0.82558004+1.38777878e-16j
     -0.99766406-8.32667268e-17j  0.67353332-2.63677968e-16j
     -0.13788348+9.02056208e-17j  2.27469931+1.11022302e-16j]
    Diff =  [-3.72763313e-06+1.65145675e-14j -4.67395367e-07-1.95399252e-14j
     -7.47000269e-06-3.05866443e-14j -2.44351980e-06-1.39888101e-14j
      5.27997280e-06-1.52655666e-16j -3.94867201e-07+2.38697950e-15j
      3.00394229e-06+8.47238946e-15j  8.05747028e-06+2.22599716e-14j
     -1.04057402e-05-8.35442826e-15j  1.47399244e-06-3.02813330e-14j
     -6.18547567e-06+2.78110868e-14j -2.85357564e-06-1.95399252e-14j
      2.87834626e-06+4.99045250e-14j -2.46074718e-06+4.43534098e-14j
      1.83344727e-06-2.23154828e-14j -9.82893314e-06-1.93872696e-14j
      2.48256844e-06-2.73392420e-15j -1.07966763e-05+2.20379270e-14j
      2.44326573e-07-2.72004641e-15j -1.09370703e-05-6.38378239e-14j
     -9.51382633e-07+3.88994392e-14j  2.34156982e-06-1.12132525e-14j
     -6.78199028e-06+3.60267371e-14j -3.43991266e-06-2.63677968e-15j
     -4.15692840e-06+2.21211938e-14j  2.80638530e-06-1.41414658e-14j
     -5.74513745e-07+1.95191086e-14j  9.47790194e-06+1.39888101e-14j]

### For statistic functional (MMD)

Next, we describe a new class of differenciable loss which can not be written as an obserable easily, the statistic functionals, for simplicity, we consider an arbitrary statistic functional $f(\xset)$, with a sequence of bit strings $\xset\equiv\{x_1,x_2,\ldots, x_r\}$ as its arguments.
Let's define the following expectation of this function

$$\begin{equation}\Expect_f(\gammaset)\equiv\expect{f(\xset)}{\{x_i\sim \pshift{\gammav_i}\}_{i=1}^{r}}. \end{equation}$$

Here, $\gammaset=\{\gammav_1, \gammav_2,\ldots,\gammav_r\}$ is the offset angles applied to circuit parameters,
%Its element $\gammav_i$ is defined in the same parameter space as $\thetav$ that represents a shift to $\thetav$.
which means the probability distributions of generated samples is
$\{\pshift{\gammav_1}, \pshift{\gammav_2},\ldots ,\pshift{\gammav_r}\}$.
Writing out the above expectation explicitly, we have

$$\begin{equation}\Expect_f(\gammaset)=\sum\limits_\xset f(\xset)\prod\limits_i \pshift{\gammav_i}(x_i),\end{equation}$$

where index $i$ runs from $1$ to $r$. Its partial derivative with respect to $\thetai$ is

$$\begin{equation}\frac{\partial \Expect_f(\gammaset)}{\partial \thetai}=\sum\limits_\xset f(\xset)\sum\limits_j\frac{\partial \pshift{\gammav_j}(x_j)}{\partial\thetai}\prod\limits_{i\neq j} \pshift{\gammav_i}(x_i)\end{equation}$$

Again, using the gradient of probability, we have

$$\begin{align}\frac{\partial \Expect_f(\gammaset)}{\partial \thetai}&=\frac{1}{2}\sum\limits_{j,s=\pm}\sum\limits_\xset f(\xset){\pshift{\gammav_j+s\frac{\pi}{2}\ei}(x_j)}\prod\limits_{i\neq j} \pshift{\gammav_i}(x_i)\\&=\frac{1}{2}\sum\limits_{j,s=\pm}\Expect_f(\{\gammav_i+s\delta_{ij}\frac{\pi}{2}\ei\}_{i=1}^{r})\end{align}$$

If $f$ is symmetric, $\Expect_f(\mathbf{0})$ becomes a V-statistic, then the gradient can be further simplified to

$$\begin{align}\frac{\partial \Expect_f(\gammaset)}{\partial \thetai}=\frac{r}{2}\sum\limits_{s=\pm}\Expect_f\left(\{\gammav_0+s\frac{\pi}{2}\ei,\gammav_1,\ldots,\gammav_r\}\right),\end{align}$$

which contains only two terms. This result can be readily verified by calculating the gradient of MMD loss,
noticing the expectation of a kernel function is a V-statistic of degree $2$.
By repeatedly applying the gradient formula, we will be able to obtain higher order gradients.

# Build The Gradient training framework for Born Machine

```
class QCBM(object):
    '''
    Quantum Circuit Born Machine,

    Args:
        circuit (BlockQueue): the circuit architechture.
        batch_size (int|None): introducing sampling error, None for no sampling error.
    '''
    def __init__(self, circuit, mmd, p_data, batch_size=None):
        self.circuit = circuit
        self.mmd = mmd
        self.p_data = p_data
        self.batch_size = batch_size

    @property
    def depth(self):
        return (len(self.circuit)-1)//2

    def pdf(self, theta_list):
        '''get probability distribution function'''
        wf = initial_wf(self.circuit.num_bit)
        self.circuit(wf, theta_list)
        pl = np.abs(wf)**2
        # introducing sampling error
        if self.batch_size is not None:
            pl = prob_from_sample(sample_from_prob(np.arange(len(pl)), pl, self.batch_size),
                    len(pl), False)
        return pl

    def mmd_loss(self, theta_list):
        '''get the loss'''
        # get and cahe probability distritbution of Born Machine
        self._prob = self.pdf(theta_list)
        # use wave function to get mmd loss
        return self.mmd(self._prob, self.p_data)

    def gradient(self, theta_list):
        '''
        cheat and get gradient.
        '''
        prob = self.pdf(theta_list)
        grad = []
        for i in range(len(theta_list)):
            # pi/2 phase
            theta_list[i] += np.pi/2.
            prob_pos = self.pdf(theta_list)
            # -pi/2 phase
            theta_list[i] -= np.pi
            prob_neg = self.pdf(theta_list)
            # recover
            theta_list[i] += np.pi/2.

            grad_pos = self.mmd.kernel_expect(prob, prob_pos) - self.mmd.kernel_expect(prob, prob_neg)
            grad_neg = self.mmd.kernel_expect(self.p_data, prob_pos) - self.mmd.kernel_expect(self.p_data, prob_neg)
            grad.append(grad_pos - grad_neg)
        return np.array(grad)

    def gradient_numerical(self, theta_list, delta=1e-2):
        '''
        numerical differenciation.
        '''
        grad = []
        for i in range(len(theta_list)):
            theta_list[i] += delta/2.
            loss_pos = self.mmd_loss(theta_list)
            theta_list[i] -= delta
            loss_neg = self.mmd_loss(theta_list)
            theta_list[i] += delta/2.

            grad_i = (loss_pos - loss_neg)/delta
            grad.append(grad_i)
        return np.array(grad)

def sample_from_prob(x, pl, num_sample):
    '''
    sample x from probability.
    '''
    pl = 1. / pl.sum() * pl
    indices = np.arange(len(x))
    res = np.random.choice(indices, num_sample, p=pl)
    return np.array([x[r] for r in res])

def prob_from_sample(dataset, hndim, packbits):
    '''
    emperical probability from data.
    '''
    if packbits:
        dataset = packnbits(dataset).ravel()
    p_data = np.bincount(dataset, minlength=hndim)
    p_data = p_data / float(np.sum(p_data))
    return p_data
```

```
def load_gaussian(num_bit, depth, batch_size=None):
    '''gaussian distribution.'''
    geometry = (num_bit,)
    hndim = 2**num_bit

    # standard circuit
    pairs = get_nn_pairs(geometry)
    circuit = get_diff_circuit(num_bit, depth, pairs)

    # bar and stripe
    p_bs = gaussian_pdf(geometry, mu=hndim/2., sigma=hndim/4.)

    # mmd loss
    mmd = RBFMMD2(sigma_list=[0.25,4], basis=digit_basis(geometry))

    # Born Machine
    bm = QCBM(circuit, mmd, p_bs, batch_size=batch_size)
    return bm
```

```
bm = load_gaussian(6, depth)
theta_list = np.random.random(bm.circuit.num_param)*2*np.pi

print(bm.mmd_loss(theta_list))
```

    0.03315822936156594

```
g1 = bm.gradient(theta_list)
print(g1)
g2 = bm.gradient_numerical(theta_list)
print(g1-g2)
```

    [ 5.42728081e-03  9.10485946e-03 -1.30299761e-02 -7.05065456e-05
      4.31816240e-03 -4.79203832e-05  3.56761090e-03  3.84413542e-03
     -4.35883820e-03  1.38246431e-03  3.58551616e-03  4.92344825e-03
      2.20604531e-03 -5.04621581e-04  1.39651786e-02  1.90459756e-03
     -3.13879420e-03 -1.11116678e-02 -2.89498314e-03  2.55574896e-03
     -9.70312153e-04 -3.34279719e-03 -1.60688966e-02  2.08389163e-03
      8.82359256e-03 -9.58209336e-03  2.78738079e-03 -2.52477728e-03
     -5.77484406e-03 -3.89837130e-03 -3.46751009e-03  2.82087082e-03
      2.96495644e-04 -6.05037633e-03 -8.88916704e-05 -7.05716254e-03
      1.12644988e-03  1.52902085e-04  8.85146949e-03  5.39032545e-03
     -1.66131241e-03  7.14929170e-04]
    [ 1.04943764e-07  2.65206660e-07 -1.97721522e-07 -8.47951755e-09
      8.58076832e-08 -4.73371911e-10  8.58643784e-08  3.15749608e-08
     -1.46184334e-07 -5.40492179e-10  1.63113497e-07  5.62470692e-08
     -1.53586267e-08 -3.59366711e-08  1.50532489e-07  1.49675920e-07
     -1.19718795e-07  1.23396361e-07 -1.65125155e-07  9.87286134e-08
     -1.76287260e-09 -3.05225820e-08 -1.53085829e-07  2.56574020e-08
      6.25544895e-08 -1.63353482e-07  7.40252018e-08 -4.93648129e-08
      2.41868066e-08 -4.79960755e-08 -4.39826612e-08  5.15779414e-08
      1.70194484e-10 -8.99080903e-08  1.81158089e-10 -1.31713408e-07
      2.71245671e-08  9.93821506e-08  2.71624440e-08  1.51837504e-09
     -1.17952479e-07  1.17750322e-07]

# Different Training Strategies

```
from scipy.optimize import OptimizeResult

def spsa(fun, x0,  args=(), bounds=None, ac=(0.2, 0.5), alpha=0.602,
         A=None, gamma=0.101, maxiter=5000, callback=None):
    '''
    simultaneous perturbation stochastic approximation.

    Spall, J. C. (1998).
    Implementation of the simultaneous perturbation algorithm for stochastic optimization.
    IEEE Transactions on Aerospace and Electronic Systems.
    https://doi.org/10.1109/7.705889

    Args:
        fun (func): loss function.
        x0 (ndarray): initial variables.
        args (tuple, default=()): additional input parameters for func.
        bounds (tuple|None, default=None): lower bound and higher bound for variables, None for no bounds.
        ac (tuple, default=(0.2,0.5)): initial learning rate and initial perturbation stength.
        A (number, default=0.1*maxiter): statbility constant.
        alpha (float, default=0.602): decay rate for learning speed.
        gamma (float, default=0.101): decay rate for perturbation strength.
        maxiter (int, default=5000): maximum number of iteration.
        callback (func): func(iiter, x) called after each update, with `iiter` the iteration step and `x` the value of variables.

    Note:
        The choice of parameters,
        * `alpha` and `gamma` have thoretically valid default values 0.602 and 0.101.
        * in hight noise setting, pick smaller `a` and larger `c`.
    '''
    if A is None:
        A = 0.1 * maxiter
    a, c = ac

    p = len(x0)
    for k in range(maxiter):
        ak = a / (k + 1 + A)**alpha
        ck = c / (k + 1)**gamma
        g = _get_g(fun, x0, args, ck, return_hessian=False)
        x0 -= ak * g
        if bounds is not None:
            np.clip(x0, *bounds, out=x0)
        if callback is not None:
            callback(x0)
    return OptimizeResult(x=x0, fun=fun(x0, *args), success=True)

def _get_g(fun, x0, args, ck, return_hessian):
    '''calculate gradient'''
    p = len(x0)
    delta = (2 * np.random.randint(0, 2, p) - 1) * ck
    xpos, xneg = x0 + delta, x0 - delta
    fpos, fneg = fun(xpos, *args), fun(xneg, *args)
    g = (fpos - fneg) / (2 * delta)
    if return_hessian:
        delta1 = (2 * np.random.randint(0, 2, p) - 1) * ck
        fneg_ = fun(xneg + delta1, *args)
        fpos_ = fun(xpos + delta1, *args)
        g1n = (fneg_ - fneg) / delta1
        g1p = (fpos_ - fpos) / delta1
        hessian = (g1p - g1n) / 4. / delta[:, None]
        hessian += hessian.T
        return g, hessian
    return g
```

```
  def train(bm, theta_list, method, max_iter=1000, popsize=50, step_rate=0.1):
    '''train a Born Machine.'''
    step = [0]
    def callback(x, *args, **kwargs):
        step[0] += 1
        if step[0]%(max_iter//10) == 1:
            print('step = %d, loss = %s'%(step[0], bm.mmd_loss(x)))

    theta_list = np.array(theta_list)
    if method == 'SPSA':
        res = spsa(bm.mmd_loss, x0=theta_list,
                       maxiter=max_iter, callback=callback,
                       )
        return res.fun, res.x
    elif method == 'Adam':
        try:
            from climin import Adam
        except:
            !pip install git+https://github.com/BRML/climin.git
            from climin import Adam
        optimizer = Adam(wrt=theta_list, fprime=bm.gradient,step_rate=step_rate)
        for info in optimizer:
            callback(theta_list)
            if step[0] == max_iter:
                break
        return bm.mmd_loss(theta_list), theta_list
    else:
        from scipy.optimize import minimize
        res = minimize(bm.mmd_loss, x0=theta_list,
                       method=method, jac = bm.gradient, tol=1e-12,
                       options={'maxiter': max_iter, 'disp': 0, 'gtol':1e-10, 'ftol':0},
                       callback=callback,
                       )
        return res.fun, res.x
```

```
depth = 6
np.random.seed(2)

bm = load_gaussian(6, depth)
theta_list = np.random.random(bm.circuit.num_param)*2*np.pi
loss, theta_list = train(bm, theta_list, 'L-BFGS-B', max_iter=20)
pl = bm.pdf(theta_list)

# display
plt.plot(bm.p_data)
plt.plot(pl)
plt.legend(['Data', 'Gradient Born Machine'])
plt.show()
```

```
def load_barstripe(geometry, depth):
    '''3 x 3 bar and stripes.'''
    num_bit = np.prod(geometry)

    # standard circuit
    pairs = get_nn_pairs(geometry)
    circuit = get_diff_circuit(num_bit, depth, pairs)

    # bar and stripe
    p_bs = barstripe_pdf(geometry)

    # mmd loss
    mmd = RBFMMD2(sigma_list=[2.], basis=binary_basis((num_bit,)))

    # Born Machine
    bm = QCBM(circuit, mmd, p_bs)
    return bm
```

# Training with Noisy Data

Sample size $N=20000$ controls the noises.
We compare training using Adam and SPSA

```
np.random.seed(2)
depth = 4
geometry = (2, 3)

bm = load_barstripe(geometry, depth)
bm.batch_size = 2000
theta_list = np.random.random(bm.circuit.num_param)*2*np.pi
loss, theta_list = train(bm, theta_list, 'Adam', max_iter=100, popsize=10)
pl = bm.pdf(theta_list)

# display
plt.plot(bm.p_data)
plt.plot(pl)
plt.legend(['Data', 'Gradient Born Machine'])
plt.show()
```

```
# generate samples
pl = bm.pdf(theta_list)
indices = np.random.choice(np.arange(len(pl)), np.prod(size), p=pl)
samples = binary_basis(geometry)[indices]

# show
size = (7,5)
plot_bs(samples, size)
```

```
theta_list = np.random.random(bm.circuit.num_param)*2*np.pi
loss, theta_list = train(bm, theta_list, 'SPSA', max_iter=100*bm.circuit.num_param, popsize=10)
pl = bm.pdf(theta_list)

# display
plt.plot(bm.p_data)
plt.plot(pl)
plt.legend(['Data', 'Gradient Born Machine (SPSA)'])
plt.show()
```

# SPSA not Working?

Lets optimize a problem with 24 parameters

```
test_func = lambda x: np.cos(x).sum()
max_iter = 2000

step = [0]
def callback(x, *args, **kwargs):
    step[0] += 1
    if step[0]%(max_iter//50) == 1:
        print('step = %d, loss = %s'%(step[0], test_func(x)))

res = spsa(test_func, x0=np.random.random(24),
               maxiter=max_iter, callback=callback,
               )
```

    step = 1, loss = 20.796581751465737
    step = 41, loss = 17.67989562802431
    step = 81, loss = 14.08103567206227
    step = 121, loss = 8.745535205647682
    step = 161, loss = 4.02103044679887
    step = 201, loss = -0.07336469454172878
    step = 241, loss = -2.797866790716694
    step = 281, loss = -5.78570627582404
    step = 321, loss = -8.718937120989885
    step = 361, loss = -11.091224065672787
    step = 401, loss = -13.055019525299844
    step = 441, loss = -14.502209466639925
    step = 481, loss = -16.099918437533287
    step = 521, loss = -17.25192374217347
    step = 561, loss = -18.078309988841006
    step = 601, loss = -18.986836301802008
    step = 641, loss = -19.573928884899914
    step = 681, loss = -20.243853615671508
    step = 721, loss = -21.268868044065307
    step = 761, loss = -21.74049667232036
    step = 801, loss = -22.37825658815569
    step = 841, loss = -22.60812656731494
    step = 881, loss = -22.882512300222977
    step = 921, loss = -23.094404364598198
    step = 961, loss = -23.30299976104161
    step = 1001, loss = -23.454831268909643
    step = 1041, loss = -23.553484173799554
    step = 1081, loss = -23.62627573507488
    step = 1121, loss = -23.661460288637393
    step = 1161, loss = -23.74388550471457
    step = 1201, loss = -23.78603435687063
    step = 1241, loss = -23.815359056210152
    step = 1281, loss = -23.841780252815113
    step = 1321, loss = -23.86293791434619
    step = 1361, loss = -23.8795268652448
    step = 1401, loss = -23.900278455052
    step = 1441, loss = -23.914259581091514
    step = 1481, loss = -23.92589747248898
    step = 1521, loss = -23.93845703198031
    step = 1561, loss = -23.950247653366162
    step = 1601, loss = -23.958551052042164
    step = 1641, loss = -23.965449023306057
    step = 1681, loss = -23.970440896625725
    step = 1721, loss = -23.974319086828523
    step = 1761, loss = -23.978254013848492
    step = 1801, loss = -23.981154360769914
    step = 1841, loss = -23.983476436793808
    step = 1881, loss = -23.98612699154696
    step = 1921, loss = -23.98821088452821
    step = 1961, loss = -23.989658502179193

SPSA is ok, but not for noisy quantum circuits.

# Inference using Amplitude Amplification

```

```

# Gradient Vanishing?

* Using random circuit
* the derivative with respect to the parameters $\theta$ is Lipschitz continuous with some parameter $\eta$ that depends on the operator $H$.

$$\langle \partial_k E\rangle =\int dU p(U)\partial _k\langle0|U(\theta)^\dagger HU(\theta)|0\rangle$$

Mcclean, J. R., Boixo, S., Smelyanskiy, V. N., Babbush, R., & Neven, H. (2018). Barren plateaus in quantum neural network training landscapes. arXiv:1803.11173, 1–7. Retrieved from https://arxiv.org/pdf/1803.11173.pdf

```
depth = 6
np.random.seed(2)

num_bit_list = np.arange(2, 14, 2)
grad_absmean_list = []
for num_bit in num_bit_list:
    print('number of bit = %d'%num_bit)
    bm = load_gaussian(num_bit, depth)
    grad = bm.gradient(np.random.random(bm.circuit.num_param)*2*np.pi)
    grad_absmean_list.append(np.abs(grad).mean())
```

    number of bit = 2
    number of bit = 4
    number of bit = 6
    number of bit = 8
    number of bit = 10
    number of bit = 12

```
plt.plot(num_bit_list, grad_absmean_list)
plt.yscale('log')
plt.ylabel("gradient", fontsize=18)
plt.xlabel("number of qubit", fontsize=18)
plt.show()
```

# Inference and Learning Wave Functions
to be added...

#### Ref
Low, Guang Hao, Theodore J. Yoder, and Isaac L. Chuang. "Quantum inference on Bayesian networks." Physical Review A 89.6 (2014): 062315.

```

```


# Ball and Racket

*(Sotiropoulos et al. 2012)* proposes to use a Ball and $N$ Bingham distributions of Sticks to describe axon dispersion in tissues with $N\geq1$ axon bundle populations. A Bingham distribution $B(\kappa_1,\kappa_2,\boldsymbol{\mu})$, oriented along orientation $\boldsymbol{\mu}\in\mathbb{S}^2$, quantifies the dispersion of Sticks around $\boldsymbol{\mu}$.
When $\kappa_1\neq\kappa_2$ the Bingham model describes "fanning" or "bending" axon populations *(Kaden et al. 2007)*.
The existence of such configurations has been confirmed with histology in the cortex *(Kleinnijenhuis et al. 2013)* and corpus callosum *(Budde and Annese 2013)*.
Aptly, this model is called Ball and Rackets (BAR). The "Racket" presumably comes from the fanning of strings on a tennis racket. Bar describes the diffusion signal as

\begin{align}
 E^{\textrm{Ball and}}_{\textrm{Rackets}}= \underbrace{f_{\textrm{iso}}\overbrace{E_{\textrm{iso}}(\lambda_{\textrm{iso}})}^{\textrm{Ball}}}_{\textrm{Extra-Axonal}} +\sum_{i=1}^N\underbrace{f_r^i\overbrace{B(\kappa_1,\kappa_2,\boldsymbol{\mu})}^{\textrm{Bingham}}\,*_{\mathbb{S}^2}\,\overbrace{E_r(\cdot|\lambda_\parallel)}^{\textrm{Stick}}}_{\textrm{Intra-Axonal}}
\end{align}

where $*_{\mathbb{S}^2}$ denotes the spherical convolution of the Bingham distribution with the convolution kernel, i.e. the Stick. As such, BAR was the first model designed to estimate axon dispersion and fanning separately per axon bundle. However, as extra-axonal diffusion in between axons is hindered and anisotropic *(Assaf et al. 2004)*, lumping it all together in one isotropic compartment will likely bias estimated model parameters. Moreover, they found that estimating dispersion in crossing tissue configurations in the presence of noise is highly unstable, effectly limiting its application to voxels with one axon bundle when $N=1$.

## Generate Ball and Racket model
The Ball and Racket model can be generated by first calling the basic Ball and Stick components:

```python
from dmipy.signal_models import gaussian_models, cylinder_models
ball = gaussian_models.G1Ball()
stick = cylinder_models.C1Stick()
```

Creating a Bingham-Dispersed Stick representation can then be done as follows:

```python
from dmipy.distributions.distribute_models import SD2BinghamDistributed
dispersed_stick = SD2BinghamDistributed([stick])
```

The Ball and Racket model then finally combined using the MultiCompartmentModel instance:

```python
from dmipy.core.modeling_framework import MultiCompartmentModel
BAR_mod = MultiCompartmentModel(models=[dispersed_stick, ball])
BAR_mod.parameter_names
```

    ['SD2BinghamDistributed_1_SD2Bingham_1_beta_fraction',
     'SD2BinghamDistributed_1_SD2Bingham_1_odi',
     'G1Ball_1_lambda_iso',
     'SD2BinghamDistributed_1_SD2Bingham_1_mu',
     'SD2BinghamDistributed_1_SD2Bingham_1_psi',
     'SD2BinghamDistributed_1_C1Stick_1_lambda_par',
     'partial_volume_0',
     'partial_volume_1']

We can visualize the model, notice that now we have three layers of models, as the Bingham distribution and Stick parameters flow into the BinghamDistributed model:

```python
from IPython.display import Image
BAR_mod.visualize_model_setup(view=False, cleanup=False, with_parameters=True)
Image('Model Setup.png')
```

As you can see, the BAR model has many parameters. To simplify the fitting process, it is possible to fix the diffusivity as we did before with the Ball and Stick model. But, keep in mind that fixing parameters can introduce biases into estimated parameters.

```python
BAR_mod.set_fixed_parameter("SD2BinghamDistributed_1_C1Stick_1_lambda_par", 1.7e-9)
```

```python
# notice that lambda_par is now no longer an estimated parameter.
BAR_mod.parameter_names
```

    ['SD2BinghamDistributed_1_SD2Bingham_1_beta_fraction',
     'SD2BinghamDistributed_1_SD2Bingham_1_odi',
     'G1Ball_1_lambda_iso',
     'SD2BinghamDistributed_1_SD2Bingham_1_mu',
     'SD2BinghamDistributed_1_SD2Bingham_1_psi',
     'partial_volume_0',
     'partial_volume_1']

# Human Connectome Project Example

```python
from dmipy.data import saved_data
scheme_hcp, data_hcp = saved_data.wu_minn_hcp_coronal_slice()
```

    This data slice originates from Subject 100307 of the Human Connectome Project, WU-Minn Consortium (Principal Investigators: David Van Essen and Kamil Ugurbil; 1U54MH091657) funded by the 16 NIH Institutes and Centers that support the NIH Blueprint for Neuroscience Research; and by the McDonnell Center for Systems Neuroscience at Washington University.

```python
import matplotlib.pyplot as plt
import matplotlib.patches as patches
%matplotlib inline

fig, ax = plt.subplots(1)
ax.imshow(data_hcp[:, 0, :, 0].T, origin=True)
rect = patches.Rectangle((70,70),20,20,linewidth=1,edgecolor='r',facecolor='none')
ax.add_patch(rect)
ax.set_axis_off()
ax.set_title('HCP coronal slice B0 with ROI');
```

## Fit Ball and Racket to HCP data
Having numba and pathos installed, fitting the model to a coronal slice takes about 40 minutes:

```python
BAR_fit_hcp = BAR_mod.fit(scheme_hcp, data_hcp, mask=data_hcp[..., 0]>0)
```

    Using parallel processing with 8 workers.
    Setup brute2fine optimizer in 42.4190790653 seconds
    Fitting of 8181 voxels complete in 2292.53698182 seconds.
    Average of 0.280226987144 seconds per voxel.

## Visualize results

```python
fitted_parameters = BAR_fit_hcp.fitted_parameters

fig, axs = plt.subplots(2, 3, figsize=[15, 10])
axs = axs.ravel()

counter = 0
for name, values in fitted_parameters.items():
    if values.squeeze().ndim != 2:
        continue
    cf = axs[counter].imshow(values.squeeze().T, origin=True, interpolation='nearest')
    axs[counter].set_title(name)
    fig.colorbar(cf, ax=axs[counter], shrink=0.5)
    counter += 1
```

Notice that we get some familiar maps, where the Ball model takes the role of extra-axonal and CSF compartments, and the stick takes the intra-axonal compartment.

The more unfamiliar maps are those of $\beta$, $\kappa$ and $\psi$. $\kappa$ represents primary isotropic concentration as in NODDI, while $\beta$ describes the anisotropy of the concentration (bending or fanning tissue). $\psi$ describes the angle of the anisotropy.

Also notice that the maps are not completely smooth (especially lambda_par) as optimization can still fall in a local minima despite the brute force optimization.

## Estimating Fiber Orientation Distributions (FODs)

We estimate FODs on the right side of the corpus callosum and use the estimated concentration $\kappa$ as background.

```python
from dipy.data import get_sphere
from dipy.viz.actor import slicer
sphere = get_sphere(name='symmetric724')
fods = BAR_fit_hcp.fod(sphere.vertices)
```

```python
import numpy as np
affine = np.eye(4)
affine[0,3] = -10
affine[1,3] = -10

odi_res = fitted_parameters['SD2BinghamDistributed_1_SD2Bingham_1_odi']
odi_im = slicer(odi_res[70:90,0, 70:90, None],interpolation='nearest', affine=affine, opacity=0.7)

beta_res = fitted_parameters['SD2BinghamDistributed_1_SD2Bingham_1_beta_fraction']
beta_im = slicer(beta_res[70:90,0, 70:90],interpolation='nearest', affine=affine, opacity=0.7)
```

```python
from dipy.viz import fvtk
ren = fvtk.ren()
fod_spheres = fvtk.sphere_funcs(fods[70:90,:, 70:90], sphere, scale=1., norm=False)
fod_spheres.RotateX(90)
fod_spheres.RotateZ(180)
fod_spheres.RotateY(180)
fvtk.add(ren, fod_spheres)
fvtk.add(ren, odi_im)
fvtk.record(ren=ren, size=[700, 700], out_path='dipy_odi.png')

ren = fvtk.ren()
fod_spheres = fvtk.sphere_funcs(fods[70:90,:, 70:90], sphere, scale=1., norm=False)
fod_spheres.RotateX(90)
fod_spheres.RotateZ(180)
fod_spheres.RotateY(180)
fvtk.add(ren, fod_spheres)
fvtk.add(ren, beta_im)
fvtk.record(ren=ren, size=[700, 700], out_path='dipy_beta.png')
```

```python
import matplotlib.image as mpimg
img1 = mpimg.imread('dipy_odi.png')
img2 = mpimg.imread('dipy_beta.png')

fig, axs = plt.subplots(1, 2, figsize=[20, 70])
axs[0].set_title('FODs with ODI background', fontsize=20)
axs[0].imshow(img1[100:-97, 100:-85])
axs[0].axis('off')
axs[1].set_title('FODs with beta_fraction background', fontsize=20)
axs[1].imshow(img2[100:-97, 100:-85])
axs[1].axis('off');
```

You can see the corpus callosum coming in the from the left and joining the centrum semiovale on the right. The intensity of the background shows the ODI of the Bingham distribution on the left, and beta_fraction on the right. Notice it is high in the corpus callosum and lower in non-white matter areas. Also notice that in the CSF on the bottom you see very high $\kappa$, but it means nothing since the intra-axonal volume fraction close to zero in these areas. Also notice that Ball and Racket cannot estimate crossings, which are expected in the centrum semiovale on the right. Using multiple compartments to model crossings is explored in the Microstructure in Crossings examples.

```python
from os import remove
remove('dipy_odi.png')
remove('dipy_beta.png')
```

## Estimating Error Metrics: MSE and $R^2$

Whenever you fit a model, it's always wise to check out the error of the model fit. Here we estimate the Mean Squared Error (MSE) and the $R^2$ coefficient of determination. In MSE the lower is better, while $R^2$ has an upper bound of one, which represents a perfect model fit.

```python
mse = BAR_fit_hcp.mean_squared_error(data_hcp)
R2 = BAR_fit_hcp.R2_coefficient_of_determination(data_hcp)

fig, axs = plt.subplots(1, 2, figsize=[15, 15])
cf = axs[0].imshow(mse.squeeze().T, origin=True, vmax=0.02, cmap='Greys_r')
fig.colorbar(cf, ax=axs[0], shrink=0.33)
axs[0].set_title('Mean Squared Error', fontsize=20)
cf = axs[1].imshow(R2.squeeze().T, origin=True, vmin=0.6, cmap='Greys_r')
fig.colorbar(cf, ax=axs[1], shrink=0.33)
axs[1].set_title('R2 - Coefficient of Determination', fontsize=20);
```

Ideally, you want the MSE and $R^2$ to show no structure of the underlying data - which would mean the model can at least fit the data equally well everywhere. Instead, in Ball and Racket we can see clearly see the structure of the different brain areas. This means that Ball and Racket model cannot fit the diffusion equally well in all brian areas, and that interpretation of model parameters will be biased when comparing them between these different areas.

When doing these studies, ALWAYS verify whether differences in estimated parameters reflect actual tissue changes, or only reflect the limitations of the model itself.

## References
- Assaf, Yaniv, et al. "New modeling and experimental framework to characterize hindered and restricted water diffusion in brain white matter." Magnetic Resonance in Medicine 52.5 (2004): 965-978.
- Sotiropoulos, Stamatios N., Timothy EJ Behrens, and Saad Jbabdi. "Ball and rackets: inferring fiber fanning from diffusion-weighted MRI." Neuroimage 60.2 (2012): 1412-1425.
- Kaden, Enrico, Thomas R. Knösche, and Alfred Anwander. "Parametric spherical deconvolution: inferring anatomical connectivity using diffusion MR imaging." NeuroImage 37.2 (2007): 474-488.
- Budde, Matthew D., and Jacopo Annese. "Quantification of anisotropy and fiber orientation in human brain histological sections." Frontiers in integrative neuroscience 7 (2013): 3.
- Kleinnijenhuis, M., et al. "Detailed laminar characteristics of the human neocortex revealed by NODDI and histology." 2013.


```
#all imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sympy import sieve
from fractions import Fraction
```

```
def mobius(n: int):
    return list(sieve.mobiusrange(n, n+1))[0]

#n must be divisible by 4
def search_t(d_max: int, n: int):
    t=0
    for d in range(1, d_max+1):
        mu = mobius(d)
        j_sum = 0
        limit = 3*n//(4*d)
        for j in range(1, limit+1):
            j_sum += min(n//(4*d), j//3)
        t += mu*j_sum
    return t

#n must be divisible by 4
def search_t2(d_max: int, n: int):
    t=0
    m=n//4
    for d in range(1, d_max+1):
        mu = mobius(d)
        j_sum = 0
        limit = (n-m)//d
        for j in range(1, limit+1):
            j_sum += int(min(m/d, j/3))
        t += mu*j_sum
    return t

arr = []
for i in range(2, 22):
    arr.append([i*4, search_t2(1, i*4)])

df = pd.DataFrame(arr, columns=['n', 't'])
df.head(20)
```

      n
      t

      0
      8
      5

      1
      12
      12

      2
      16
      22

      3
      20
      35

      4
      24
      51

      5
      28
      70

      6
      32
      92

      7
      36
      117

      8
      40
      145

      9
      44
      176

      10
      48
      210

      11
      52
      247

      12
      56
      287

      13
      60
      330

      14
      64
      376

      15
      68
      425

      16
      72
      477

      17
      76
      532

      18
      80
      590

      19
      84
      651

```
class Fr(Fraction):
    def __repr__(self):
        return '(%s/%s)' % (self.numerator, self.denominator)

def farey(n, length=False):
    if not length:
        return [Fr(0, 1)] + sorted({Fr(m, k) for k in range(1, n+1) for m in range(1, k+1)})
    else:
        return (n*(n+3))//2 - sum(farey(n//k, True) for k in range(2, n+1))

def farey_n_m(n, m):
    f_n = farey(n)
    res = []
    for f_n_member in f_n:
        h = f_n_member.numerator
        k = f_n_member.denominator
        if m+k-n <= h and h <= m:
            res.append(f_n_member)
    return res

print('F(B(12),3):')
print(farey_n_m(12,3))
print('F(B(12),6):')
print(farey_n_m(12,6))

#print('Farey sequence for order 7')
#print(farey(7))

#print('Number of fractions in the Farey sequence for order 100 through 1,000 (inclusive) by hundreds:')
#print([farey(i, length=True) for i in range(100, 1001, 100)])
```

    F(B(12),3):
    [(0/1), (1/10), (1/9), (1/8), (1/7), (1/6), (2/11), (1/5), (2/9), (1/4), (3/11), (2/7), (3/10), (1/3), (3/8), (2/5), (3/7), (1/2), (3/5), (2/3), (3/4), (1/1)]
    F(B(12),6):
    [(0/1), (1/7), (1/6), (1/5), (1/4), (2/7), (1/3), (3/8), (2/5), (3/7), (4/9), (5/11), (1/2), (6/11), (5/9), (4/7), (3/5), (5/8), (2/3), (5/7), (3/4), (4/5), (5/6), (6/7), (1/1)]

```

```


# Density Estimation

### Preliminaries

- Goal
  - Simple maximum likelihood estimates for Gaussian and categorical distributions
- Materials
  - Mandatory
    - These lecture notes
  - Optional
    - Bishop pp. 67-70, 74-76, 93-94

### Why Density Estimation?

Density estimation relates to building a model $p(x|\theta)$ from observations $D=\{x_1,\dotsc,x_N\}$.

Why is this interesting? Some examples:

- **Outlier detection**. Suppose $D=\{x_n\}$ are benign mammogram images. Build $p(x | \theta)$ from $D$. Then low value for $p(x^\prime | \theta)$ indicates that $x^\prime$ is a risky mammogram.

- **Compression**. Code a new data item based on **entropy**, which is a functional of $p(x|\theta)$:
$$
H[p] = -\sum_x p(x | \theta)\log p(x |\theta)
$$

- **Classification**. Let $p(x | \theta_1)$ be a model of attributes $x$ for credit-card holders that paid on time and $p(x | \theta_2)$ for clients that defaulted on payments. Then, assign a potential new client $x^\prime$ to either class based on the relative probability of $p(x^\prime | \theta_1)$ vs. $p(x^\prime|\theta_2)$.

### Example Problem

Consider a set of observations $D=\{x_1,…,x_N\}$ in the 2-dimensional plane (see Figure). All observations were generated by the same process. We now draw an extra observation $x_\bullet = (a,b)$ from the same data generating process. What is the probability that $x_\bullet$ lies within the shaded rectangle $S$?

```julia
using Distributions, PyPlot
N = 100
generative_dist = MvNormal([0,1.], [0.8 0.5; 0.5 1.0])
function plotObservations(obs::Matrix)
    plot(obs[1,:], obs[2,:], "kx", zorder=3)
    fill_between([0., 2.], 1., 2., color="k", alpha=0.4, zorder=2) # Shaded area
    text(2.05, 1.8, "S", fontsize=12)
    xlim([-3,3]); ylim([-2,4]); xlabel("a"); ylabel("b")
end
D = rand(generative_dist, N) # Generate observations from generative_dist
plotObservations(D)
x_dot = rand(generative_dist) # Generate x∙
plot(x_dot[1], x_dot[2], "ro");

```

### Log-Likelihood for a Multivariate Gaussian (MVG)

- Assume we are given a set of IID data points $D=\{x_1,\ldots,x_N\}$, where $x_n \in \mathbb{R}^D$. We want to build a model for these data.

- **Model specification**: Let's assume a MVG model $x_n=\mu+\epsilon_n$ with $\epsilon_n \sim \mathcal{N}(0,\Sigma)$, or equivalently,

$$\begin{align*}
p(x_n|\mu,\Sigma) &= \mathcal{N}(x_n|\mu,\Sigma)
= |2 \pi \Sigma|^{-1/2} \mathrm{exp} \left\{-\frac{1}{2}(x_n-\mu)^T
\Sigma^{-1} (x_n-\mu) \right\}
\end{align*}$$

- Since the data are IID, $p(D|\theta)$ factorizes as

$$
p(D|\theta) = p(x_1,\ldots,x_N|\theta) \stackrel{\text{IID}}{=} \prod_n p(x_n|\theta)
$$

- This choice of model yields the following log-likelihood (use (B-C.9) and (B-C.4)),

$$\begin{align*}
 \log &p(D|\theta) = \log \prod_n p(x_n|\theta) = \sum_n \log \mathcal{N}(x_n|\mu,\Sigma) \tag{1}\\
     &= N \cdot \log | 2\pi\Sigma |^{-1/2} - \frac{1}{2} \sum\nolimits_{n} (x_n-\mu)^T \Sigma^{-1} (x_n-\mu)
 \end{align*}$$

### Maximum Likelihood estimation of mean of MVG

- We want to maximize $\log p(D|\theta)$ wrt the parameters $\theta=\{\mu,\Sigma\}$. Let's take derivatives; first to mean $\mu$, (making use of  (B-C.25) and (B-C.27)),

$$\begin{align*}
\nabla_\mu \log p(D|\theta) &= -\frac{1}{2}\sum_n \nabla_\mu \left[ (x_n-\mu)^T \Sigma^{-1} (x_n-\mu) \right] \\
&= -\frac{1}{2}\sum_n \nabla_\mu \mathrm{Tr} \left[ -2\mu^T\Sigma^{-1}x_n + \mu^T\Sigma^{-1}\mu \right]  \\
&= -\frac{1}{2}\sum_n \left( -2\Sigma^{-1}x_n + 2\Sigma^{-1}\mu \right) \\
&= \Sigma^{-1}\,\sum_n \left( x_n-\mu \right)
\end{align*}$$

- Setting the derivative to zero yields the **sample mean**

$$\begin{equation*}
\boxed{
\hat \mu = \frac{1}{N} \sum_n x_n
}
\end{equation*}$$

### Maximum Likelihood estimation of variance of MVG

- Now we take the gradient of the log-likelihood wrt the **precision matrix** $\Sigma^{-1}$ (making use of B-C.28 and B-C.24)

$$\begin{align*}
\nabla_{\Sigma^{-1}}  &\log p(D|\theta) \\
&= \nabla_{\Sigma^{-1}} \left[ \frac{N}{2} \log |2\pi\Sigma|^{-1} - \frac{1}{2} \sum_{n=1}^N (x_n-\mu)^T \Sigma^{-1} (x_n-\mu)\right] \\
&= \nabla_{\Sigma^{-1}} \left[ \frac{N}{2} \log |\Sigma^{-1}| - \frac{1}{2} \sum_{n=1}^N \mathrm{Tr} \left[ (x_n-\mu) (x_n-\mu)^T \Sigma^{-1}\right] \right]\\
&= \frac{N}{2}\Sigma -\frac{1}{2}\sum_n (x_n-\mu)(x_n-\mu)^T
\end{align*}$$

Get optimum by setting the gradient to zero,
$$\begin{equation*}
\boxed{
\hat \Sigma = \frac{1}{N} \sum_n (x_n-\hat\mu)(x_n - \hat\mu)^T}
\end{equation*}$$
which is also known as the **sample variance**.

### Sufficient Statistics

- Note that the ML estimates can also be written as
$$\begin{equation*}
\hat \Sigma = \sum_n x_n x_n^T - \left( \sum_n x_n\right)\left( \sum_n x_n\right)^T, \quad \hat \mu = \frac{1}{N} \sum_n x_n
\end{equation*}$$

- I.o.w., the two statistics (a 'statistic' is a function of the data) $\sum_n x_n$ and $\sum_n x_n x_n^T$ are sufficient to estimate the parameters $\mu$ and $\Sigma$ from $N$ observations. In the literature, $\sum_n x_n$ and  $\sum_n x_n x_n^T$ are called **sufficient statistics** for the Gaussian PDF.

- The actual parametrization of a PDF is always a re-parameteriation of the sufficient statistics.

- Sufficient statistics are useful because they summarize all there is to learn about the data set in a minimal set of variables.

### Solution to Example Problem

We apply maximum likelihood estimation to fit a 2-dimensional Gaussian model ($m$) to data set $D$. Next, we evaluate $p(x_\bullet \in S | m)$ by (numerical) integration of the Gaussian pdf over $S$: $p(x_\bullet \in S | m) = \int_S p(x|m) \mathrm{d}x$.

```julia
using HCubature, LinearAlgebra# Numerical integration package
# Maximum likelihood estimation of 2D Gaussian
μ = 1/N * sum(D,dims=2)[:,1]
D_min_μ = D - repeat(μ, 1, N)
Σ = Hermitian(1/N * D_min_μ*D_min_μ')
m = MvNormal(μ, convert(Matrix, Σ));

# Contour plot of estimated Gaussian density
A = Matrix{Float64}(undef,100,100); B = Matrix{Float64}(undef,100,100)
density = Matrix{Float64}(undef,100,100)
for i=1:100
    for j=1:100
        A[i,j] = a = (i-1)*6/100 .- 2
        B[i,j] = b = (j-1)*6/100 .- 3
        density[i,j] = pdf(m, [a,b])
    end
end
c = contour(A, B, density, 6, zorder=1)
PyPlot.set_cmap("cool")
clabel(c, inline=1, fontsize=10)

# Plot observations, x∙, and the countours of the estimated Gausian density
plotObservations(D)
plot(x_dot[1], x_dot[2], "ro")

# Numerical integration of p(x|m) over S:
(val,err) = hcubature((x)->pdf(m,x), [0., 1.], [2., 2.])
println("p(x⋅∈S|m) ≈ $(val)")
```

### Discrete Data: the 1-of-K Coding Scheme

- Consider a coin-tossing experiment with outcomes $x \in\{0,1\}$ (tail and head) and let $0\leq \mu \leq 1$ represent the probability of heads. This model can written as a **Bernoulli distribution**:
$$
p(x|\mu) = \mu^{x}(1-\mu)^{1-x}
$$
  - Note that the variable $x$ acts as a (binary) **selector** for the tail or head probabilities. Think of this as an 'if'-statement in programming.

- **1-of-K coding scheme**. Now consider a $K$-sided coin (a _die_ (pl.: dice)). It is convenient to code the outcomes by $x=(x_1,\ldots,x_K)^T$ with **binary selection variables**
$$
x_k = \begin{cases} 1 & \text{if die landed on $k$th face}\\
0 & \text{otherwise} \end{cases}
$$
  - E.g., for $K=6$, if the die lands on the 3rd face $\,\Rightarrow x=(0,0,1,0,0,0)^T$.

- Assume the probabilities $p(x_k=1) = \mu_k$ with  $\sum_k \mu_k  = 1$. The data generating distribution is then (note the similarity to the Bernoulli distribution)

$$
p(x|\mu) = \mu_1^{x_1} \mu_2^{x_2} \cdots \mu_k^{x_k}=\prod_k \mu_k^{x_k}
$$

- This generalized Bernoulli distribution is called the **categorical distribution** (or sometimes the 'multi-noulli' distribution).

### Categorical vs. Multinomial Distribution

- Observe a data set $D=\{x_1,\ldots,x_N\}$  of $N$ IID rolls of a $K$-sided die, with generating PDF
$$
p(D|\mu) = \prod_n \prod_k \mu_k^{x_{nk}} = \prod_k \mu_k^{\sum_n x_{nk}} = \prod_k \mu_k^{m_k}
$$
where $m_k= \sum_n x_{nk}$ is the total number of occurrences that we 'threw' $k$ eyes.

- This distribution depends on the observations **only** through the quantities $\{m_k\}$, with generally $K \ll N$.

- A related distribution is the distribution over $D_m=\{m_1,\ldots,m_K\}$, which is called the **multinomial distribution**,
$$
p(D_m|\mu) =\frac{N!}{m_1! m_2!\ldots m_K!} \,\prod_k \mu_k^{m_k}\,.
$$

- The catagorical distribution $p(D|\mu) = p(\,x_1,\ldots,x_N\,|\,\mu\,)$ is a distribution over the **observations** $\{x_1,\ldots,x_N\}$, whereas the multinomial distribution $p(D_m|\mu) = p(\,m_1,\ldots,m_K\,|\,\mu\,)$ is a distribution over the **data frequencies** $\{m_1,\ldots,m_K\}$.

### Maximum Likelihood Estimation for the Multinomial

- Now let's find the ML estimate for $\mu$, based on $N$ throws of a $K$-sided die. Again we use the shorthand $m_k \triangleq \sum_n x_{nk}$.

- The log-likelihood for the multinomial distribution is given by

$$\begin{align*}
\mathrm{L}(\mu) &\triangleq \log p(D_m|\mu) \propto \log \prod_k \mu_k^{m_k} =  \sum_k m_k \log \mu_k \tag{2}
\end{align*}$$

- When doing ML estimation, we must obey the constraint $\sum_k \mu_k  = 1$, which can be accomplished by a Lagrange multiplier. The **augmented log-likelihood** with Lagrange multiplier is then

$$
\mathrm{L}^\prime(\mu) = \sum_k m_k \log \mu_k  + \lambda \cdot (1 - \sum_k \mu_k )
$$

- Set derivative to zero yields the **sample proportion** for $\mu_k$

$$\begin{equation*}
\nabla_{\mu_k}   \mathrm{L}^\prime = \frac{m_k }
{\hat\mu_k } - \lambda  \overset{!}{=} 0 \; \Rightarrow \; \boxed{\hat\mu_k = \frac{m_k }{N}}
\end{equation*}$$

where we get $\lambda$ from the constraint

$$\begin{equation*}
\sum_k \hat \mu_k = \sum_k \frac{m_k}
{\lambda} = \frac{N}{\lambda} \overset{!}{=}  1
\end{equation*}$$

### Recap ML for Density Estimation

Given $N$ IID observations $D=\{x_1,\dotsc,x_N\}$.

- For a **multivariate Gaussian** model $p(x_n|\theta) = \mathcal{N}(x_n|\mu,\Sigma)$, we obtain ML estimates

$$\begin{align}
\hat \mu &= \frac{1}{N} \sum_n x_n \tag{sample mean} \\
\hat \Sigma &= \frac{1}{N} \sum_n (x_n-\hat\mu)(x_n - \hat \mu)^T \tag{sample variance}
\end{align}$$

- For discrete outcomes modeled by a 1-of-K **categorical distribution** we find

$$\begin{align}
\hat\mu_k  = \frac{1}{N} \sum_n x_{nk} \quad \left(= \frac{m_k}{N} \right) \tag{sample proportion}
\end{align}$$

- Note the similarity for the means between discrete and continuous data.

- We didn't use a co-variance matrix for discrete data. Why?

```julia
open("../../styles/aipstyle.html") do f
    display("text/html", read(f, String))
end
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>

```julia

```


```
from IPython.core.display import HTML

with open('../common/creativecommons.html', 'r') as f:
    html = f.read()

with open('../common/custom.css', 'r') as f:
    styles = f.read()

HTML(styles)
```

```
import numpy as np
import matplotlib.pyplot as plt

from sympy import init_printing, symbols, pi
from sympy.plotting import plot3d_parametric_surface

init_printing()
```

```
r, h, u = symbols('r h u')
V = (r**2 * h * pi) / 3
V
```

```
circ = V.diff(r)
circ
```

```
area = V.diff(h)
area
```

## Gradient

```
x, y = np.mgrid[-100:101:25., -100:101:25.]

V = 2 * x ** 2 + 3 * y ** 2

Ex, Ey = np.gradient(V)
Ex = - Ex
Ey = - Ey

fig, ax = plt.subplots(figsize=(4, 4))
ax.contourf(x, y, V, cmap=plt.cm.Greys_r)
_ = ax.quiver(x, y, Ex, Ey, color='b')
# fig.savefig('gradient.png')
```

## Divergence (Fake!)
### Horizontally non-divergent (u=U_ox, v=U_oy)

```
dx = 0.5
x = np.arange(-4, 4 + dx, dx)
y = x.copy()

X, Y = np.meshgrid(x, y)
f = X ** 2 - Y ** 2
Dx, Dy = np.gradient(f)

fig, ax = plt.subplots(figsize=(4, 4))
ax.contourf(x, y, f, cmap=plt.cm.Greys)
_ = ax.quiver(x, y, Dy, Dx, color='b')

# fig.savefig('divergence.png')
```

## Curl (Fake!)
### z-component of curl = constant
### (u=-U_oy, v=U_ox)

```
fig, ax = plt.subplots(figsize=(4, 4))
_ = ax.quiver(x, y, Dx, Dy, color='b')
# fig.savefig('curl.png')
```

```
HTML(html)
```


```python
from kamodo.kamodo import Kamodo
```

## LaTeX support
Kamodo supports both python and LaTex-formatted expressions as input. For LaTeX, you must wrap your expression in ```$ $```:

```python
Kamodo(f = 'x**2 + y**2', g = '$2x^2 + 3y^2$')
```

\begin{equation}f{\left(x,y \right)} = x^{2} + y^{2}\end{equation}\begin{equation}g{\left(x,y \right)} = 2 x^{2} + 3 y^{2}\end{equation}

## Conventions
Kamodo's variable names have to follow python's naming conventions - only numbers, letters, and underscores, which are too restrictive for mathematical symbols. Therefore, Kamodo uses sympy's conventions when generating LaTeX from variable names, which provide a means to write mathematical symbols in a way ammenable to python. More details of sympy's conventions may be found [here](https://docs.sympy.org/0.7.0/modules/galgebra/latex_ex/latex_ex.html#). Kamodo also adds some additional features not covered by sympy.

#### Superscripts/Subscripts

Subscripts are encoded with single underscores. Superscripts are encoded with double underscores. Combinations are possible.

```python
Kamodo('x_i = a', 'y__j = b', 'z_oxygen__2 = c')
```

\begin{equation}\operatorname{x_{i}}{\left(a \right)} = a\end{equation}\begin{equation}\operatorname{y^{j}}{\left(b \right)} = b\end{equation}\begin{equation}\operatorname{z^{2}_{oxygen}}{\left(c \right)} = c\end{equation}

#### Greek letters
Most greek letters are supported using their corresponding english name. Use capitalization if the greek letter should also be capitalized.

```python
Kamodo(rho = 'ALPHA+BETA+Gamma')
```

\begin{equation}\rho{\left(\alpha,\beta,\Gamma \right)} = \alpha + \beta + \Gamma\end{equation}

!!! warning
    Some greek letters (e.g. pi, zeta) may conflict with Sympy's namespace. In that case, use all caps (e.g. PI, ZETA)

#### plus/minus operators

In Python we cannot have variables embedded with ```+``` or ```-```, but we may still need these symbols to represent, say ionization or simulation time step. The table below shows how we map from (part of) a variable name to its corresponding latex output.

variable | to latex
------------ | -------------
plus | +
minus | -
comma | ,
LEFT | \\\left (
RIGHT | \\\right )
prime | '

Here is how you would use these in your functions:

```python
Kamodo(x_iplus1 = 'x_i*.9', O__minus = 'e**-h', OLEFT3PRIGHT = 't', fprime = 'x')
```

\begin{equation}\operatorname{x_{i+1}}{\left(x_{i} \right)} = 0.9 x_{i}\end{equation}\begin{equation}\operatorname{O^{-}}{\left(e,h \right)} = e^{- h}\end{equation}\begin{equation}\operatorname{O\left (3P\right )}{\left(t \right)} = t\end{equation}\begin{equation}\operatorname{{f}'}{\left(x \right)} = x\end{equation}

## Variable reuse

Variables may only have one function representing their evaluation. If you try to define a variable twice, the second version will override the first. However, if you want to represent that variable in a different context but keep using its name, there are two options:

1. Annotation - add superscripts/subscripts to distinguish between the different implentations.
2. Mimicry - use a new name that produces the same LaTeX output.

```python
Kamodo(rho = 'x + y + z', RHO = 'r*sin(theta)*cos(phi)', rho_2D = 'x + y')
```

\begin{equation}\rho{\left(x,y,z \right)} = x + y + z\end{equation}\begin{equation}\rho{\left(\phi,r,\theta \right)} = r \sin{\left(\theta \right)} \cos{\left(\phi \right)}\end{equation}\begin{equation}\rho_{2D}{\left(x,y \right)} = x + y\end{equation}

!!! warning
    Mimicry can cause confusion if the signature of the left-hand-side does not change, as in the example below:

```python
Kamodo(rho = 'x + y', RHO = '3*x + y')
```

\begin{equation}\rho{\left(x,y \right)} = x + y\end{equation}\begin{equation}\rho{\left(x,y \right)} = 3 x + y\end{equation}