```python
import numpy as np
import numba
import matplotlib.pyplot as plt
import sympy as sym
plt.style.use('presentation.mplstyle')
%matplotlib notebook

def d2np(d):

    names = []
    numbers = ()
    dtypes = []
    for item in d:
        names += item
        if type(d[item]) == float:
            numbers += (d[item],)
            dtypes += [(item,float)]
        if type(d[item]) == int:
            numbers += (d[item],)
            dtypes += [(item,int)]
        if type(d[item]) == np.ndarray:
            numbers += (d[item],)
            dtypes += [(item,np.float64,d[item].shape)]
    return np.array([numbers],dtype=dtypes)
```

```python
i_1d,i_1q,i_2d,i_2q,v_cd,v_cq = sym.symbols('i_1d,i_1q,i_2d,i_2q,v_cd,v_cq')
di_1d,di_1q,di_2d,di_2q,dv_cd,dv_cq = sym.symbols('di_1d,di_1q,di_2d,di_2q,dv_cd,dv_cq')
L_1,L_2,C_ac,C_dc = sym.symbols('L_1,L_2,C_ac,C_dc')
R_1,R_2 = sym.symbols('R_1,R_2')
omega,dummy = sym.symbols('omega,dummy')
v_sd,v_sq,v_dc= sym.symbols('v_sd,v_sq,v_dc')
eta_d,eta_q = sym.symbols('eta_d,eta_q')
p_ref,q_ref = sym.symbols('p_ref,q_ref')
i_2d_ref,i_2q_ref = sym.symbols('i_2d_ref,i_2q_ref')

#i_2d = i_2d_ref
#i_2q = i_2q_ref

di_1d = 1/L_1*(0.5*eta_d*v_dc - R_1*i_1d + L_1*omega*i_1q - v_cd)
di_1q = 1/L_1*(0.5*eta_q*v_dc - R_1*i_1q - L_1*omega*i_1d - v_cq)
dv_cd = 1/C_ac*( i_1d + C_ac*omega*v_cq - i_2d)
dv_cq = 1/C_ac*( i_1q - C_ac*omega*v_cd - i_2q)
di_2d = 1/L_2*(v_cd - R_2*i_2d + L_2*omega*i_2q - v_sd)
di_2q = 1/L_2*(v_cq - R_2*i_2q - L_2*omega*i_2d - v_sq)

'''
'''

```

    '    \n'

```python
s = sym.solve([ di_1d,  di_1q,  dv_cd, dv_cq, di_2d,  di_2q],
              [  i_1d,   i_1q,   v_cd,  v_cq,  i_2d,   i_2q])

for item in s:
    print(item, '=', sym.simplify(s[item]))
```

    i_1d = (0.5*eta_d*v_dc*(C_ac**2*L_2**2*R_1*omega**4 + C_ac**2*R_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1*omega**2 + R_1 + R_2) + 0.5*eta_q*omega*v_dc*(C_ac**2*L_1*L_2**2*omega**4 + C_ac**2*L_1*R_2**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*L_2**2*omega**2 - C_ac*R_2**2 + L_1 + L_2) - omega*v_sq*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) + v_sd*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
    i_1q = (-0.5*eta_d*omega*v_dc*(C_ac**2*L_1*L_2**2*omega**4 + C_ac**2*L_1*R_2**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*L_2**2*omega**2 - C_ac*R_2**2 + L_1 + L_2) + 0.5*eta_q*v_dc*(C_ac**2*L_2**2*R_1*omega**4 + C_ac**2*R_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1*omega**2 + R_1 + R_2) + omega*v_sd*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) + v_sq*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
    v_cd = (0.5*eta_d*v_dc*(-C_ac*L_1*L_2**2*omega**4 - C_ac*L_1*R_2**2*omega**2 + L_1*L_2*omega**2 + L_2**2*omega**2 + R_1*R_2 + R_2**2) + 0.5*eta_q*omega*v_dc*(C_ac*L_2**2*R_1*omega**2 + C_ac*R_1*R_2**2 + L_1*R_2 - L_2*R_1) + omega*v_sq*(C_ac*L_1**2*R_2*omega**2 + C_ac*R_1**2*R_2 - L_1*R_2 + L_2*R_1) + v_sd*(-C_ac*L_1**2*L_2*omega**4 - C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + L_1*L_2*omega**2 + R_1**2 + R_1*R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
    v_cq = (-0.5*eta_d*omega*v_dc*(C_ac*L_2**2*R_1*omega**2 + C_ac*R_1*R_2**2 + L_1*R_2 - L_2*R_1) + 0.5*eta_q*v_dc*(-C_ac*L_1*L_2**2*omega**4 - C_ac*L_1*R_2**2*omega**2 + L_1*L_2*omega**2 + L_2**2*omega**2 + R_1*R_2 + R_2**2) - omega*v_sd*(C_ac*L_1**2*R_2*omega**2 + C_ac*R_1**2*R_2 - L_1*R_2 + L_2*R_1) + v_sq*(-C_ac*L_1**2*L_2*omega**4 - C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + L_1*L_2*omega**2 + R_1**2 + R_1*R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
    i_2d = (-0.5*eta_d*v_dc*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2) + 0.5*eta_q*omega*v_dc*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) - omega*v_sq*(C_ac**2*L_1**2*L_2*omega**4 + C_ac**2*L_2*R_1**2*omega**2 - C_ac*L_1**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*R_1**2 + L_1 + L_2) - v_sd*(C_ac**2*L_1**2*R_2*omega**4 + C_ac**2*R_1**2*R_2*omega**2 - 2.0*C_ac*L_1*R_2*omega**2 + R_1 + R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
    i_2q = (-0.5*eta_d*omega*v_dc*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) - 0.5*eta_q*v_dc*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2) + omega*v_sd*(C_ac**2*L_1**2*L_2*omega**4 + C_ac**2*L_2*R_1**2*omega**2 - C_ac*L_1**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*R_1**2 + L_1 + L_2) - v_sq*(C_ac**2*L_1**2*R_2*omega**4 + C_ac**2*R_1**2*R_2*omega**2 - 2.0*C_ac*L_1*R_2*omega**2 + R_1 + R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)

```python
s = sym.solve([ di_1d,  di_1q,  dv_cd, dv_cq, di_2d,  di_2q],
              [ eta_d,  eta_q,   i_1d,  i_1q,  v_cd,   v_cq])

for item in s:
    print(item, '=', sym.simplify(s[item]))
```

    eta_d = 2.0*(-C_ac*R_1*omega*(L_2*i_2d*omega + R_2*i_2q + v_sq) - L_1*i_2q*omega + R_1*i_2d - (C_ac*L_1*omega**2 - 1.0)*(-L_2*i_2q*omega + R_2*i_2d + v_sd))/v_dc
    eta_q = 2.0*(C_ac*R_1*omega*(-L_2*i_2q*omega + R_2*i_2d + v_sd) + L_1*i_2d*omega + R_1*i_2q - (C_ac*L_1*omega**2 - 1.0)*(L_2*i_2d*omega + R_2*i_2q + v_sq))/v_dc
    i_1d = -C_ac*omega*(L_2*i_2d*omega + R_2*i_2q + v_sq) + i_2d
    i_1q = C_ac*omega*(-L_2*i_2q*omega + R_2*i_2d + v_sd) + i_2q
    v_cd = -L_2*i_2q*omega + R_2*i_2d + v_sd
    v_cq = L_2*i_2d*omega + R_2*i_2q + v_sq

```python
s = sym.solve([ dv_cd, dv_cq, di_2d,  di_2q],
              [  v_cd,  v_cq,  i_2d,   i_2q])

for item in s:
    print(item, '=', sym.simplify(s[item]))
```

    v_cd = (C_ac*R_2*omega*v_sq + R_2*i_1d + i_1q*omega*(C_ac*L_2**2*omega**2 + C_ac*R_2**2 - L_2) - v_sd*(C_ac*L_2*omega**2 - 1))/(C_ac**2*L_2**2*omega**4 + C_ac**2*R_2**2*omega**2 - 2*C_ac*L_2*omega**2 + 1)
    v_cq = (-C_ac*R_2*omega*v_sd + R_2*i_1q - i_1d*omega*(C_ac*L_2**2*omega**2 + C_ac*R_2**2 - L_2) - v_sq*(C_ac*L_2*omega**2 - 1))/(C_ac**2*L_2**2*omega**4 + C_ac**2*R_2**2*omega**2 - 2*C_ac*L_2*omega**2 + 1)
    i_2d = (-C_ac**2*R_2*omega**2*v_sd + C_ac*R_2*i_1q*omega - C_ac*omega*v_sq*(C_ac*L_2*omega**2 - 1) - i_1d*(C_ac*L_2*omega**2 - 1))/(C_ac**2*L_2**2*omega**4 + C_ac**2*R_2**2*omega**2 - 2*C_ac*L_2*omega**2 + 1)
    i_2q = (-C_ac**2*R_2*omega**2*v_sq - C_ac*R_2*i_1d*omega + C_ac*omega*v_sd*(C_ac*L_2*omega**2 - 1) - i_1q*(C_ac*L_2*omega**2 - 1))/(C_ac**2*L_2**2*omega**4 + C_ac**2*R_2**2*omega**2 - 2*C_ac*L_2*omega**2 + 1)

```python
s = sym.solve([ di_1d,  di_1q],
              [ i_1d,    i_1q])

for item in s:
    print(item, '=', sym.simplify(s[item]))
```

    i_1d = 0.5*(L_1*omega*(eta_q*v_dc - 2.0*v_cq) + R_1*(eta_d*v_dc - 2.0*v_cd))/(L_1**2*omega**2 + R_1**2)
    i_1q = 0.5*(-L_1*omega*(eta_d*v_dc - 2.0*v_cd) + R_1*(eta_q*v_dc - 2.0*v_cq))/(L_1**2*omega**2 + R_1**2)

```python
eq_p = p_ref - 3.0/2.0*(v_sd*i_2d + v_sq*i_2q)
eq_q = q_ref - 3.0/2.0*(v_sd*i_2q - v_sq*i_2d)

s = sym.solve([ eq_p,  eq_q],
              [i_2d,   i_2q])

for item in s:
    print(item, '=', sym.simplify(s[item]))
```

    i_2d = 0.666666666666667*(p_ref*v_sd - q_ref*v_sq)/(v_sd**2 + v_sq**2)
    i_2q = 0.666666666666667*(p_ref*v_sq + q_ref*v_sd)/(v_sd**2 + v_sq**2)

```python
v_dc = 700.0
v_sq = 325.0
v_sd = 0.0
p_ref = 10.0e3
q_ref = 900.0e3
omega = 2.0*np.pi*50.0
L_1 = 200.0e-6
L_2 = 200.0e-6
R_1 = 0.1
R_2 = 0.1
C_ac = 100e-6

i_2d_ref = 2.0/3.0*(p_ref*v_sd - q_ref*v_sq)/(v_sd**2 + v_sq**2)
i_2q_ref = 2.0/3.0*(p_ref*v_sq + q_ref*v_sd)/(v_sd**2 + v_sq**2)

eta_d = 2.0*(-C_ac*R_1*omega*(L_2*i_2d_ref*omega + R_2*i_2q_ref + v_sq) - L_1*i_2q_ref*omega + R_1*i_2d_ref - (C_ac*L_1*omega**2 - 1.0)*(-L_2*i_2q_ref*omega + R_2*i_2d_ref + v_sd))/v_dc
eta_q = 2.0*(C_ac*R_1*omega*(-L_2*i_2q_ref*omega + R_2*i_2d_ref + v_sd) + L_1*i_2d_ref*omega + R_1*i_2q_ref - (C_ac*L_1*omega**2 - 1.0)*(L_2*i_2d_ref*omega + R_2*i_2q_ref + v_sq))/v_dc
print(i_2d_ref)
print(i_2q_ref)

print('eta_d = ',eta_d)
print(eta_q)

eta = eta_d + 1j*eta_q
angle = np.angle(eta)
eta_m = np.abs(eta)

if eta_m > 1.0:
    eta_m = 1.0

eta = eta_m*np.exp(1j*angle)
eta_d = eta.real
eta_q = eta.imag
i_2d = (-0.5*eta_d*v_dc*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2) + 0.5*eta_q*omega*v_dc*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) - omega*v_sq*(C_ac**2*L_1**2*L_2*omega**4 + C_ac**2*L_2*R_1**2*omega**2 - C_ac*L_1**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*R_1**2 + L_1 + L_2) - v_sd*(C_ac**2*L_1**2*R_2*omega**4 + C_ac**2*R_1**2*R_2*omega**2 - 2.0*C_ac*L_1*R_2*omega**2 + R_1 + R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)
i_2q = (-0.5*eta_d*omega*v_dc*(-C_ac*L_1*L_2*omega**2 + C_ac*R_1*R_2 + L_1 + L_2) - 0.5*eta_q*v_dc*(C_ac*L_1*R_2*omega**2 + C_ac*L_2*R_1*omega**2 - R_1 - R_2) + omega*v_sd*(C_ac**2*L_1**2*L_2*omega**4 + C_ac**2*L_2*R_1**2*omega**2 - C_ac*L_1**2*omega**2 - 2.0*C_ac*L_1*L_2*omega**2 - C_ac*R_1**2 + L_1 + L_2) - v_sq*(C_ac**2*L_1**2*R_2*omega**4 + C_ac**2*R_1**2*R_2*omega**2 - 2.0*C_ac*L_1*R_2*omega**2 + R_1 + R_2))/(C_ac**2*L_1**2*L_2**2*omega**6 + C_ac**2*L_1**2*R_2**2*omega**4 + C_ac**2*L_2**2*R_1**2*omega**4 + C_ac**2*R_1**2*R_2**2*omega**2 - 2.0*C_ac*L_1**2*L_2*omega**4 - 2.0*C_ac*L_1*L_2**2*omega**4 - 2.0*C_ac*L_1*R_2**2*omega**2 - 2.0*C_ac*L_2*R_1**2*omega**2 + L_1**2*omega**2 + 2.0*L_1*L_2*omega**2 + L_2**2*omega**2 + R_1**2 + 2.0*R_1*R_2 + R_2**2)

print(i_2d)
print(i_2q)

p = 3.0/2.0*(v_sd*i_2d + v_sq*i_2q)
q = 3.0/2.0*(v_sd*i_2q - v_sq*i_2d)

print('p = ', p/1000)
print('q = ', q/1000)

print('p = ', p/1000)
print('q = ', q/1000)

```

    -1846.1538461538462
    20.51282051282051
    eta_d =  -1.063155919300432
    0.2745925438638739
    -1746.44514159
    -85.3469189756
    p =  -41.6066230006
    q =  851.392006525
    p =  -41.6066230006
    q =  851.392006525

```python

```

```python

@numba.jit(nopython=True, cache=True)
def b2b_ctrl1(struct,i,m):
    '''
    Doubly Fed Induction Machine in with neglected dynamics and
    rotor side converter and control level 1 already implemented.
    i_rd = i_rd_ref and i_rq = i_rq_ref without dynamics
    '''

    x_idx = struct[i]['b2b_idx']
    v_dc = float(struct[i]['x'][x_idx+0,0])

    L_1 = struct[i]['L_1']
    L_2 = struct[i]['L_2']
    R_1 = struct[i]['R_1']
    R_2 = struct[i]['R_2']
    C_dc = struct[i]['C_dc']

    omega_1 = struct[i]['omega_1']
    omega_2 = struct[i]['omega_2']

    i_1d_ref = struct[i]['i_1d_ref']
    i_1q_ref = struct[i]['i_1q_ref']
    i_2d_ref = struct[i]['i_2d_ref']
    i_2q_ref = struct[i]['i_2q_ref']

    i_1d = i_1d_ref
    i_1q = i_1q_ref
    i_2d = i_2d_ref
    i_2q = i_2q_ref

    v_1d = struct[i]['v_1d']
    v_1q = struct[i]['v_1q']
    v_2d = struct[i]['v_2d']
    v_2q = struct[i]['v_2q']

    eta_1d = 2.0*(R_1*i_1d - L_1*i_1q*omega_1 + v_1d)/v_dc
    eta_1q = 2.0*(R_1*i_1q + L_1*i_1d*omega_1 + v_1q)/v_dc
    eta_2d = 2.0*(R_2*i_2d - L_2*i_2q*omega_2 + v_2d)/v_dc
    eta_2q = 2.0*(R_2*i_2q + L_2*i_2d*omega_2 + v_2q)/v_dc

    i_dc_1 = 3.0/4.0*(eta_1d*i_1d + eta_1q*i_1q)
    i_dc_2 = 3.0/4.0*(eta_2d*i_2d + eta_2q*i_2q)
    dv_dc = 1.0/C_dc*(-i_dc_1 - i_dc_2)

    struct[i]['eta_1d'] = eta_1d
    struct[i]['eta_1q'] = eta_1q
    struct[i]['eta_2d'] = eta_2d
    struct[i]['eta_2q'] = eta_2q
    struct[i]['i_dc_1'] = i_dc_1
    struct[i]['i_dc_2'] = i_dc_2

    struct[i]['p_1'] = 3.0/2.0*(v_1d*i_1d + v_1q*i_1q)
    struct[i]['q_1'] = 3.0/2.0*(v_1d*i_1q - v_1q*i_1d)

    struct[i]['p_2'] = 3.0/2.0*(v_2d*i_2d + v_2q*i_2q)
    struct[i]['q_2'] = 3.0/2.0*(v_2d*i_2q - v_2q*i_2d)

    struct[i]['f'][x_idx+0,0] = dv_dc

    return 0

```

```python

```

```python
@numba.jit(nopython=True, cache=True)
def b2b_ctrl2(struct,i,m):
    '''
    Control level 2 for DC Voltage

    '''

    x_idx = struct[i]['b2b_ctrl_idx']
    xi_v_dc = float(struct[i]['x'][x_idx+0,0])

    S_b = struct[i]['S_b']
    V_dc_b = struct[i]['V_dc_b']

    K_v_p = struct[i]['K_v_p']
    K_v_i = struct[i]['K_v_i']

    v_dc = struct[i]['v_dc']

    v_dc_ref = struct[i]['v_dc_ref']
    p_1_ref = struct[i]['p_1_ref']
    q_1_ref = struct[i]['q_1_ref']
    p_2_ref = struct[i]['p_2_ref']
    q_2_ref = struct[i]['q_2_ref']

    v_1d = struct[i]['v_1d']
    v_1q = struct[i]['v_1q']
    v_2d = struct[i]['v_2d']
    v_2q = struct[i]['v_2q']

    error_v_dc = (v_dc - v_dc_ref)/V_dc_b

    p_ref = (K_v_p * error_v_dc + K_v_i*xi_v_dc)*S_b

    if struct[i]['vdc_ctrl'] == 1:
        p_ref_1 = p_ref

    if struct[i]['vdc_ctrl'] == 2:
        p_ref_2 = p_ref

    den = (v_1d**2 + v_1q**2)

    den_1 = 0.001
    if den_1 > 0.0:
        den_1 = (v_1d**2 + v_1q**2)

    den_2 = 0.001
    if den_2 > 0.0:
        den_2 = (v_2d**2 + v_2q**2)

    i_1d_ref = 2.0/3.0*(p_1_ref*v_1d - q_1_ref*v_1q)/den_1
    i_1q_ref = 2.0/3.0*(p_1_ref*v_1q + q_1_ref*v_1d)/den_1

    i_2d_ref = 2.0/3.0*(p_2_ref*v_2d - q_2_ref*v_1q)/den_2
    i_2q_ref = 2.0/3.0*(p_2_ref*v_2q + q_2_ref*v_1d)/den_2

    struct[i]['i_1d_ref'] = i_1d_ref
    struct[i]['i_1q_ref'] = i_1q_ref
    struct[i]['i_2d_ref'] = i_2d_ref
    struct[i]['i_2q_ref'] = i_2q_ref

    dxi_v_dc = error_v_dc

    struct[i]['f'][x_idx+0,0] = dxi_v_dc

    return 0

```

```python

```

```python
R_1 = R_2 = 0.1
L_1 = L_2 = 0.5e-3
Omega_b = 2.0*np.pi*50.0
C_dc = 2200.0e-6
omega_1 = omega_2 = Omega_b

d =dict(R_1 = R_1,
        R_2 = R_2,
        L_1 = L_1,
        L_2 = L_2,
        C_dc = C_dc,
        b2b_idx = 0,
        b2b_ctrl_idx = 1,
        v_dc = 800.0,
        omega_1 = omega_1,
        omega_2 = omega_2,
        i_1d_ref = 0.0,
        i_1q_ref = 100.0,
        i_2d_ref = 0.0,
        i_2q_ref = -100.0,
        i_dc_1 = 0.0,
        i_dc_2 = 0.0,
        eta_1d = 0.0,
        eta_1q = 0.0,
        eta_2d = 0.0,
        eta_2q = 0.0,
        v_1d = 0.0,
        v_1q = 325.0,
        v_2d = 0.0,
        v_2q = 325.0,
        p_1 = 0.0,
        q_1 = 0.0,
        p_2 = 0.0,
        q_2 = 0.0,
        x_idx = 0,
        xi_v_dc = 0.0,
        S_b = 0.5e6,
        V_dc_b = 800.0,
        K_v_p = 0.1,
        K_v_i = 0.0,
        v_dc_ref = 750.0,
        p_1_ref = 0.0,
        q_1_ref = 0.0,
        p_2_ref = 0.0,
        q_2_ref = 0.0,
        vdc_ctrl = 1,
        x = np.array([[800.0],[0.0]]),
        f = np.array([[0.0],[0.0]])
        )

struct = d2np(d)

i=0
m=2
b2b_ctrl1(struct,i,m)
b2b_ctrl2(struct,i,m)
print(struct[i]['p_1'])
print(struct[i]['p_2'])
print(struct[i]['i_dc_1'])
print(struct[i]['i_dc_2'])
print(struct[i]['f'])
```

    48750.0
    -48750.0
    62.8125
    -59.0625
    [[ -1.70454545e+03]
     [  6.25000000e-02]]

```python
struct = d2np(d)

sys_d = dict(x = np.array([[800.0],[0.0]]),
             f = np.zeros((2,1)))

sys_struct = d2np(sys_d)

@numba.jit(nopython=True, cache=True)
def f_eval(sys_struct,struct):
    N_states = 2
    for i in range(1):
        struct[i]['x'][:,0] = sys_struct[0]['x'][N_states*i:N_states*(i+1),0]
        b2b_ctrl1(struct,i,m)
        b2b_ctrl2(struct,i,m)
        sys_struct[0]['f'][N_states*i:N_states*(i+1),:] = struct[i]['f']
    return 0

```

```python
@numba.jit(nopython=True, cache=True)
def run(sys_struct,struct):
    N_steps = 1000
    N_states = 2

    Dt = 10.0e-3
    Omega_r = np.zeros((N_steps,1))
    Omega_t = np.zeros((N_steps,1))
    P_1 = np.zeros((N_steps,1))
    Q_1 = np.zeros((N_steps,1))
    P_2 = np.zeros((N_steps,1))
    Q_2 = np.zeros((N_steps,1))
    V_dr = np.zeros((N_steps,1))
    V_qr = np.zeros((N_steps,1))
    I_dr = np.zeros((N_steps,1))
    I_qr = np.zeros((N_steps,1))
    Tau_e = np.zeros((N_steps,1))
    T     = np.zeros((N_steps,1))
    X = np.zeros((N_steps,N_states))

    V_dc = np.zeros((N_steps,1))
    p_ref = 0.0
    q_ref = 0.0
    xi_p = 0.0
    xi_q = 0.0

    struct[0]['x'][:,0] = np.copy(sys_struct[0]['x'][0:2,0])

    for it in range(N_steps):
        t = Dt*float(it)

        # perturbations and references

        struct[0]['p_1_ref'] = 0.0
        struct[0]['p_2_ref'] = 0.0
        struct[0]['q_1_ref'] = 0.0
        struct[0]['q_2_ref'] = 0.0

        if t>2.0:
            struct[0]['p_1_ref'] = 1.0e6
        if t>3.0:
            struct[0]['p_2_ref'] = 0.1e6

        ## solver

        f_eval(sys_struct,struct)
        f1 = np.copy(sys_struct[0]['f'])
        x1 = np.copy(sys_struct[0]['x'])

        sys_struct[0]['x'][:]= np.copy(x1 + Dt*f1)
        f_eval(sys_struct,struct)
        f2 = np.copy(sys_struct[0]['f'])

        sys_struct[0]['x'][:]= np.copy(x1 + 0.5*Dt*(f1 + f2))

        for i in range(1):
            struct[i]['x'][:,0] = sys_struct[0]['x'][2*i:2*(i+1),0]

        T[it,0] = t
        V_dc[it,0] = float(struct[0]['v_dc'])
        X[it,:] = sys_struct[0]['x'][:].T
    return T,X,V_dc
%timeit run(sys_struct, struct)
```

    724 µs ± 41.2 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

```python
run(sys_struct, struct)
```

    (array([[ 0.  ],
            [ 0.01],
            [ 0.02],
            [ 0.03],
            [ 0.04],
            [ 0.05],
            [ 0.06],
            [ 0.07],
            [ 0.08],
            [ 0.09],
            [ 0.1 ],
            [ 0.11],
            [ 0.12],
            [ 0.13],
            [ 0.14],
            [ 0.15],
            [ 0.16],
            [ 0.17],
            [ 0.18],
            [ 0.19],
            [ 0.2 ],
            [ 0.21],
            [ 0.22],
            [ 0.23],
            [ 0.24],
            [ 0.25],
            [ 0.26],
            [ 0.27],
            [ 0.28],
            [ 0.29],
            [ 0.3 ],
            [ 0.31],
            [ 0.32],
            [ 0.33],
            [ 0.34],
            [ 0.35],
            [ 0.36],
            [ 0.37],
            [ 0.38],
            [ 0.39],
            [ 0.4 ],
            [ 0.41],
            [ 0.42],
            [ 0.43],
            [ 0.44],
            [ 0.45],
            [ 0.46],
            [ 0.47],
            [ 0.48],
            [ 0.49],
            [ 0.5 ],
            [ 0.51],
            [ 0.52],
            [ 0.53],
            [ 0.54],
            [ 0.55],
            [ 0.56],
            [ 0.57],
            [ 0.58],
            [ 0.59],
            [ 0.6 ],
            [ 0.61],
            [ 0.62],
            [ 0.63],
            [ 0.64],
            [ 0.65],
            [ 0.66],
            [ 0.67],
            [ 0.68],
            [ 0.69],
            [ 0.7 ],
            [ 0.71],
            [ 0.72],
            [ 0.73],
            [ 0.74],
            [ 0.75],
            [ 0.76],
            [ 0.77],
            [ 0.78],
            [ 0.79],
            [ 0.8 ],
            [ 0.81],
            [ 0.82],
            [ 0.83],
            [ 0.84],
            [ 0.85],
            [ 0.86],
            [ 0.87],
            [ 0.88],
            [ 0.89],
            [ 0.9 ],
            [ 0.91],
            [ 0.92],
            [ 0.93],
            [ 0.94],
            [ 0.95],
            [ 0.96],
            [ 0.97],
            [ 0.98],
            [ 0.99],
            [ 1.  ],
            [ 1.01],
            [ 1.02],
            [ 1.03],
            [ 1.04],
            [ 1.05],
            [ 1.06],
            [ 1.07],
            [ 1.08],
            [ 1.09],
            [ 1.1 ],
            [ 1.11],
            [ 1.12],
            [ 1.13],
            [ 1.14],
            [ 1.15],
            [ 1.16],
            [ 1.17],
            [ 1.18],
            [ 1.19],
            [ 1.2 ],
            [ 1.21],
            [ 1.22],
            [ 1.23],
            [ 1.24],
            [ 1.25],
            [ 1.26],
            [ 1.27],
            [ 1.28],
            [ 1.29],
            [ 1.3 ],
            [ 1.31],
            [ 1.32],
            [ 1.33],
            [ 1.34],
            [ 1.35],
            [ 1.36],
            [ 1.37],
            [ 1.38],
            [ 1.39],
            [ 1.4 ],
            [ 1.41],
            [ 1.42],
            [ 1.43],
            [ 1.44],
            [ 1.45],
            [ 1.46],
            [ 1.47],
            [ 1.48],
            [ 1.49],
            [ 1.5 ],
            [ 1.51],
            [ 1.52],
            [ 1.53],
            [ 1.54],
            [ 1.55],
            [ 1.56],
            [ 1.57],
            [ 1.58],
            [ 1.59],
            [ 1.6 ],
            [ 1.61],
            [ 1.62],
            [ 1.63],
            [ 1.64],
            [ 1.65],
            [ 1.66],
            [ 1.67],
            [ 1.68],
            [ 1.69],
            [ 1.7 ],
            [ 1.71],
            [ 1.72],
            [ 1.73],
            [ 1.74],
            [ 1.75],
            [ 1.76],
            [ 1.77],
            [ 1.78],
            [ 1.79],
            [ 1.8 ],
            [ 1.81],
            [ 1.82],
            [ 1.83],
            [ 1.84],
            [ 1.85],
            [ 1.86],
            [ 1.87],
            [ 1.88],
            [ 1.89],
            [ 1.9 ],
            [ 1.91],
            [ 1.92],
            [ 1.93],
            [ 1.94],
            [ 1.95],
            [ 1.96],
            [ 1.97],
            [ 1.98],
            [ 1.99],
            [ 2.  ],
            [ 2.01],
            [ 2.02],
            [ 2.03],
            [ 2.04],
            [ 2.05],
            [ 2.06],
            [ 2.07],
            [ 2.08],
            [ 2.09],
            [ 2.1 ],
            [ 2.11],
            [ 2.12],
            [ 2.13],
            [ 2.14],
            [ 2.15],
            [ 2.16],
            [ 2.17],
            [ 2.18],
            [ 2.19],
            [ 2.2 ],
            [ 2.21],
            [ 2.22],
            [ 2.23],
            [ 2.24],
            [ 2.25],
            [ 2.26],
            [ 2.27],
            [ 2.28],
            [ 2.29],
            [ 2.3 ],
            [ 2.31],
            [ 2.32],
            [ 2.33],
            [ 2.34],
            [ 2.35],
            [ 2.36],
            [ 2.37],
            [ 2.38],
            [ 2.39],
            [ 2.4 ],
            [ 2.41],
            [ 2.42],
            [ 2.43],
            [ 2.44],
            [ 2.45],
            [ 2.46],
            [ 2.47],
            [ 2.48],
            [ 2.49],
            [ 2.5 ],
            [ 2.51],
            [ 2.52],
            [ 2.53],
            [ 2.54],
            [ 2.55],
            [ 2.56],
            [ 2.57],
            [ 2.58],
            [ 2.59],
            [ 2.6 ],
            [ 2.61],
            [ 2.62],
            [ 2.63],
            [ 2.64],
            [ 2.65],
            [ 2.66],
            [ 2.67],
            [ 2.68],
            [ 2.69],
            [ 2.7 ],
            [ 2.71],
            [ 2.72],
            [ 2.73],
            [ 2.74],
            [ 2.75],
            [ 2.76],
            [ 2.77],
            [ 2.78],
            [ 2.79],
            [ 2.8 ],
            [ 2.81],
            [ 2.82],
            [ 2.83],
            [ 2.84],
            [ 2.85],
            [ 2.86],
            [ 2.87],
            [ 2.88],
            [ 2.89],
            [ 2.9 ],
            [ 2.91],
            [ 2.92],
            [ 2.93],
            [ 2.94],
            [ 2.95],
            [ 2.96],
            [ 2.97],
            [ 2.98],
            [ 2.99],
            [ 3.  ],
            [ 3.01],
            [ 3.02],
            [ 3.03],
            [ 3.04],
            [ 3.05],
            [ 3.06],
            [ 3.07],
            [ 3.08],
            [ 3.09],
            [ 3.1 ],
            [ 3.11],
            [ 3.12],
            [ 3.13],
            [ 3.14],
            [ 3.15],
            [ 3.16],
            [ 3.17],
            [ 3.18],
            [ 3.19],
            [ 3.2 ],
            [ 3.21],
            [ 3.22],
            [ 3.23],
            [ 3.24],
            [ 3.25],
            [ 3.26],
            [ 3.27],
            [ 3.28],
            [ 3.29],
            [ 3.3 ],
            [ 3.31],
            [ 3.32],
            [ 3.33],
            [ 3.34],
            [ 3.35],
            [ 3.36],
            [ 3.37],
            [ 3.38],
            [ 3.39],
            [ 3.4 ],
            [ 3.41],
            [ 3.42],
            [ 3.43],
            [ 3.44],
            [ 3.45],
            [ 3.46],
            [ 3.47],
            [ 3.48],
            [ 3.49],
            [ 3.5 ],
            [ 3.51],
            [ 3.52],
            [ 3.53],
            [ 3.54],
            [ 3.55],
            [ 3.56],
            [ 3.57],
            [ 3.58],
            [ 3.59],
            [ 3.6 ],
            [ 3.61],
            [ 3.62],
            [ 3.63],
            [ 3.64],
            [ 3.65],
            [ 3.66],
            [ 3.67],
            [ 3.68],
            [ 3.69],
            [ 3.7 ],
            [ 3.71],
            [ 3.72],
            [ 3.73],
            [ 3.74],
            [ 3.75],
            [ 3.76],
            [ 3.77],
            [ 3.78],
            [ 3.79],
            [ 3.8 ],
            [ 3.81],
            [ 3.82],
            [ 3.83],
            [ 3.84],
            [ 3.85],
            [ 3.86],
            [ 3.87],
            [ 3.88],
            [ 3.89],
            [ 3.9 ],
            [ 3.91],
            [ 3.92],
            [ 3.93],
            [ 3.94],
            [ 3.95],
            [ 3.96],
            [ 3.97],
            [ 3.98],
            [ 3.99],
            [ 4.  ],
            [ 4.01],
            [ 4.02],
            [ 4.03],
            [ 4.04],
            [ 4.05],
            [ 4.06],
            [ 4.07],
            [ 4.08],
            [ 4.09],
            [ 4.1 ],
            [ 4.11],
            [ 4.12],
            [ 4.13],
            [ 4.14],
            [ 4.15],
            [ 4.16],
            [ 4.17],
            [ 4.18],
            [ 4.19],
            [ 4.2 ],
            [ 4.21],
            [ 4.22],
            [ 4.23],
            [ 4.24],
            [ 4.25],
            [ 4.26],
            [ 4.27],
            [ 4.28],
            [ 4.29],
            [ 4.3 ],
            [ 4.31],
            [ 4.32],
            [ 4.33],
            [ 4.34],
            [ 4.35],
            [ 4.36],
            [ 4.37],
            [ 4.38],
            [ 4.39],
            [ 4.4 ],
            [ 4.41],
            [ 4.42],
            [ 4.43],
            [ 4.44],
            [ 4.45],
            [ 4.46],
            [ 4.47],
            [ 4.48],
            [ 4.49],
            [ 4.5 ],
            [ 4.51],
            [ 4.52],
            [ 4.53],
            [ 4.54],
            [ 4.55],
            [ 4.56],
            [ 4.57],
            [ 4.58],
            [ 4.59],
            [ 4.6 ],
            [ 4.61],
            [ 4.62],
            [ 4.63],
            [ 4.64],
            [ 4.65],
            [ 4.66],
            [ 4.67],
            [ 4.68],
            [ 4.69],
            [ 4.7 ],
            [ 4.71],
            [ 4.72],
            [ 4.73],
            [ 4.74],
            [ 4.75],
            [ 4.76],
            [ 4.77],
            [ 4.78],
            [ 4.79],
            [ 4.8 ],
            [ 4.81],
            [ 4.82],
            [ 4.83],
            [ 4.84],
            [ 4.85],
            [ 4.86],
            [ 4.87],
            [ 4.88],
            [ 4.89],
            [ 4.9 ],
            [ 4.91],
            [ 4.92],
            [ 4.93],
            [ 4.94],
            [ 4.95],
            [ 4.96],
            [ 4.97],
            [ 4.98],
            [ 4.99],
            [ 5.  ],
            [ 5.01],
            [ 5.02],
            [ 5.03],
            [ 5.04],
            [ 5.05],
            [ 5.06],
            [ 5.07],
            [ 5.08],
            [ 5.09],
            [ 5.1 ],
            [ 5.11],
            [ 5.12],
            [ 5.13],
            [ 5.14],
            [ 5.15],
            [ 5.16],
            [ 5.17],
            [ 5.18],
            [ 5.19],
            [ 5.2 ],
            [ 5.21],
            [ 5.22],
            [ 5.23],
            [ 5.24],
            [ 5.25],
            [ 5.26],
            [ 5.27],
            [ 5.28],
            [ 5.29],
            [ 5.3 ],
            [ 5.31],
            [ 5.32],
            [ 5.33],
            [ 5.34],
            [ 5.35],
            [ 5.36],
            [ 5.37],
            [ 5.38],
            [ 5.39],
            [ 5.4 ],
            [ 5.41],
            [ 5.42],
            [ 5.43],
            [ 5.44],
            [ 5.45],
            [ 5.46],
            [ 5.47],
            [ 5.48],
            [ 5.49],
            [ 5.5 ],
            [ 5.51],
            [ 5.52],
            [ 5.53],
            [ 5.54],
            [ 5.55],
            [ 5.56],
            [ 5.57],
            [ 5.58],
            [ 5.59],
            [ 5.6 ],
            [ 5.61],
            [ 5.62],
            [ 5.63],
            [ 5.64],
            [ 5.65],
            [ 5.66],
            [ 5.67],
            [ 5.68],
            [ 5.69],
            [ 5.7 ],
            [ 5.71],
            [ 5.72],
            [ 5.73],
            [ 5.74],
            [ 5.75],
            [ 5.76],
            [ 5.77],
            [ 5.78],
            [ 5.79],
            [ 5.8 ],
            [ 5.81],
            [ 5.82],
            [ 5.83],
            [ 5.84],
            [ 5.85],
            [ 5.86],
            [ 5.87],
            [ 5.88],
            [ 5.89],
            [ 5.9 ],
            [ 5.91],
            [ 5.92],
            [ 5.93],
            [ 5.94],
            [ 5.95],
            [ 5.96],
            [ 5.97],
            [ 5.98],
            [ 5.99],
            [ 6.  ],
            [ 6.01],
            [ 6.02],
            [ 6.03],
            [ 6.04],
            [ 6.05],
            [ 6.06],
            [ 6.07],
            [ 6.08],
            [ 6.09],
            [ 6.1 ],
            [ 6.11],
            [ 6.12],
            [ 6.13],
            [ 6.14],
            [ 6.15],
            [ 6.16],
            [ 6.17],
            [ 6.18],
            [ 6.19],
            [ 6.2 ],
            [ 6.21],
            [ 6.22],
            [ 6.23],
            [ 6.24],
            [ 6.25],
            [ 6.26],
            [ 6.27],
            [ 6.28],
            [ 6.29],
            [ 6.3 ],
            [ 6.31],
            [ 6.32],
            [ 6.33],
            [ 6.34],
            [ 6.35],
            [ 6.36],
            [ 6.37],
            [ 6.38],
            [ 6.39],
            [ 6.4 ],
            [ 6.41],
            [ 6.42],
            [ 6.43],
            [ 6.44],
            [ 6.45],
            [ 6.46],
            [ 6.47],
            [ 6.48],
            [ 6.49],
            [ 6.5 ],
            [ 6.51],
            [ 6.52],
            [ 6.53],
            [ 6.54],
            [ 6.55],
            [ 6.56],
            [ 6.57],
            [ 6.58],
            [ 6.59],
            [ 6.6 ],
            [ 6.61],
            [ 6.62],
            [ 6.63],
            [ 6.64],
            [ 6.65],
            [ 6.66],
            [ 6.67],
            [ 6.68],
            [ 6.69],
            [ 6.7 ],
            [ 6.71],
            [ 6.72],
            [ 6.73],
            [ 6.74],
            [ 6.75],
            [ 6.76],
            [ 6.77],
            [ 6.78],
            [ 6.79],
            [ 6.8 ],
            [ 6.81],
            [ 6.82],
            [ 6.83],
            [ 6.84],
            [ 6.85],
            [ 6.86],
            [ 6.87],
            [ 6.88],
            [ 6.89],
            [ 6.9 ],
            [ 6.91],
            [ 6.92],
            [ 6.93],
            [ 6.94],
            [ 6.95],
            [ 6.96],
            [ 6.97],
            [ 6.98],
            [ 6.99],
            [ 7.  ],
            [ 7.01],
            [ 7.02],
            [ 7.03],
            [ 7.04],
            [ 7.05],
            [ 7.06],
            [ 7.07],
            [ 7.08],
            [ 7.09],
            [ 7.1 ],
            [ 7.11],
            [ 7.12],
            [ 7.13],
            [ 7.14],
            [ 7.15],
            [ 7.16],
            [ 7.17],
            [ 7.18],
            [ 7.19],
            [ 7.2 ],
            [ 7.21],
            [ 7.22],
            [ 7.23],
            [ 7.24],
            [ 7.25],
            [ 7.26],
            [ 7.27],
            [ 7.28],
            [ 7.29],
            [ 7.3 ],
            [ 7.31],
            [ 7.32],
            [ 7.33],
            [ 7.34],
            [ 7.35],
            [ 7.36],
            [ 7.37],
            [ 7.38],
            [ 7.39],
            [ 7.4 ],
            [ 7.41],
            [ 7.42],
            [ 7.43],
            [ 7.44],
            [ 7.45],
            [ 7.46],
            [ 7.47],
            [ 7.48],
            [ 7.49],
            [ 7.5 ],
            [ 7.51],
            [ 7.52],
            [ 7.53],
            [ 7.54],
            [ 7.55],
            [ 7.56],
            [ 7.57],
            [ 7.58],
            [ 7.59],
            [ 7.6 ],
            [ 7.61],
            [ 7.62],
            [ 7.63],
            [ 7.64],
            [ 7.65],
            [ 7.66],
            [ 7.67],
            [ 7.68],
            [ 7.69],
            [ 7.7 ],
            [ 7.71],
            [ 7.72],
            [ 7.73],
            [ 7.74],
            [ 7.75],
            [ 7.76],
            [ 7.77],
            [ 7.78],
            [ 7.79],
            [ 7.8 ],
            [ 7.81],
            [ 7.82],
            [ 7.83],
            [ 7.84],
            [ 7.85],
            [ 7.86],
            [ 7.87],
            [ 7.88],
            [ 7.89],
            [ 7.9 ],
            [ 7.91],
            [ 7.92],
            [ 7.93],
            [ 7.94],
            [ 7.95],
            [ 7.96],
            [ 7.97],
            [ 7.98],
            [ 7.99],
            [ 8.  ],
            [ 8.01],
            [ 8.02],
            [ 8.03],
            [ 8.04],
            [ 8.05],
            [ 8.06],
            [ 8.07],
            [ 8.08],
            [ 8.09],
            [ 8.1 ],
            [ 8.11],
            [ 8.12],
            [ 8.13],
            [ 8.14],
            [ 8.15],
            [ 8.16],
            [ 8.17],
            [ 8.18],
            [ 8.19],
            [ 8.2 ],
            [ 8.21],
            [ 8.22],
            [ 8.23],
            [ 8.24],
            [ 8.25],
            [ 8.26],
            [ 8.27],
            [ 8.28],
            [ 8.29],
            [ 8.3 ],
            [ 8.31],
            [ 8.32],
            [ 8.33],
            [ 8.34],
            [ 8.35],
            [ 8.36],
            [ 8.37],
            [ 8.38],
            [ 8.39],
            [ 8.4 ],
            [ 8.41],
            [ 8.42],
            [ 8.43],
            [ 8.44],
            [ 8.45],
            [ 8.46],
            [ 8.47],
            [ 8.48],
            [ 8.49],
            [ 8.5 ],
            [ 8.51],
            [ 8.52],
            [ 8.53],
            [ 8.54],
            [ 8.55],
            [ 8.56],
            [ 8.57],
            [ 8.58],
            [ 8.59],
            [ 8.6 ],
            [ 8.61],
            [ 8.62],
            [ 8.63],
            [ 8.64],
            [ 8.65],
            [ 8.66],
            [ 8.67],
            [ 8.68],
            [ 8.69],
            [ 8.7 ],
            [ 8.71],
            [ 8.72],
            [ 8.73],
            [ 8.74],
            [ 8.75],
            [ 8.76],
            [ 8.77],
            [ 8.78],
            [ 8.79],
            [ 8.8 ],
            [ 8.81],
            [ 8.82],
            [ 8.83],
            [ 8.84],
            [ 8.85],
            [ 8.86],
            [ 8.87],
            [ 8.88],
            [ 8.89],
            [ 8.9 ],
            [ 8.91],
            [ 8.92],
            [ 8.93],
            [ 8.94],
            [ 8.95],
            [ 8.96],
            [ 8.97],
            [ 8.98],
            [ 8.99],
            [ 9.  ],
            [ 9.01],
            [ 9.02],
            [ 9.03],
            [ 9.04],
            [ 9.05],
            [ 9.06],
            [ 9.07],
            [ 9.08],
            [ 9.09],
            [ 9.1 ],
            [ 9.11],
            [ 9.12],
            [ 9.13],
            [ 9.14],
            [ 9.15],
            [ 9.16],
            [ 9.17],
            [ 9.18],
            [ 9.19],
            [ 9.2 ],
            [ 9.21],
            [ 9.22],
            [ 9.23],
            [ 9.24],
            [ 9.25],
            [ 9.26],
            [ 9.27],
            [ 9.28],
            [ 9.29],
            [ 9.3 ],
            [ 9.31],
            [ 9.32],
            [ 9.33],
            [ 9.34],
            [ 9.35],
            [ 9.36],
            [ 9.37],
            [ 9.38],
            [ 9.39],
            [ 9.4 ],
            [ 9.41],
            [ 9.42],
            [ 9.43],
            [ 9.44],
            [ 9.45],
            [ 9.46],
            [ 9.47],
            [ 9.48],
            [ 9.49],
            [ 9.5 ],
            [ 9.51],
            [ 9.52],
            [ 9.53],
            [ 9.54],
            [ 9.55],
            [ 9.56],
            [ 9.57],
            [ 9.58],
            [ 9.59],
            [ 9.6 ],
            [ 9.61],
            [ 9.62],
            [ 9.63],
            [ 9.64],
            [ 9.65],
            [ 9.66],
            [ 9.67],
            [ 9.68],
            [ 9.69],
            [ 9.7 ],
            [ 9.71],
            [ 9.72],
            [ 9.73],
            [ 9.74],
            [ 9.75],
            [ 9.76],
            [ 9.77],
            [ 9.78],
            [ 9.79],
            [ 9.8 ],
            [ 9.81],
            [ 9.82],
            [ 9.83],
            [ 9.84],
            [ 9.85],
            [ 9.86],
            [ 9.87],
            [ 9.88],
            [ 9.89],
            [ 9.9 ],
            [ 9.91],
            [ 9.92],
            [ 9.93],
            [ 9.94],
            [ 9.95],
            [ 9.96],
            [ 9.97],
            [ 9.98],
            [ 9.99]]), array([[  1.57952317e+07,   5.06937562e+03],
            [  1.57952317e+07,   5.06937625e+03],
            [  1.57952317e+07,   5.06937687e+03],
            ...,
            [  1.57948365e+07,   5.06999875e+03],
            [  1.57948360e+07,   5.06999937e+03],
            [  1.57948355e+07,   5.07000000e+03]]), array([[ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.],
            [ 800.]]))

```python
sys_struct['x'][:]= np.zeros((6,1))
struct['v_qs'] = 0.0
struct['v_ds'] = 690.0*np.sqrt(2.0/3.0)
struct['tau_t'] = 0.0
sys_struct[0]['x'][0,0] = Omega_b*0.9/struct[0]['N_tr']/struct[0]['N_pp']
sys_struct[0]['x'][3,0] = Omega_b*1.1/struct[1]['N_tr']/struct[0]['N_pp']
T,X,Tau_e,P_s_1,Q_s_1,P_r_1,Q_r_1,P_s_2,Q_s_2,P_r_2,Q_r_2,V_dr,V_qr,Omega_r,Omega_t,I_dr,I_qr = run(sys_struct, struct)
```

```python
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(8, 5), sharex = True)
axes.plot(T,Tau_e)
fig.savefig('dfim_tau_e.svg', bbox_inches='tight')
```

```python
fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), sharex = True)
axes[0].plot(T,P_s_1/1e6, label='$\sf p_{s1}$')
axes[0].plot(T,Q_s_1/1e6, label='$\sf q_{s1}$')
axes[0].plot(T,P_s_2/1e6, label='$\sf p_{s2}$')
axes[0].plot(T,Q_s_2/1e6, label='$\sf q_{s2}$')

axes[1].plot(T,P_r_1/1e6, label='$\sf p_{r1}$')
axes[1].plot(T,Q_r_1/1e6, label='$\sf q_{r1}$')
axes[1].plot(T,P_r_2/1e6, label='$\sf p_{r2}$')
axes[1].plot(T,Q_r_2/1e6, label='$\sf q_{r2}$')

axes[0].legend()
axes[1].legend()
fig.savefig('dfim_tau_e.svg', bbox_inches='tight')

```

```python
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(8, 5), sharex = True)
axes.plot(T,Omega_t)
```

```python
fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), sharex = True)
axes[0].plot(T,V_dr, label='$\sf v_{dr}$')
axes[0].plot(T,V_qr, label='$\sf v_{qr}$')
```

```python
fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), sharex = True)
axes[0].plot(T,Omega_t, label='$\sf v_{dr}$')

```

```python
Omega_t[0]
```

```python
fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), sharex = True)
axes[0].plot(T,I_dr, label='$\sf i_{dr}$')
axes[0].plot(T,I_qr, label='$\sf i_{qr}$')
```

```python
fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), sharex = True)
axes[0].plot(T,X[:,5], label='$\sf x$')

```

```python
np.random.normal(500e3,100e3)
```

```python

```

```python

```

```python

```


# Decoding Board from Data

```python
#necessary imports
import pandas
import numpy as np
import os
import tensorflow.keras as keras
from keras.models import Model
from keras.layers import Dense, Input
from IPython.display import display
import sympy as sp
sp.init_printing(use_latex = True)
import math

import matplotlib.pyplot as plt
%matplotlib inline

%run Drivers.ipynb

EMPTY = 1;
COLOR = 0;

BLACK = -1;
WHITE = 1;

WIDTH = 9;
```

# Data Categorization and Assignment

```python
Boards = []
Moves = []
def Main():
    path = "./go9"
    for entry in os.scandir(path): #I changed my mind i love python
        Go = True
        Board = createEmptyBoard() # 0 - 80 = [color, empty], 81 = [turn, turn]
        with open(entry) as f:
            if Go:
                for line in f:
                    if line[0] == ';': # this is the line with all the moves.
                        Go = False
                        copy = ""
                        for c in line:
                            if c != "[" and c != "]" and c != ")":
                                copy += c
                        arr = copy[1:].split(';')
                        for a in arr:
                            move = Decode_Move(a[1:])
                            if Decode_Move(a[1:]) == -1:
                                print(entry)
                                return
                            else:
                                color = 1
                                if(a[0] == 'B'):
                                    color = -1
                                Boards.append(Board)
                                Moves.append(move)
                                if(move != 81):
                                    Board = Move(Board, move, color)[1]
Main()
Boards = np.array(Boards)
Moves = np.array(Moves)
```

```python
print(Boards.shape)
```

    (414124, 82, 2)

```python
# Example Position:
printBoard(Boards[25])
```

    #  A B C D E F G H I
    1  . . . . . . . . .
    2  . . . . @ . . . .
    3  . . O . @ O O . .
    4  . . . O O @ . @ .
    5  . . @ @ @ @ @ O .
    6  . . . @ O O O O .
    7  . . @ O O . . . .
    8  . . . @ @ . . . .
    9  . . . . . . . . .

```python
X = Boards
Y = keras.utils.to_categorical(Moves, 82)

training_samples = int(0.9 * X.shape[0])
X_train, X_test = X[:training_samples], X[training_samples:] # Inputs
Y_train, Y_test = Y[:training_samples], Y[training_samples:] # Outputs

print(X.shape)
print(Y.shape)
```

    (414124, 82, 2)
    (414124, 82)

# Building the Model

```python
input_shape = (82, 2)

model = keras.models.Sequential()
model.add(keras.layers.Dense(2, input_shape = input_shape))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(82, activation = 'relu'))
model.add(keras.layers.Dense(82, activation = 'relu'))
model.add(keras.layers.Dense(82, activation = 'relu'))
model.add(keras.layers.Dense(82, activation = 'relu'))
model.add(keras.layers.Dropout(.1))
model.add(keras.layers.Dense(82, activation = 'relu'))

model.compile(loss=keras.losses.CategoricalCrossentropy(), optimizer = keras.optimizers.Adam(), metrics = [keras.metrics.CategoricalAccuracy()])

model.summary()
```

    Model: "sequential_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_6 (Dense)              (None, 82, 2)             6
    _________________________________________________________________
    flatten_1 (Flatten)          (None, 164)               0
    _________________________________________________________________
    dense_7 (Dense)              (None, 82)                13530
    _________________________________________________________________
    dense_8 (Dense)              (None, 82)                6806
    _________________________________________________________________
    dense_9 (Dense)              (None, 82)                6806
    _________________________________________________________________
    dense_10 (Dense)             (None, 82)                6806
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 82)                0
    _________________________________________________________________
    dense_11 (Dense)             (None, 82)                6806
    =================================================================
    Total params: 40,760
    Trainable params: 40,760
    Non-trainable params: 0
    _________________________________________________________________

# Training

```python
# Load Weights
#model.load_weights('mini_weights.h5')
```

```python
#Train the model
print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)
history = model.fit(X_train, Y_train, batch_size = 32, epochs = 128, workers = 10, verbose = 1, validation_data = (X_test, Y_test))
```

```python
# Save Weights
model.save_weights('mini_weights.h5')
```

```python
plt.figure(1)

plt.subplot(211)
plt.plot(history.history['categorical_accuracy'])
plt.plot(history.history['val_categorical_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')

plt.subplot(212)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')

plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.tight_layout()
plt.show()
```

```python

```


# 1. Recall

### 1.1 Recall: Transformer

### 1.2 Recall: BERT

### 1.3 Recall: Language Model

- For the sequence $\color{MediumOrchid}{w_{1}, w_{2}, \ldots, w_{n}}$, using the chain rule, we have:

$$
P\left(w_{1}, \ldots, w_{n}\right)=P\left(w_{n} \mid w_{1}, \ldots, w_{n-1}\right) P\left(w_{n-1} \mid w_{1}, \ldots, w_{n-2}\right) \ldots P\left(w_{2} \mid w_{1}\right) P\left(w_{1}\right)
$$

- N-Gram Approximation: $\color{MediumOrchid}{P\left(w_{1}, \ldots, w_{n}\right)=\prod_{i=1}^{n} P\left(w_{i} \mid w_{i-N+1}, \ldots, w_{i-1}\right)}$

- Applications:
    - Machine Translation: $\color{MediumOrchid}{P(\text{the cat is small })>P(\text{ small is the cat} )}$
    - Grammar Checking: $\color{MediumOrchid}{P(\text{ He graduated from SJTU. )>P(He graduated on SJTU.)}}$

# 2. GPT

GPT: Generative Pre-Training

相关论文:

1. Radford, A., & Narasimhan, K. (2018). Improving Language Understanding by Generative Pre-Training.

2. Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., & Sutskever, I. (2019). Language Models are Unsupervised Multitask Learners.

3. Brown, T. et al. “Language Models are Few-Shot Learners.” ArXiv abs/2005.14165 (2020): n. pag.

### 2.1 Introduction

1. In this paper, we explore a semi-supervised approach for language understanding tasks using a combination of unsupervised pre-training and supervised fine-tuning.

2. Our goal is to learn a universal representation that transfers with little adaptation to a wide range of tasks.

3. We employ a two-stage training procedure.
    - First, we use a language modeling objective on the unlabeled data to learn the initial parameters of a neural network model.
    - Subsequently, we adapt these parameters to a target task using the corresponding supervised objective.

4. For our model architecture, we use the Transformer, This model choice provides us with a more structured memory for handling long-term dependencies in text, compared to alternatives like recurrent networks, resulting in robust transfer performance across diverse tasks.

5. We evaluate our approach on four types of language understanding tasks:
    - natural language inference;
    - question answering;
    - semantic similarity;
    - text classification.

### 2.2 Related Work

#### Semi-supervised learning for NLP

Over the last few years, researchers have demonstrated the benefits of using word embeddings, which are trained on unlabeled corpora, to improve performance on a variety of tasks. These approaches, however, mainly transfer word-level information, whereas we aim to capture higher-level semantics.

#### Unsupervised pre-training

1. Unsupervised pre-training is a special case of semi-supervised learning where the goal is to find a good initialization point instead of modifying the supervised learning objective.

2. Subsequent research demonstrated that pre-training acts as a regularization scheme, enabling better generalization in deep neural networks.

3. The closest line of work to ours involves pre-training a neural network using a language modeling objective and then fine-tuning it on a target task with supervision.

4. Our choice of **transformer** networks allows us to **capture longer-range linguistic structure**, as demonstrated in our experiments.

5. Other approaches use **hidden representations** from a pre-trained language or machine translation model as auxiliary features while training a supervised model on the target task.

#### Auxiliary training objectives

1. Adding auxiliary unsupervised training objectives is an alternative form of semi-supervised learning.

2. Our experiments also use an auxiliary objective, but as we show, unsupervised pre-training already learns several linguistic aspects relevant to target tasks.

### 2.3 Framework

#### 2.3.1 Unsupervised pre-training

给定语料库的token集合 $\color{MediumOrchid}{\mathcal{U}=\left\{u_{1}, \ldots, u_{n}\right\}}$，使用标准的语言模型极大化似然函数：

$$
\color{MediumOrchid}{L_{1}(\mathcal{U})=\sum_{i} \log P\left(u_{i} \mid u_{i-k}, \ldots, u_{i-1} ; \Theta\right)} \tag{1}
$$

- $k:$ size of the context window

GPT使用Multi-layer Transformer Decoder来训练语言模型:

$$
\begin{eqnarray}
\color{red}{h_{0}} &=& \color{red}{UW_{e} + W_{p}~~ ???} \\
h_{l} &=& \text{transformer_block}(h_{l-1}) \quad \forall l \in [1, n]  \tag{2}\\
P(u) &=& \text{softmax}(h_{n}W_{e}^{\mathrm{T}})
\end{eqnarray}
$$

- $U=(u_{-k}, \cdots, u_{-1}):$ context vector of tokens;
- $n:$ number of layers;
- $W_{e}:$ token embedding matrix;
- $W_{p}:$ position embedding matrix.

#### 2.3.2 Supervised fine-tuning

设有标注数据集 $\mathcal{C}$，每一个样本由输入序列 $x^{1}, \cdots, x^{m}$ 和标签 $y$ 组成。

将输入序列传入 pre-trained 模型中，取最后一层transformer block的激活项 $h_{l}^{m}$ ，送进一个线性输出层，预测 $y$:

$$
P\left(y \mid x^{1}, \ldots, x^{m}\right)=\operatorname{softmax}\left(h_{l}^{m} W_{y}\right) \tag{3}
$$

目标是最大化如下目标函数:

$$
L_{2}(\mathcal{C})=\sum_{(x, y)} \log P\left(y \mid x^{1}, \ldots, x^{m}\right) \tag{4}
$$

最终和$L_{1}$一起优化，目标函数为:

$$
L_{3}(\mathcal{C})=L_{2}(\mathcal{C})+\lambda * L_{1}(\mathcal{C}) \tag{5}
$$

加上fine-tuning的目标函数一起优化，可以:
- improving generalization of the supervised model;
- accelerating convergence.

# 3. Transformer-XL

- Segment-level Recurrence
- Relative Positional Embedding

在NLP领域，处理语言建模问题有两种最先进的架构：RNN和Transformer。RNN按照序列顺序逐个学习输入的单词或字符之间的关系；而Transformer则接收一整段序列，然后使用self-Attention机制来学习它们之间的依赖关系。

但它们都有一个共同不足之处: unable to model dependencies longer than a fixed length.

Transformer-XL 同时结合了RNN序列建模和Transformer自注意力机制的优点，在输入数据的每个片段上使用Transformer的Self-Attention模块，并使用循环机制来学习连续段之间的依赖关系。

### 3.1 vanilla Transformer

Al-Rfou等人基于Transformer提出了vanilla model，它根据之前的字符预测片段中的下一个字符。例如，它使用$x_{1}, x_{2}, \cdots, x_{n-1}$预测字符$x_{n}$，而在$x_{n}$之后的序列则被mask掉。论文中使用64层模型，并仅限于处理512个字符这种相对较短的输入，因此它将输入分成段，并分别从每个段中进行学习，如上图所示。在Evaluation阶段，该模型会在每一步中将输入向右移动一个字符，以此实现对单个字符的预测。

但vanilla model仍有些缺点:

1. 因为segments之间独立训练，所以不同的token之间，最长的依赖关系，就取决于segment的长度；

2. 出于效率考虑，在划分segments的时候，不考虑句子的自然边界，而是根据固定的长度来划分序列，导致分割出来的segments在语义上是不完整的。（context fragmentation problem）；

3. 推理速度慢: 在Evaluation阶段，一般取最后一个位置的隐向量作为输出。为了充分利用上下文关系，在每做完一次预测之后，就对整个序列向右移动一个位置，再做一次计算，如上图（b）所示，则导致计算效率非常低。

### 3.2 Segment-Level Recurrence with State Reuse

Transformer-XL在对当前segment进行处理的时候，缓存并利用上一个segment中所有layer的隐向量序列，而且上一个segment的所有隐向量序列只参与前向计算，不再进行反向传播，这就是所谓的segment-level recurrence。

#### 符号说明:

两个连续的segments表示为 $s_{\tau}=\left[x_{\tau, 1}, x_{\tau, 2}, \ldots, x_{\tau, L}\right],\ s_{\tau+1}=\left[x_{\tau+1, 1}, x_{\tau+1, 2}, \ldots, x_{\tau+1, L}\right], \ L$是序列长度；

假设整个模型中，包含$~N~$层Transformer-block，那么每个segment中就有$~N~$组长度为$L$的隐向量序列；

$\mathbf{h}_{\tau}^{n} \in \mathbb{R}^{L \times d}$——表示第$~\tau~$个segment的第$~n~$层隐向量序列；

$\text{SG}$是stop-gradient，不在对$~s_{\tau}$ 的隐向量做反向传播；

$\widetilde{\mathbf{h}}_{\tau+1}^{n-1}$ 是对两个隐向量序列沿长度方向的拼接，\[\]内两个隐向量的维度都是$L \times d$，拼接之后的向量维度是 $2L \times d$;

$\mathbf{q}$ 的计算方式不变，只使用当前segment中隐向量，计算得到的$\mathbf{q}$序列长度仍是$L$；

$\mathbf{k}, \mathbf{v}$采用拼接之后的$\widetilde{\mathbf{h}}$来计算，计算出来的序列长度是$2L$;

Transformer的输出隐向量序列长度取决于query的序列长度，而不是key和value.

$$
\begin{array}{l}
\widetilde{\mathbf{h}}_{\tau+1}^{n-1}=\left[\mathrm{SG}\left(\mathbf{h}_{\tau}^{n-1}\right) \circ \mathbf{h}_{\tau+1}^{n-1}\right] \\
\mathbf{q}_{\tau+1}^{n}, \mathbf{k}_{\tau+1}^{n}, \mathbf{v}_{\tau+1}^{n}=\mathbf{h}_{\tau+1}^{n-1} \mathbf{W}_{q}^{\top}, \widetilde{\mathbf{h}}_{\tau+1}^{n-1} \mathbf{W}_{k}^{\top}, \widetilde{\mathbf{h}}_{\tau+1}^{n-1} \mathbf{W}_{v}^{\top} \\
\mathbf{h}_{\tau+1}^{n}= \text{Transformer-Layer} \left(\mathbf{q}_{\tau+1}^{n}, \mathbf{k}_{\tau+1}^{n}, \mathbf{v}_{\tau+1}^{n}\right)
\end{array}
$$

训练和预测过程如Fig2所示。需注意的一点: 在当前segment中，第$~n~$层的每个隐向量的计算，除了依赖当前位置的下一层隐向量，还与前$L-1$个位置的隐向量存在依赖关系，而且每往下走一层，依赖关系长度都会增加$(L-1)$，所以最长的依赖关系是$N(L-1)$。在对长文本进行计算的时候，可以缓存上一个segment的隐向量的结果，不必重复计算，大幅提高计算效率。

上文中，我们只保存了上一个segment，实际操作的时候，可以保存尽可能多的segments，只要内存或者显存放得下。论文中的试验在训练的时候，只缓存一个segment，在预测的时候，会缓存多个segments。

### 3.3 Relative Positional Encodings

Transformer-XL放弃了vanilla transformer绝对位置编码，而采用相对位置编码。具体地，在计算Attention Score的时候，只考虑query向量与key向量的相对位置关系，并且将这种相对位置关系，加入到每一层Transformer的Attention的计算中。

Vanilla Transformer中位置Embedding的计算公式如下:

$$
\begin{aligned}
\mathbf { A } _ { i , j } ^ { \mathrm { abs } }
& = \left\{ \mathbf { W }_{ q } \left( \mathbf { E }_{ x_{ i } } + \mathbf { U }_{ i } \right) \right\}^{\top}
\left\{ \mathbf { W }_{ k } \left( \mathbf { E }_{ x_{ j } } + \mathbf { U }_{ j } \right) \right\}\\
\quad \\
& = \underbrace { \mathbf { E } _ { x _ { i } } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k } \mathbf { E } _ { x _ { j } } } _ { ( a ) } + \underbrace { \mathbf { E } _ { x _ { i } } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k } \mathbf { U } _ { j } } _ { ( b ) }  + \underbrace { \mathbf { U } _ { i } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k } \mathbf { E } _ { x _ { j } } } _ { ( c ) } + \underbrace { \mathbf { U } _ { i } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k } \mathbf { U } _ { j } } _ { ( d ) }
\end{aligned}
$$

而Transformer-XL中使用相对位置计算attention score的公式如下:

$$
\begin{aligned}
\mathbf { A } _ { i , j } ^ { \mathrm { rel } }
& = \underbrace { \mathbf { E } _ { x _ { i } } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k,E } \mathbf { E } _ { x _ { j } } } _ { ( a ) } + \underbrace { \mathbf { E } _ { x _ { i } } ^ { \top } \mathbf { W } _ { q } ^ { \top } \mathbf { W } _ { k,R } \color{DeepSkyBlue}{ \mathbf { R } _ { i-j }} } _ { ( b ) }  + \underbrace { \color{red}{u ^ { \top }} \mathbf { W } _ { k,E }  \mathbf { E } _ { x _ { j } } } _ { ( c ) } + \underbrace { \color{red}{ v ^ { \top }} \mathbf { W } _ { k,R }  \color{DeepSkyBlue}{ \mathbf { R } _ { i-j }} } _ { ( d ) }
\end{aligned}
$$

其中:
- $\color{red}{u,v}$ 是trainable parameters;

- $\mathbf{W}_{k,E}$ 用于生成基于内容的key向量；

- $\mathbf{W}_{k,R}$ 用于生成基于位置的key向量；

- $\mathbf{R} \in \mathbb{R}^{L_{max} \ \  \times d}$，第$~i~$行表示相对位置间隔为$~i~$的位置向量。论文中强调$\mathbf{R}$采用正弦函数生成，而不是通过学习得到的。

最后，对于一个$N$ 层的 single attention head 的Transformer-XL的计算公式如下:

$\text{For} \quad  n=1,\cdots, N:$

# 3. XLNet

*XLNet: Generalized Autoregressive Pretraining for Language Understanding*

Part of Advances in Neural Information Processing Systems 32 (NeurIPS 2019)

**Authors:**

Zhilin Yang, Zihang Dai, Yiming Yang, Jaime Carbonell, Russ R. Salakhutdinov, Quoc V. Le

## 3.1 AR vs. AE

### 3.1.1 Auto-Regression LM

ELMo(2018.03)/BERT(2018.10.11)出来之前，大家通常讲的Language Model其实是根据上文内容预测下一个可能跟随的单词，就是常说的自左向右的Language Model，或者反过来也行。这种Language Model被称为自回归语言模型（Auto-Regression LM）。GPT是典型的AR LM。ELMo尽管看上去是利用了上文，也利用了下文，但是本质上是Auto-Regression LM。ELMo是分别做了两个方向的自回归LM，然后把LSTM的两个方向的隐状态拼接到一起，来体现双向语言模型这个事情。所以本质上认识Auto-Regression LM

给定文本序列 $\pmb{x}=\left[x_{1}, \ldots, x_{T}\right]$，Language Model的目标是调整参数使得训练数据上的似然函数最大:

$$
\max _{\theta} \log p_{\theta}(\pmb{x}) \color{red}{=} \sum_{t=1}^{T} \log p_{\theta}\left(x_{t} \mid \pmb{x}_{<t}\right)=\sum_{t=1}^{T} \log \frac{\exp \left(h_{\theta}\left(\pmb{x}_{1: t-1}\right)^{T} e\left(x_{t}\right)\right)}{\sum_{x^{\prime}} \exp \left(h_{\theta}\left(\pmb{x}_{1: t-1}\right)^{T} e\left(x^{\prime}\right)\right)}
$$

- $\pmb{x}_{<t}$: 表示$~t~$时刻之前的所有$x$，也就是 $\pmb{x}_{1:t-1}$；
- $h_{\theta}\left(\pmb{x}_{1: t-1}\right)$ 是RNN或者Transformer编码的$t$时刻之前的隐状态；
- $e(x)$: 是词$x$的Embedding;
- '=' 等号成立，是因为使用了chain rule.

Auto-Regression LM的优点:
- No discrepancy;
- 考虑依赖关系

Auto-Regression LM的缺点:
- 是无法同时利用上下文的信息。

### 3.1.2 Auto-Encoder LM

BERT是一种典型的自编码语言模型（Auto-Encoder LM）。它通过将序列$~\pmb{x}~$中随机挑选15%的Token变成\[MASK\]得到带噪声版本的 $\hat{\pmb{x}}$。假设被Mask的原始值为$\bar{\pmb{x}}$，那么BERT希望尽量根据上下文恢复（猜测）出原始值，也就是:

$$
\max _{\theta} \log p_{\theta}(\overline{\mathbf{x}} \mid \hat{\mathbf{x}}) \color{red}{ \approx } \sum_{t=1}^{T} m_{t} \log p_{\theta}\left(x_{t} \mid \hat{\mathbf{x}}\right)=\sum_{t=1}^{T} m_{t} \log \frac{\exp \left(H_{\theta}(\mathbf{x})_{t}^{T} e\left(x_{t}\right)\right)}{\sum_{x^{\prime}} \exp \left(H_{\theta}(\mathbf{x})_{t}^{T} e\left(x^{\prime}\right)\right)}
$$

- $m_{t}=1$ 表示$t$时刻是一个MASK，需要恢复；
- $H_{\theta}$ 是一个Transformer，它把长度为$T$的序列$\pmb{x}$映射为隐状态的序列 $H_{\theta}(\mathbf{x})=\left[H_{\theta}(\mathbf{x})_{1}, H_{\theta}(\mathbf{x})_{2}, \ldots, H_{\theta}(\mathbf{x})_{T}\right]$
- '$\color{red}{\approx}$' ，是因为引入了条件独立的假设(Independent Assumption)，$P(New ~ York|is,a,city) \color{red}{\approx} P(New|is,a,city)\cdot P(York|is,a,city)$

Auto-Encoder LM能比较自然地融入双向语言模型，同时看到被预测单词的上文和下文，这是优点。但是，在输入侧引入\[MASK\]标记，导致Pre-training和Fine-tuning阶段不一致的问题，因为Fine-tuning阶段是看不到\[MASK\]标记的。

XLNet 的出发点就是：能否融合自回归 LM 和 DAE LM 两者的优点。具体来说就是，站在 AR 的角度，如何引入和双向语言模型等价的效果.

## 3.2 Permutation Language Model

#### 要点：
- Sample a factorization order;

- Determine the attention masks based on the order;

- Optimize a standard language modeling objective:

$$
\max _{\theta} \mathbb{E}_{\mathbf{z} \sim \mathcal{Z}_{T}}\left[\sum_{t=1}^{T} \log p_{\theta}\left(x_{z_{t}} \mid \mathbf{x}_{\mathbf{z}_{<t}}\right)\right]
$$

- Benifits:
     - Autoregressive, avoding disadvantages of AE;
     - Able to model bidirectional context

具体实现方式:  通过随机取一句话排列的一种，然后将末尾一定量的词给“遮掩”（和BERT的[MASK]有些不同）掉，最后用Auto-Regression的方式来按照这种排列方式依次预测被“遮掩”掉的词。

最后“遮掩”的token长度怎么选择呢？
作者设了一个超参数$K$，$K$等于总长度除以需要预测的个数。以上图为例，中长为7，需要预测的长度为2，于是$K=7/2$。论文中实验得出的最佳$K$值介于6和7之间。如果去$K$的导数（即$\frac{1}{6}, \frac{1}{7}$），转化为百分比为（14.3%，16.7%）之间。而BERT中将Token替换为\[MASK\]的比列就是15%，二者之间应该有本质上的联系。

关于句子排列的采样：对于一个长度为$T$的句子，有$T!$中排列，如果遍历每种排列，是不现实的。用 $\mathcal{Z}_{T}$ 表示所有排列组成的集合，$\mathcal{z}$ 表示从$\mathcal{Z}_{T}$采样得到一种排序，记为$\mathcal{z} \sim \mathcal{Z}_{T}$

XLNet并不是打乱输入句子的顺序，而是通过Transformer的Attention Masks来巧妙实现的。

## 3.3 Reparameterization

Permutation Language Model的思想很简单，但如果还是用standard Transformer parameterization，就会有问题，standard Transformer parameterization公式为:

$$
\max _{\theta} \ \log p_{\theta}(\pmb{x}) \color{red}{=} \sum_{t=1}^{T} \log p_{\theta}\left(x_{t} \mid \pmb{x}_{<t}\right)=\sum_{t=1}^{T} \log \frac{\exp \left(h_{\theta}\left(\pmb{x}_{1: t-1}\right)^{T} e\left(x_{t}\right)\right)}{\sum_{x^{\prime}} \exp \left(h_{\theta}\left(\pmb{x}_{1: t-1}\right)^{T} e\left(x^{\prime}\right)\right)}
$$

举例来说:

假设输入的句子是 "New York is a city"，它由两种排列：
1. is a city New York ——> \[3,4,5,1,2\]
2. is a city York New ——> \[3,4,5,2,1\]

对于第1种排列，假设我们要预测$z_{4} = New$，则有:

$$
p_{\theta}\left( \text{New} \mid \text{is, a, city} \right) =  \frac{\exp
\left \{ h_{\theta} \left( \text{is, a, city} \right)^{\mathrm{T}} \cdot e(\text{New}) \right\} }{ \sum_{x^{\prime}} \exp
\left \{ h_{\theta} \left( \text{is, a, city} \right)^{\mathrm{T}} \cdot e(x^{\prime}) \right\} }
$$

同理，对于第2中排列，假设我们要预测$z_{4} = New$，同样有:

$$
p_{\theta}\left( \text{New} \mid \text{is, a, city} \right) =  \frac{\exp
\left \{ h_{\theta} \left( \text{is, a, city} \right)^{\mathrm{T}} \cdot e(\text{New}) \right\} }{ \sum_{x^{\prime}} \exp
\left \{ h_{\theta} \left( \text{is, a, city} \right)^{\mathrm{T}} \cdot e(x^{\prime}) \right\} }
$$

上面两个公式得到的概率是相等的，但是对于两种排列，它们的概率应该是不相等的，而问题的原因在于$h_{\theta}\left(\pmb{x}_{1: t-1}\right)$没有建模位置信息。

为了解决上述问题，XLNet提出了一种新的参数化表示方法:

$$
p_{\theta}\left(X_{z_{t}}=x \mid \mathbf{x}_{z_{<t}}\right)=\frac{\exp \left(e(x)^{\top} g_{\theta}\left(\mathbf{x}_{\mathbf{z}_{<t}} \ , z_{t}\right)\right)}{\sum_{x^{\prime}} \exp \left(e\left(x^{\prime}\right)^{\top} g_{\theta}\left(\mathbf{x}_{\mathbf{z}_{<t}} \ ,  z_{t}\right)\right)}
$$

- $\mathbf{x}_{\mathbf{z}_{<t}}$  表示$\mathbf{x}_{1:t-1}$的意思（包含了内容和位置信息）
- $z_{t}$ 表示$x_{t}$（预测）的位置信息

## 3.4 Two-Stream Self-Attention

对于3.3中提出的参数化表示，该用什么模型来表示$g_{\theta} \left(\mathbf{x}_{\mathbf{z} {<t}},z_{t} \right)$呢？

$g_{\theta}$需满足下面两个条件:

1. 预测$~x_{z_{t}}~$ 时，只能使用位置信息$z_{t}$，而不能使用内容信息$x_{z_{t}}$；
2. 而预测其它tokens $x_{z_{j}}(j > t)$时，需要包含内容信息$x_{z_{t}}$

对于上面的两点要求，普通的Transformer Self-Attention是不能满足的，举例说明:

为解决上述问题，XLNet引入了Two-Stream Self-Attention的设计:

从上图可以看到，在计算Attention时，引入了两个Stream，也就是两个隐状态:

- 内容隐状态 $h_{\theta}(\mathbf{x}_{z<t})$，简写为$h_{z_{t}}$，它和标准的Transformer一样，既编码上下文内容，也编码$\mathbf{x}_{z_{t}}$的内容；

- 查询隐状态 $g_{\theta}(\mathbf{x}_{z<t},z_{t})$，简写为$g_{z_{t}}$，它只编码上下文和要预测的位置$z_{t}$，但不包含$\mathbf{x}_{z_{t}}$

#### 计算过程——初始化

1. 把查询隐状态$g_{i}^{(0)}$初始化为一个变量 $w$；

2. 把内容隐状态$h_{i}^{(0)}$初始化为token的Embedding $e(x_{i})$

这里上标(0)表示第0层（并不存在，只是用于计算第1层）

#### 计算过程——逐层计算

$$
\begin{align}
\text{Query Stream}&: \quad g_{z_{t}}^{(m)} \leftarrow \text{Attention} \left(Q=\color{red}{g_{z_{t}}^{(m-1)}}, K V=h_{\color{red}{z_{<t}}}^{(m-1)} ; \theta\right) \\
\text{Content Stream}&: \quad h_{z_{t}}^{(m)} \leftarrow \text{Attention} \left(Q=\color{red}{h_{z_{t}}^{(m-1)}}, K V=h_{\color{red}{z_{\leq t}}}^{(m-1)} ; \theta\right)
\end{align}
$$

- Query Stream: use $z_{t}$ but cannot see $\mathbf{x}_{z_{t}}$

- Content Stream: use both $z_{t}$ and $\mathbf{x}_{z_{t}}$

上面的梯度更新和Standard Transformer Self-Attention一样，在Fine-tuning时，可以丢弃掉Query Stream而只用 Content Stream。最后在计算公式的时候，我们可以用上面一层的Query向量$g_{z_{t}}^{(M)}$。

## 3.5 XLNet vs. BERT

假设输入Sentence='New York is a city'. 并且\[New, York\]是要预测的目标，则目标函数是 $\text{maximize} ~ \log p(\text{New York }\mid \text{ is, a, city)}$;

假设XLNet采样得到的句子排列是\[is, a, city, New, York\]，则BERT和XLNet的目标函数分别是:

$$
\begin{align}
\mathcal{J}_{BERT} &= \log p\left(\text{New} \mid \text{is a city} \right) + \log p\left(\text{York} \mid \text{is a city} \right) \\
\mathcal{J}_{XLNet} &= \log p\left(\text{New} \mid \text{is a city} \right) + \log p\left(\text{York} \mid \color{red}{\text{New}}, \text{is a city} \right)
\end{align}
$$

可以看到，XLNet可以捕捉(New, York)词对之间的相互依赖关系，而BERT不能。

## 3.6 Summary