```python
import numpy as np
import pandas as pd
from sympy import pprint
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import learning_curve

#combined_data
train = pd.read_csv('train.csv',header=None)
test = pd.read_csv('test.csv',header=None)
val = pd.read_csv('val.csv',header=None)

#using validation data for different prediction models and check for least E_in
#processing_data to 20 columns
i=0;
data1=pd.DataFrame()
while(i<1080):
    data1=pd.DataFrame.append(data1,val.iloc[:,i:(i+120)].mean(axis=1),ignore_index=True)
    i=i+120;
i=1080;
while(i<1094):
    data1=pd.DataFrame.append(data1,val.iloc[:,i:(i+2)].T,ignore_index=True)
    i=i+2;
    data1=pd.DataFrame.append(data1,val.iloc[:,i:(i+2)].sum(axis=1),ignore_index=True)
    i=i+2;

data1=data1.T
data1=data1.iloc[:,0:20]
Xval=data1.iloc[:,0:18]
Yval=data1.iloc[:,18:20]

j=0;
test1=pd.DataFrame()
while(j<1080):
    test1=pd.DataFrame.append(test1,test.iloc[:,j:(j+120)].mean(axis=1),ignore_index=True)
    j=j+120;
j=1080;
while(j<1094):
    test1=pd.DataFrame.append(test1,test.iloc[:,j:(j+2)].T,ignore_index=True)
    j=j+2;
    test1=pd.DataFrame.append(test1,test.iloc[:,j:(j+2)].sum(axis=1),ignore_index=True)
    j=j+2;

test1=test1.T
test1=test1.iloc[:,0:20]
Xtest=test1.iloc[:,0:18]
Ytest=test1.iloc[:,18:20]

k=0;
train1=pd.DataFrame()
while(k<1080):
    train1=pd.DataFrame.append(train1,train.iloc[:,k:(k+120)].mean(axis=1),ignore_index=True)
    k=k+120;
k=1080;
while(k<1094):
    train1=pd.DataFrame.append(train1,train.iloc[:,k:(k+2)].T,ignore_index=True)
    k=k+2;
    train1=pd.DataFrame.append(train1,train.iloc[:,k:(k+2)].sum(axis=1),ignore_index=True)
    k=k+2;

train1=train1.T
train1=train1.iloc[:,0:20]
Xtrain=train1.iloc[:,0:18]
Ytrain=train1.iloc[:,18:20]

#LINEAR REGRESSION
from sklearn.linear_model import LinearRegression
model=LinearRegression()
model.fit(Xtrain,Ytrain)
weights=model.coef_
Yval_pred=model.predict(Xval)
Eval=mean_squared_error(Yval.iloc[:,0:2],Yval_pred[:,:], multioutput='raw_values')
print('Eval for LINEAR REGRESSION ',Eval)

#learning curve
training_sizes, training_scores,validation_scores = learning_curve(
    estimator = model,
    X = Xval,
    y = Yval,
    train_sizes = np.linspace(5, len(Xval) * 0.8, dtype = int)
)
line1 = plt.plot(
    training_sizes, training_scores.mean(axis = 1), 'r')

```

```python
import tensorflow
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense

neural = Sequential([
    Dense(18, activation='tanh'),
    Dense(18, activation='tanh'),
    Dense(2, activation='sigmoid')
])
neural.compile(optimizer='sgd',
              loss='binary_crossentropy')

#for translational velocity

hist = neural.fit(Xtrain, Ytrain, batch_size=1000, epochs=50,validation_data=(Xval, Yval))

Yval_pred1=neural.predict(Xval)
Eval1=mean_squared_error(Yval.iloc[:,0:1],Yval_pred1[:,0], multioutput='raw_values')
print('Eval of translational velocity for Neural Network',Eval1)
Eval2=mean_squared_error(Yval.iloc[:,1:2],Yval_pred1[:,1], multioutput='raw_values')
print('Eval of rotational velocity for Neural Network',Eval2)

#learning curve
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')

plt.show()
```

```python
from keras.layers import Dropout
from keras import regularizers

#data is purposefully overfitted to show hor regularization works with λ being 0.01
neural1 = Sequential([
    Dense(1000, activation='tanh'),
    Dense(1000, activation='tanh'),
    Dense(2, activation='sigmoid')])

neural1.compile(optimizer='sgd',
              loss='binary_crossentropy')

hist1 = neural1.fit(Xtrain, Ytrain,batch_size=1000, epochs=50,validation_data=(Xval, Yval))

plt.plot(hist1.history['loss'])
plt.plot(hist1.history['val_loss'])
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()
neural2 = Sequential([
    Dense(1000, activation='tanh', kernel_regularizer=regularizers.l2(0.01)),
    Dropout(0.3),
    Dense(1000, activation='tanh', kernel_regularizer=regularizers.l2(0.01)),
    Dropout(0.3),
    Dense(2, activation='sigmoid', kernel_regularizer=regularizers.l2(0.01))])

neural2.compile(optimizer='sgd',
              loss='binary_crossentropy')

hist2 = neural2.fit(Xtrain, Ytrain,batch_size=1000, epochs=50,validation_data=(Xval, Yval))
plt.plot(hist2.history['loss'])
plt.plot(hist2.history['val_loss'])
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val'], loc='upper right')
plt.show()
```

```python
#to_test
Xtest=test1.iloc[:,0:18]
Ytest=test1.iloc[:,18:20]
Ytest_pred=neural.predict(Xtest)

Etest_linvel=mean_squared_error(Ytest.iloc[:,0:1],Ytest_pred[:,0])
print('Etest for lin vel',Etest_linvel)
Etest_angvel=mean_squared_error(Ytest.iloc[:,1:2],Ytest_pred[:,1])
print('Etest for ang vel',Etest_angvel)
dvc=19
tol=0.1

E_out = Etest_linvel + np.sqrt((8/35861)*np.log(4*((2*35861)**(dvc+1))/tol))
print('Eout for translational velocity',E_out)
E_out1 = Etest_angvel + np.sqrt((8/35861)*np.log(4*((2*35861)**(dvc+1))/tol))
print('Eout for rotational velocity',E_out1)

```

    Etest for lin vel 0.0843801184253638
    Etest for ang vel 0.04773384888149761
    Eout for translational velocity 0.30956200431383674
    Eout for rotational velocity 0.27291573476997055

```python

```


# Unrolled generative adversarial networks on a toy dataset

This notebook demos a simple implementation of unrolled generative adversarial networks on a 2d mixture of Gaussians dataset. See the [paper](https://arxiv.org/abs/1611.02163) for a better description of the technique, experiments, results, and other good stuff. Note that the architecture and hyperparameters used in this notebook are not identical to the one in the paper.

## Motivation
The GAN learning problem is to find the optimal parameters $\theta_G^*$ for a generator function $G\left( z; \theta_G\right)$ in a minimax objective,
$$\begin{align}
    \theta_G^* &= \underset{\theta_G}{\text{argmin}} \underset{\theta_D}{\max} f\left(\theta_G, \theta_D\right) \\
&= \underset{\theta_G}{\text{argmin}} \;f\left(\theta_G, \theta_D^*\left(\theta_G\right)\right)\\
\theta_D^*\left(\theta_G\right) &= \underset{\theta_D}{\max}  \;f\left(\theta_G, \theta_D\right),
\end{align}$$
where the saddle objective $f$ is the standard GAN loss:
$$f\left(\theta_G, \theta_D\right) =
    \mathbb{E}_{x\sim p_{data}}\left[\mathrm{log}\left(D\left(x; \theta_D\right)\right)\right] +
    \mathbb{E}_{z \sim \mathcal{N}(0,I)}\left[\mathrm{log}\left(1 - D\left(G\left(z; \theta_G\right); \theta_D\right)\right)\right].
$$

In unrolled GANs, we approximate $\theta_D^*\left(\theta_G\right)$ using a few steps of gradient ascent:
$$\theta_D^*\left(\theta_G\right) \approx \hat{\theta}_D\left(\theta_G\right) \equiv\text{ a few steps of SGD maximizing}\;f\left(\theta_G, \theta_D\right).$$

We can then compute the update for the generator parameters, $\theta_G$, by computing the gradient of the saddle objective with respect to $\theta_G$ and the optimized discriminator parameters, $\hat{\theta}_D$:
$$\frac{d}{d \theta_G} f\left(\theta_G, \hat{\theta}_D\left(\theta_G\right)\right)$$.

## Implementation details
To backpropagate through the optimization process, we need to create a symbolic computational graph that includes all the operations from the initial weights to the optimized weights. TensorFlow's built-in optimizers use custom C++ code for efficiency, and do not construct a symbolic graph that is differentiable. For this notebook, we use the optimization routines from `keras` to compute updates.  Next, we use `tf.contrib.graph_editor.graph_replace` to build a copy of the graph containing the mapping from initial weights to updated weights after one optimization iteration, but replacing the initial weights with the last iteration's weights:

This yields a new graph that allows us to backprop from $\theta_D^2$ back to $\theta_D^0$. We can then plug $\theta_D^2$ into the loss function to get the final objective that the generator optimizes. Using the magic of `graph_replace` we can write the unrolled optimization procedure in just a few lines:
```python
# update_dict contains a dictionary mapping from variables (\theta_D^0)
# to their values after one step of optimization (\theta_D^1)
cur_update_dict = update_dict
for i in xrange(params['unrolling_steps'] - 1):
    # Compute variable updates given the previous iteration's updated variable
    cur_update_dict = graph_replace(update_dict, cur_update_dict)
# Final unrolled loss uses the parameters at the last time step
unrolled_loss = graph_replace(loss, cur_update_dict)
```

Note there are many other ways of implementing unrolled optimization that don't use graph rewriting. For example, if we created a function that takes weights as inputs and returns the updated weights, we could just iteratively call that function.

```python
%pylab inline
from collections import OrderedDict
import tensorflow as tf
ds = tf.contrib.distributions
slim = tf.contrib.slim

from keras.optimizers import Adam

try:
    from moviepy.video.io.bindings import mplfig_to_npimage
    import moviepy.editor as mpy
    generate_movie = True
except:
    print("Warning: moviepy not found.")
    generate_movie = False
```

    Populating the interactive namespace from numpy and matplotlib

    c:\users\donatas\appdata\local\programs\python\python36\lib\site-packages\IPython\core\magics\pylab.py:160: UserWarning: pylab import has clobbered these variables: ['f']
    `%matplotlib` prevents importing * from pylab and numpy
      "\n`%matplotlib` prevents importing * from pylab and numpy"
    Using TensorFlow backend.

    Warning: moviepy not found.

`graph_replace` is broken in TensorFlow 1.0 (see this [issue](https://github.com/tensorflow/tensorflow/issues/9125)). We get around this issue with an ugly hack that removes the problematic attribute from all ops in the graph on every call to `graph_replace`.

```python
_graph_replace = tf.contrib.graph_editor.graph_replace

def remove_original_op_attributes(graph):
    """Remove _original_op attribute from all operations in a graph."""
    for op in graph.get_operations():
        op._original_op = None

def graph_replace(*args, **kwargs):
    """Monkey patch graph_replace so that it works with TF 1.0"""
    remove_original_op_attributes(tf.get_default_graph())
    return _graph_replace(*args, **kwargs)
```

### Utility functions

```python
def extract_update_dict(update_ops):
    """Extract variables and their new values from Assign and AssignAdd ops.

    Args:
        update_ops: list of Assign and AssignAdd ops, typically computed using Keras' opt.get_updates()

    Returns:
        dict mapping from variable values to their updated value
    """
    name_to_var = {v.name: v for v in tf.global_variables()}
    updates = OrderedDict()
    for update in update_ops:
        var_name = update.op.inputs[0].name
        var = name_to_var[var_name]
        value = update.op.inputs[1]
        if update.op.type == 'Assign':
            updates[var.value()] = value
        elif update.op.type == 'AssignAdd':
            updates[var.value()] = var + value
        else:
            raise ValueError("Update op type (%s) must be of type Assign or AssignAdd"%update_op.op.type)
    return updates
```

### Data creation

```python
def sample_mog(batch_size, n_mixture=8, std=0.01, radius=1.0):
    thetas = np.linspace(0, 2 * np.pi, n_mixture)
    xs, ys = radius * np.sin(thetas), radius * np.cos(thetas)
    cat = ds.Categorical(tf.zeros(n_mixture))
    comps = [ds.MultivariateNormalDiag([xi, yi], [std, std]) for xi, yi in zip(xs.ravel(), ys.ravel())]
    data = ds.Mixture(cat, comps)
    return data.sample(batch_size)
```

### Generator and discriminator architectures

```python
def generator(z, output_dim=2, n_hidden=128, n_layer=2):
    with tf.variable_scope("generator"):
        h = slim.stack(z, slim.fully_connected, [n_hidden] * n_layer, activation_fn=tf.nn.tanh)
        x = slim.fully_connected(h, output_dim, activation_fn=None)
    return x

def discriminator(x, n_hidden=128, n_layer=2, reuse=False):
    with tf.variable_scope("discriminator", reuse=reuse):
        h = x
        h = slim.stack(h, slim.fully_connected, [n_hidden] * n_layer, activation_fn=tf.nn.tanh)
        log_d = slim.fully_connected(h, 1, activation_fn=None)
    return log_d
```

### Hyperparameters

```python
params = dict(
    batch_size=512,
    disc_learning_rate=1e-4,
    gen_learning_rate=1e-3,
    beta1=0.5,
    epsilon=1e-8,
    max_iter=25000,
    viz_every=5000,
    z_dim=256,
    x_dim=2,
    unrolling_steps=5,
)
```

## Construct model and training ops

```python
print(disc_vars)
```

    [<tf.Variable 'discriminator/Stack/fully_connected_1/weights:0' shape=(2, 128) dtype=float32_ref>, <tf.Variable 'discriminator/Stack/fully_connected_1/biases:0' shape=(128,) dtype=float32_ref>, <tf.Variable 'discriminator/Stack/fully_connected_2/weights:0' shape=(128, 128) dtype=float32_ref>, <tf.Variable 'discriminator/Stack/fully_connected_2/biases:0' shape=(128,) dtype=float32_ref>, <tf.Variable 'discriminator/fully_connected/weights:0' shape=(128, 1) dtype=float32_ref>, <tf.Variable 'discriminator/fully_connected/biases:0' shape=(1,) dtype=float32_ref>]

```python
loss
```

    <tf.Tensor 'Mean:0' shape=() dtype=float32>

```python
tf.reset_default_graph()

data = sample_mog(params['batch_size'])

noise = ds.Normal(tf.zeros(params['z_dim']),
                  tf.ones(params['z_dim'])).sample(params['batch_size'])
# Construct generator and discriminator nets
with slim.arg_scope([slim.fully_connected], weights_initializer=tf.orthogonal_initializer(gain=1.4)):
    samples = generator(noise, output_dim=params['x_dim'])
    real_score = discriminator(data)
    fake_score = discriminator(samples, reuse=True)

# Saddle objective
loss = tf.reduce_mean(
    tf.nn.sigmoid_cross_entropy_with_logits(logits=real_score, labels=tf.ones_like(real_score)) +
    tf.nn.sigmoid_cross_entropy_with_logits(logits=fake_score, labels=tf.zeros_like(fake_score)))

gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator")
disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "discriminator")

# Vanilla discriminator update
d_opt = Adam(lr=params['disc_learning_rate'], beta_1=params['beta1'], epsilon=params['epsilon'])
# d_opt = tf.train.AdamOptimizer(params['disc_learning_rate'], beta1=params['beta1'], epsilon=params['epsilon'])
updates = d_opt.get_updates(disc_vars, [], loss)
d_train_op = tf.group(*updates, name="d_train_op")
# Unroll optimization of the discrimiantor
if params['unrolling_steps'] > 0:
    # Get dictionary mapping from variables to their update value after one optimization step
    update_dict = extract_update_dict(updates)
    cur_update_dict = update_dict
    for i in range(params['unrolling_steps'] - 1):
        # Compute variable updates given the previous iteration's updated variable
        cur_update_dict = graph_replace(update_dict, cur_update_dict)
    # Final unrolled loss uses the parameters at the last time step
    unrolled_loss = graph_replace(loss, cur_update_dict)
else:
    unrolled_loss = loss

# Optimize the generator on the unrolled loss
g_train_opt = tf.train.AdamOptimizer(params['gen_learning_rate'], beta1=params['beta1'], epsilon=params['epsilon'])
g_train_op = g_train_opt.minimize(-unrolled_loss, var_list=gen_vars)
```

    WARNING:tensorflow:VARIABLES collection name is deprecated, please use GLOBAL_VARIABLES instead; VARIABLES will be removed after 2017-03-02.

## Train!

```python
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
```

    c:\users\donatas\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\client\session.py:1645: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).
      warnings.warn('An interactive session is already active. This can '

```python
from tqdm import tqdm
xmax = 3
fs = []
frames = []
np_samples = []
n_batches_viz = 10
viz_every = params['viz_every']
for i in tqdm(range(params['max_iter'])):
    f, _, _ = sess.run([[loss, unrolled_loss], g_train_op, d_train_op])
    fs.append(f)
    if i % viz_every == 0:
        np_samples.append(np.vstack([sess.run(samples) for _ in range(n_batches_viz)]))
        xx, yy = sess.run([samples, data])
        fig = figure(figsize=(5,5))
        scatter(xx[:, 0], xx[:, 1], edgecolor='none')
        scatter(yy[:, 0], yy[:, 1], c='g', edgecolor='none')
        axis('off')
        if generate_movie:
            frames.append(mplfig_to_npimage(fig))
        show()
```

## Visualize results

```python
import seaborn as sns
```

```python
np_samples_ = np_samples[::1]
cols = len(np_samples_)
bg_color  = sns.color_palette('Greens', n_colors=256)[0]
figure(figsize=(2*cols, 2))
for i, samps in enumerate(np_samples_):
    if i == 0:
        ax = subplot(1,cols,1)
    else:
        subplot(1,cols,i+1, sharex=ax, sharey=ax)
    ax2 = sns.kdeplot(samps[:, 0], samps[:, 1], shade=True, cmap='Greens', n_levels=20, clip=[[-xmax,xmax]]*2)
    ax2.set_facecolor(bg_color)
    xticks([]); yticks([])
    title('step %d'%(i*viz_every))
ax.set_ylabel('%d unrolling steps'%params['unrolling_steps'])
gcf().tight_layout()
```

```python
fs = np.array(fs)
plot(fs)
legend(('loss', 'unrolled loss'))
```

```python
plot(fs[:, 0] - fs[:, 1])
legend('optimized loss - initial loss')
```

```python
#clip = mpy.ImageSequenceClip(frames[::], fps=30)
#clip.ipython_display()
```


## Teoría de perturbaciones

Consiste en resolver un sistema perturbado(se conoce la solución al no perturbado), y donde el interés es conocer la contribución de la parte perturbada $H'$ al nuevo sistema total.

$$ H = H^{0} + H'$$

Para sistemas no degenerados, la corrección a la energía a primer orden se calcula como

$$ E_{n}^{(1)} = \int\psi_{n}^{(0)*} H' \psi_{n}^{(0)}d\tau$$

** Tarea 1 : Programar esta ecuación si conoces $H^{0}$ y sus soluciones. **

```python
from IPython.display import clear_output
from sympy import *
from sympy import init_printing;  init_printing(use_latex='mathjax')
import scipy.constants as cc
#from sympy.functions.special.spherical_harmonics import Ynm
import sympy.physics.hydrogen as hidrogeno
import sympy.physics.qho_1d as qho
```

```python
var('x r phi theta L')
var('n m l Z m_e', positive = 'True', integer = 'True')
var('hbar omega a_0 e c', real = 'True', constant = 'True')

def PIAB_F(L,n,x):
    Psi = sqrt(2/L)*sin(n*pi*x/L)
    return Psi
def PIAB_E(n,L,m_e):
    E = (hbar**2*pi**2*n**2)/(2*m_e*L**2)
    return E
```

```python
### Solución
print('         Bienvenido')
print('El Objetivo de este programa es calcular las correciones a las energias a primer orden.')
print('Elija que sistema quiere utilizar:\n Partícula en una caja: PIAB\n Oscilador Cuántico: HO\n Átomo de Hidrógeno: Hydrogen')
res = input()
clear_output()
Hper = sympify(input('Ingrese la perturbación del sistema: '))
lim = (input('Ingrese los limites de la perturbacion: ')).split()
a = sympify(lim[0])
b = sympify(lim[1])
if res == 'PIAB':
    psi = PIAB_F(L,n,x)
    E0 = PIAB_E(n,L,m)
    E1 = Integral(psi*Hper*psi,(x,a,b)).doit()
elif res == 'HO':
    n = int(input('Ingrese el numero cuántico n: '))
    psi = qho.psi_n(n,x,m,omega)
    E0 = qho.E_n(n,omega)
    E1 = Integral(psi*Hper*psi,(x,a,b)).doit().args[0][0]
elif res == 'Hydrogen':
    n = int(input('Ingrese el numero cuántico n: '))
    l = int(input('Ingrese el numero cuántico l: '))
    psi = hidrogeno.R_nl(n,l,r,Z)
    E0 = hidrogeno.E_nl(n,Z)*(e**2/a_0)
    E1 = integrate(psi*Hper*psi*r**2,(r,a,b)).args[0][0]*(e**2/a_0)
else:
    print('Error intente otra vez')

E = Matrix([['E_0', E0], ['E_1', E1], ['E_0+E_1',(E0+E1).simplify()]])
print('La correccion a primer orden a la energía es:')
E
```

    Ingrese la perturbación del sistema: x
    Ingrese los limites de la perturbacion: 0 L
    La correccion a primer orden a la energía es:

$$\left[\begin{matrix}E_{0} & \frac{\pi^{2} \hbar^{2} n^{2}}{2 L^{2} m}\\E_{1} & \frac{L}{2}\\E_{0} + E_{1} & \frac{L}{2} + \frac{\pi^{2} \hbar^{2} n^{2}}{2 L^{2} m}\end{matrix}\right]$$

Y la corrección a la función de onda, también a primer orden, se obtiene como:

 $$ \psi_{n}^{(1)} = \sum_{m\neq n} \frac{\langle\psi_{m}^{(0)} | H' | \psi_{n}^{(0)} \rangle}{E_{n}^{(0)} - E_{m}^{(0)}} \psi_{m}^{(0)}$$

**Tarea 2: Programar esta ecuación si conoces $H^{0}$ y sus soluciones. **

```python
### Solución a las funciones de onda a primer orden
print('         Bienvenido')
print('El Objetivo de este programa es calcular las correciones a las funciones de onda a primer orden.')
print('Elija que sistema quiere utilizar:\n Partícula en una caja: PIAB\n Oscilador Cuántico: HO\n Átomo de Hidrógeno: Hydrogen')
res = input()
clear_output()
Hper = sympify(input('Ingrese la perturbación del sistema: '))
lim = (input('Ingrese los limites de la perturbacion: ')).split()
a = sympify(lim[0])
b = sympify(lim[1])
if res == 'PIAB':
    n = int(input('Ingrese el numero cuántico n: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = PIAB_F(L,n,x)
    PSI = 0
    for m in range(c,d+1):
        if m != n:
            PSI =+ (integrate(PIAB_F(L,m,x)*Hper*psi,(x,a,b))*PIAB_F(L,m,x))/(PIAB_E(n,L,m_e)-PIAB_E(m,L,m_e))
    Psi_1 = psi + PSI
elif res == 'HO':
    n = int(input('Ingrese el numero cuántico n: '))
    m_e = sympify(input('Ingrese la masa de la partícula: '))
    omega = sympify(input('Ingrese la frecuencia de vibración: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = qho.psi_n(n,x,m_e,omega)
    PSI = 0
    for m in range(c,d+1):
        if m != n:
            PSI =+ (integrate(qho.psi_n(m,x,m_e,omega)*Hper*psi,(x,a,b))*qho.psi_n(m,x,m_e,omega))/(qho.E_n(n,omega)-qho.E_n(m,omega))
    Psi_1 = psi + PSI
elif res == 'Hydrogen':
    Z = int(input('Ingrese el numero atómico: '))
    n = int(input('Ingrese el numero cuántico n: '))
    l = int(input('Ingrese el numero cuántico l: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = hidrogeno.R_nl(n,l,r,Z)
    PSI = 0
    for m in range(c,d+1):
        if m != n:
            PSI =+ (integrate(hidrogeno.R_nl(m,l,r,Z)*Hper*psi,(x,a,b))*hidrogeno.R_nl(m,l,r,Z))/(hidrogeno.E_nl(n,Z)-hidrogeno.E_nl(m,Z))
    Psi_1 = psi + PSI
else:
    print('Error intente otra vez')

P = Matrix([['psi_0', psi], ['psi_1', Psi_1], ['psi_0+psi_1',(psi+Psi_1).simplify()]])
print('La correccion a la función de onda a primer orden es:')
P
```

**Tarea 3: Investigue las soluciones a segundo orden y también programe las soluciones. **

```python
### Solución a las energias a segundo orden
print('         Bienvenido')
print('El Objetivo de este programa es calcular las correciones a las energias a segundo orden.')
print('Elija que sistema quiere utilizar:\n Partícula en una caja: PIAB\n Oscilador Cuántico: HO\n Átomo de Hidrógeno: Hydrogen')
r = input()
clear_output()
Hper = sympify(input('Ingrese la perturbación del sistema: '))
lim = (input('Ingrese los limites de la perturbacion: ')).split()
a = sympify(lim[0])
b = sympify(lim[1])
if r == 'PIAB':
    n = int(input('Ingrese el numero cuántico n: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = PIAB_F(L,n,x)
    E0 = PIAB_E(n,L,m_e)
    E1 = Integral(psi*Hper*psi,(x,a,b)).doit()
    E2 = 0
    for m in range(c,d+1):
        if m != n:
            E2 =+ (integrate(PIAB_F(L,m,x)*Hper*psi,(x,a,b)))**2/(PIAB_E(n,L,m_e)-PIAB_E(m,L,m_e))
elif r == 'HO':
    n = int(input('Ingrese el numero cuántico n: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = qho.psi_n(n,x,m_e,omega)
    E0 = qho.E_n(n,omega)
    E1 = Integral(psi*Hper*psi,(x,a,b)).doit()
    E2 = 0
    for m in range(c,d+1):
        if m != n:
            E2 =+ (integrate(qho.psi_n(m,x,m_e,omega)*Hper*psi,(x,a,b)))**2/(qho.E_n(n,omega)-qho.E_n(m,omega))
elif r == 'Hydrogen':
    n = int(input('Ingrese el numero cuántico n: '))
    l = int(input('Ingrese el numero cuántico l: '))
    rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
    c = sympify(rangos_m[0])
    d = sympify(rangos_m[1])
    psi = hidrogeno.R_nl(n,l,r,Z)
    E0 = hidrogeno.E_nl(n,Z)
    E1 = integrate(psi*Hper*psi,(r,a,b))
    E2 = 0
    for m in range(c,d+1):
        if m != n:
            E2 =+ (integrate(hidrogeno.R_nl(m,l,r,Z)*Hper*psi,(r,a,b)))**2/(hidrogeno.E_nl(n,Z)-hidrogeno.E_nl(m,Z))
else:
    print('Error intente otra vez')

E = Matrix([['E_0', E0], ['E_1', E1], ['E_2', E2], ['E_0+E_1+E_2',(E0+E1+E2).simplify()]])
print('La correccion a segundo orden a la energía es:')
E
```

```python
from scipy.integrate import quad
print('         Bienvenido')
print('El Objetivo de este programa es calcular las correciones a las energias a segundo orden.')
#print('Elija que sistema quiere utilizar:\n Partícula en una caja: PIAB\n Oscilador Cuántico: HO\n Átomo de Hidrógeno: Hydrogen')
#r = input()
#clear_output()
Hper = sympify(input('Ingrese la perturbación del sistema: '))
lim = (input('Ingrese los limites de la perturbacion: ')).split()
a = sympify(lim[0])
b = sympify(lim[1])
L = abs(b - a)
n = int(input('Ingrese el numero cuántico n: '))
m_e = float(input('Ingrese la masa de la particula: '))
rangos_m = (input('Ingrese el rango para la funcion de onda "m": ')).split()
c = sympify(rangos_m[0])
d = sympify(rangos_m[1])
psi = PIAB_F(L,n,x)
E0 = PIAB_E(n,L,m_e).subs([(pi,cc.pi),(hbar,cc.hbar)])
E1 = Integral(psi*Hper*psi,(x,a,b)).doit().subs([(pi,cc.pi),(hbar,cc.hbar)])
E2 = 0

for m in range(c,d+1):
    if m != n:
        E2 =+ quad(lambdify(x,psi*Hper*PIAB_F(L,m,x)),a,b)[0]**2/((PIAB_E(1,1,9.31e-31)-PIAB_E(2,1,9.31e-31)).subs([(pi,cc.pi),(hbar,cc.hbar)]))

E = Matrix([['E_0', E0], ['E_1', E1], ['E_2', E2], ['E_0+E_1+E_2',(E0+E1+E2).simplify()]])
print('La correccion a segundo orden a la energía es:')
E
```

             Bienvenido
    El Objetivo de este programa es calcular las correciones a las energias a segundo orden.
    Ingrese la perturbación del sistema: x
    Ingrese los limites de la perturbacion: 0 1
    Ingrese el numero cuántico n: 1
    Ingrese la masa de la particula: 9.31e-31
    Ingrese el rango para la funcion de onda "m": 0 4
    La correccion a segundo orden a la energía es:

$$\left[\begin{matrix}E_{0} & 5.89484481404211 \cdot 10^{-38}\\E_{1} & \frac{1}{2}\\E_{2} & -1.17419924947323 \cdot 10^{33}\\E_{0} + E_{1} + E_{2} & -1.17419924947323 \cdot 10^{33}\end{matrix}\right]$$

**Tarea 4.  Resolver el átomo de helio aplicando los programas anteriores.**

```python
from sympy import *
from sympy import init_printing;  init_printing(use_latex='mathjax')
import sympy.physics.hydrogen as hidrogeno

var('r_1 r_2 e a_0 Z n')
psi_1 = hidrogeno.R_nl(1,0,r_1,2)
psi_2 = hidrogeno.R_nl(1,0,r_2,2)

E = (e**2/a_0)*integrate(r_2**2*psi_2*(integrate((r_1**2/r_2)*psi_1,(r_1,0,r_2)) + integrate((r_1**2/r_1)*psi_1,(r_1,r_2,oo))),(r_2,0,oo))
print('La correcion a primer orden de la energia del átomo de Helio es:')
E*Z/2
```

    La correcion a primer orden de la energia del átomo de Helio es:

$$\frac{5 Z e^{2}}{8 a_{0}}$$

**Tarea 5: Método variacional-perturbativo. **

Este método nos permite estimar de forma precisa $E^{(2)}$ y correcciones perturbativas de la energía de órdenes más elevados para el estado fundamental del sistema, sin evaluar sumas infinitas. Ver ecuación 9.38 del libro.

**Resolver el átomo de helio, considerando este método (sección 9.4), como mejor le parezca. **

**Tarea 6. Revisar sección 9.7. **

Inicialmente a mano, y sengunda instancia favor de intentar programar sección del problema, i.e. integral de Coulomb  e integral de intercambio.

## Siguiente: Segunda parte, Octubre
Simetrías moleculares y Hartree-Fock

```python

```


# Global Sensitivity Analysis with test functions

**Samuele Lo Piano**, [s.lopiano@gmail.com](mailto:s.lopiano@gmail.com)

```
# ipython magic
%matplotlib notebook
%load_ext autoreload
%autoreload 2
```

```
%matplotlib inline

# plot configuration
import matplotlib
import matplotlib.pyplot as plt
plt.style.use("ggplot")
# import seaborn as sns # sets another style
matplotlib.rcParams['lines.linewidth'] = 3
fig_width, fig_height = (7.0,5.0)
matplotlib.rcParams['figure.figsize'] = (fig_width, fig_height)

# font = {'family' : 'sans-serif',
#   'weight' : 'normal',
#   'size' : 18.0}
# matplotlib.rc('font', **font) # pass in the font dict as kwar
```

[Global sensitivity
analysis](https://onlinelibrary.wiley.com/doi/book/10.1002/9780470725184) is an
approached aimed at correlating the uncertainty in the output to the uncertainty
in the input parameters. When dealing with non-additive models, it outperforms
one-variable-at-a-time sensitivity analysis as this latter does not account for
high-order interactions among parameters. This results in incorrect apportion of
output uncertainty onto input parameters.

A statistical property typically used
to account for uncertainty is variance, leading to variance-based sensitivity
analysis as a largely adopted practice.

It may usually happen in models that
not all the input-parameters variability equally affects the output variability.
Typical trends consist in [Pareto
distributions](https://en.wikipedia.org/wiki/Pareto_distribution) having a small
share of the input variables responsible for most of the output variance. Yet
one may come across other situations with important high-order interactions
cutting across (almost) all the input parameters.

The seven analytical
functions studied in
[Kucherenko_et_al_2011](https://www.sciencedirect.com/science/article/pii/S0951832010002437/)
have precisely this aim: represent the possible cases one may come across when
computing sensitivity indices. Three typical situations are present: A-type
functions with not equally important variables, where one can identify a small
set of leading variables while the remainder playing the role of complementary
variables; B-type functions, having (normally more than one) dominant low-order
terms; C-type functions, for which (almost) all of the parameters are important
with significant high-order interactions.

Plenty of algorithms have been
proposed to compute estimators that could adequately account for the above-
mentioned sensitivity indices. Exact analytical expressions are available for
these analytical functions, hance putting one in the condition to estimate the
quality of the assessment performed with these estimators. One can measure the
convergence rate and see how quickly an estimator approaches the actual
sensitivity indices value by enlarging the sample size.

As we shall see in this
notebook, the convergence rate is lower for C-type functions over B-type
functions and finally A-type function, these latter presenting the quickest
convergence. We also compare here the convergence pace of samples based on [Low-
discrepancy Sobol sequency](sobol_interactive.ipynb) against standard random
numbers. More details in the [dedicated appendix](sobol_interactive.ipynb).

## Defining the test functions

One can find the expression of the seven test functions we will be dealing within this notebook in equation 1 ("eq:equation 1": "(#eq:test_functions)" below

$$
\begin{equation}
A1: f(X) = \sum_{j=1}^{k} (-1)^j \, \prod_{l=1}^{j} x_l \
\label{_auto1} \tag{1}
\end{equation}
$$

$$
\begin{equation}
A2: f(X) = \prod_{j=1}^{k}
\frac{|{4x_j}-2|+{a_j}}{1+{a_j}} \
\label{_auto2} \tag{2}
\end{equation}
$$

$$
\begin{equation}
B1: f(X) = \prod_{j=1}^{k} \frac{k-{4x_j}}{k-0.5} \
\label{_auto3} \tag{3}
\end{equation}
$$

$$
\begin{equation}
B2: f(X) = \left(1+\frac{1}{k}\right)^k
\prod_{j=1}^{k} \sqrt{x_j} \
\label{_auto4} \tag{4}
\end{equation}
$$

$$
\begin{equation}
B3: f(X) = \prod_{j=1}^{k} \frac{|{4x_j}-2|+{b_j}}{1+{b_j}} \
\label{_auto5} \tag{5}
\end{equation}
$$

$$
\begin{equation}
C1: f(X) = \prod_{j=1}^{k}
|{4x_j}-2|\
\label{_auto6} \tag{6}
\end{equation}
$$

$$
\begin{equation}
C2: f(X) = 2^k
\prod_{j=1}^{k} x_j \
\label{_auto7} \tag{7}
\end{equation}
$$

In our case we
will be working with a set of six parameters. The reader may change the pool as
s/he pleases along with the dimension of the lists of the *a* and *b*
coefficients. The minimal parameter-pool size able to produce a meaningful
inference is set at three.

```
k = 6 # k represents the number of parameters inquired

# a and b are the constants' vectors used in the G-function expressions
a = [0, 0, 6.52, 6.52, 6.52, 6.52]
b = [6.52, 6.52, 6.52, 6.52, 6.52, 6.52]
```

The *a* list allows to cover the case where one has two leading parameters with
the remaining four in the position of complementarity. Conversely, the *b* list
represents the case where all the parameters are equally important. In general,
the lower the constant !bt a_k !et, the higher the importance of a parameter for
the *G functions* A2, B3 and C1 presented in the study. C1 is the most
challenging one as all the additive constants are set at 0.

The analytical
formulae for the values of the sensitivity indices were retrieved from the
quoted
[kucherenko_2011](https://www.sciencedirect.com/science/article/pii/S0951832010002437/)
or
[saltelli_annoni_2010](https://www.sciencedirect.com/science/article/pii/S0010465509003087/)
(for the functions A1, A2 and B3). The functions analytical values will serve as
external referent when testing the estimators convergence pace.

A dataframe of
low-discrepancy quasi-random numbers in the interval [0,1] of convenient size is
then generated by using a specific algorithm, the [Sobol
sequence](https://en.wikipedia.org/wiki/Sobol_sequence). The sample size is
defined in relation to powers of 2 due to the specific properties of this
algorithm.

The import of a non-standard module, 'sobol_seq', is required. It
can be easily installed by typing `pip install sobol_seq` from your terminal
(Linux) or from the anaconda-prompt environment in case you use Anaconda package
manager.

```
# import modules
import numpy as np
import pandas as pd
import sobol_seq
from string import ascii_lowercase
from testfunctions import create_dict, functions, AE_dic, AEF_dic
```

```
letters = [l for l in ascii_lowercase]

p_sample = []
p_sampleR = []
p_sample_name = []
f_sample = []
f_sampleR = []
p = 12

df = pd.DataFrame(sobol_seq.i4_sobol_generate(2*k, 2**p-4))
df2 = pd.DataFrame(np.random.rand(2**p-4,k*2))
```

```
qamples = []
qamplesr = []
for s in range (2,p):
    qamples.append(df.iloc[(-4+2**s):(-4+2**(s+1))].reset_index(drop=True))
    qamplesr.append(df2.iloc[(-4+2**s):(-4+2**(s+1))].reset_index(drop=True))

    CheckMAE_mean = []
    CheckMAER_mean = []
    CheckMAEF_mean = []
    CheckMAEFR_mean = []

    sample_Matrices = []
    sample_MatricesR = []
    for m in range (0, 2):
        sample_Matrices.append(qamples[s-2].T.iloc[int(m*(len(qamples[s-2].columns)/2)):int((m+1)*(len(qamples[s-2].columns)/2))].reset_index(drop=True).T)
        sample_MatricesR.append(qamplesr[s-2].T.iloc[int(m*(len(qamplesr[s-2].columns)/2)):int((m+1)*(len(qamplesr[s-2].columns)/2))].reset_index(drop=True).T)

    sample_Matrices_dic = create_dict(letters, sample_Matrices)
    sample_MatricesR_dic = create_dict(letters, sample_MatricesR)

    mixed_Matrices = []
    mm_names = []
    mixed_MatricesR = []
    for sm in range (0,len(sample_Matrices)):
        for sm1 in range (0,len(sample_Matrices)):
            if sm == sm1:
                continue
            else:
                for c in sample_Matrices[sm]:
                    mixed_Matrices.append(sample_Matrices[sm].copy())
                    mixed_Matrices[len(mixed_Matrices)-1][c]=sample_Matrices[sm1].copy()[c]
                    mm_names.append(str(letters[sm] + letters[sm1] + str(c+1)))
                    mixed_MatricesR.append(sample_MatricesR[sm].copy())
                    mixed_MatricesR[len(mixed_MatricesR)-1][c]=sample_MatricesR[sm1].copy()[c]

    mixed_Matrices_dic = create_dict(mm_names, mixed_Matrices)
    mixed_MatricesR_dic = create_dict(mm_names, mixed_MatricesR)

    matrices_dic = {**sample_Matrices_dic, **mixed_Matrices_dic}
    matricesR_dic = {**sample_MatricesR_dic, **mixed_MatricesR_dic}

    names1 = []
    values1R = []
    values1 = []
    names2 = []
    values2 = []
    values2R = []
    for f in functions:
        for sq, zq in mixed_Matrices_dic.items():
            names1.append(f.__name__+str(sq))
            values1.append(f(zq))
        for sqR, zqR in mixed_MatricesR_dic.items():
            values1R.append(f(zqR))

        for sM, zM in matrices_dic.items():
            names2.append(f.__name__+str(sM))
            values2.append(f(zM))
        for sMR, zMR in matricesR_dic.items():
            values2R.append(f(zMR))

    f_MM_dic = create_dict(names1, values1)
    f_matrices_dic = create_dict(names2, values2)
    f_MMR_dic = create_dict(names1, values1R)
    f_matricesR_dic = create_dict(names2, values2R)

    Check=[]
    CheckR=[]
    CheckName = []
    Check3=[]
    Check3R=[]
    Check3Name = []
    for f in functions:
        for j in range(1,k+1):
            difference = []
            difference3 = []
            differenceR = []
            difference3R = []
            for mk, mz in f_matrices_dic.items():
                if mk[0:2]==f.__name__:
                    validkeys = []
                    validkeys3 = []
                    for fk1 in f_MM_dic.keys():
                        if len(mk)==3 and mk[2]=='a':
                            if fk1[0:3]==mk[0:3] and fk1[-1]==str(j):
                                validkeys.append(fk1)
                            if fk1[0:2]==mk[0:2] and fk1[2]!=mk[2] and fk1[-1]==str(j):
                                validkeys3.append(fk1)
                    z1 = dict(filter(lambda i:i[0] in validkeys, f_MM_dic.items()))
                    z3 = dict(filter(lambda i3:i3[0] in validkeys3, f_MM_dic.items()))
                    for zk, zv in z1.items():
                        difference.append(0.5*(((mz-zv)**2).mean())/mz.var())
                    for zk3, zv3 in z3.items():
                        difference3.append(((mz*zv3).mean()-mz.mean()**2)/mz.var())
            Check.append(sum(difference)/len(difference))
            CheckName.append('Jansen'+ str(f.__name__) +'ST'+str(j))
            Check3.append(sum(difference3)/len(difference3))
            Check3Name.append('Sobol'+ str(f.__name__) +'S'+str(j))
            for mkR, mzR in f_matricesR_dic.items():
                if mkR[0:2]==f.__name__:
                    validkeysR = []
                    validkeys3R = []
                    for fk1R in f_MMR_dic.keys():
                        if len(mkR)==3 and mkR[2]=='a':
                            if fk1R[0:3]==mkR[0:3] and fk1R[-1]==str(j):
                                validkeysR.append(fk1R)
                            if fk1R[0:2]==mkR[0:2] and fk1R[2]!=mkR[2] and fk1R[-1]==str(j):
                                validkeys3R.append(fk1R)
                    z1R = dict(filter(lambda iR:iR[0] in validkeysR, f_MMR_dic.items()))
                    z3R = dict(filter(lambda iR3:iR3[0] in validkeys3R, f_MMR_dic.items()))
                    for zkR, zvR in z1R.items():
                        differenceR.append(0.5*(((mzR-zvR)**2).mean())/mzR.var())
                    for zk3R, zv3R in z3R.items():
                        difference3R.append(((mzR*zv3R).mean()-mzR.mean()**2)/mzR.var())
            CheckR.append(sum(differenceR)/len(differenceR))
            Check3R.append(sum(difference3R)/len(difference3R))
    Check_dic = create_dict(CheckName, Check)
    Check3_dic = create_dict(Check3Name, Check3)
    CheckR_dic = create_dict(CheckName, CheckR)
    Check3R_dic = create_dict(Check3Name, Check3R)

    CheckMAEs = []
    CheckMAEsR = []
    CheckMAENames = []
    CheckMAEsF = []
    CheckMAEsFR = []
    CheckMAEFNames = []
    for ae, av in AE_dic.items():
        for Lk, Lv in Check_dic.items():
            if ae[-5:]==Lk[-5:]:
                CheckMAEs.append(abs(Lv-av))
                CheckMAENames.append('CheckMAE'+ str(ae[2:4]) + str(ae[-1]))
        for LkR, LvR in CheckR_dic.items():
            if ae[-5:]==LkR[-5:]:
                CheckMAEsR.append(abs(LvR-av))
    for af, afv in AEF_dic.items():
        for Lk3, Lv3 in Check3_dic.items():
            if af[-4:]==Lk3[-4:]:
                CheckMAEsF.append(abs(Lv3-afv))
                CheckMAEFNames.append('CheckMAE'+ str(af[2:4]) + str(af[-1]))
        for Lk3R, Lv3R in Check3R_dic.items():
            if af[-4:]==Lk3R[-4:]:
                CheckMAEsFR.append(abs(Lv3R-afv))
    CheckMAEs_dic = create_dict(CheckMAENames, CheckMAEs)
    CheckMAEsF_dic = create_dict(CheckMAEFNames, CheckMAEsF)
    CheckMAEsR_dic = create_dict(CheckMAENames, CheckMAEsR)
    CheckMAEsFR_dic = create_dict(CheckMAEFNames, CheckMAEsFR)

    CheckMAE = []
    CheckMAER = []
    CheckMAE_name = []
    CheckMAEF = []
    CheckMAEFR = []
    for f in functions:
        validkeys2 = []
        validkeys4 = []
        validkeys2R = []
        validkeys4R = []
        for Lmk, Lmv in CheckMAEs_dic.items():
            if Lmk[-3:-1]==f.__name__:
                 validkeys2.append(Lmk)
        for Fmk, Fmv in CheckMAEsF_dic.items():
            if Fmk[-3:-1]==f.__name__:
                 validkeys4.append(Fmk)
        z2 = dict(filter(lambda i2:i2[0] in validkeys2, CheckMAEs_dic.items()))
        z4 = dict(filter(lambda i4:i4[0] in validkeys4, CheckMAEsF_dic.items()))
        CheckMAE.append(sum(z2.values())/len(z2))
        CheckMAE_name.append('CheckMAE'+f.__name__)
        CheckMAEF.append(sum(z4.values())/len(z4))
        for LmkR, LmvR in CheckMAEsR_dic.items():
            if LmkR[-3:-1]==f.__name__:
                validkeys2R.append(LmkR)
        for FmkR, FmvR in CheckMAEsFR_dic.items():
            if FmkR[-3:-1]==f.__name__:
                 validkeys4R.append(FmkR)
        z2R = dict(filter(lambda i2R:i2R[0] in validkeys2R, CheckMAEsR_dic.items()))
        z4R = dict(filter(lambda i4R:i4R[0] in validkeys4R, CheckMAEsFR_dic.items()))
        CheckMAER.append(sum(z2R.values())/len(z2R))
        CheckMAEFR.append(sum(z4R.values())/len(z4R))
    CheckMAE_dic = create_dict(CheckMAE_name, CheckMAE)
    CheckMAEF_dic = create_dict(CheckMAE_name, CheckMAEF)
    CheckMAER_dic = create_dict(CheckMAE_name, CheckMAER)
    CheckMAEFR_dic = create_dict(CheckMAE_name, CheckMAEFR)
    CheckMAE_mean.append(CheckMAE_dic)
    CheckMAER_mean.append(CheckMAER_dic)
    CheckMAEF_mean.append(CheckMAEF_dic)
    CheckMAEFR_mean.append(CheckMAEFR_dic)
    CheckMAE_mean_dic = {Lmk1:[CheckMAE_mean[Lmv1][Lmk1] for Lmv1 in range(len(CheckMAE_mean))] for Lmk1 in CheckMAE_mean[0].keys()}
    CheckMAER_mean_dic = {Lmk2:[CheckMAER_mean[Lmv2][Lmk2] for Lmv2 in range(len(CheckMAER_mean))] for Lmk2 in CheckMAER_mean[0].keys()}
    CheckMAEF_mean_dic = {Lmk3:[CheckMAEF_mean[Lmv3][Lmk3] for Lmv3 in range(len(CheckMAEF_mean))] for Lmk3 in CheckMAEF_mean[0].keys()}
    CheckMAEFR_mean_dic = {Lmk4:[CheckMAEFR_mean[Lmv4][Lmk4] for Lmv4 in range(len(CheckMAEFR_mean))] for Lmk4 in CheckMAEFR_mean[0].keys()}
    for Lmk1,Lmv1 in CheckMAE_mean_dic.items():
        CheckMAE_mean_dic[Lmk1] = sum(CheckMAE_mean_dic[Lmk1])/len(CheckMAE_mean_dic[Lmk1])
    for Lmk2,Lmv2 in CheckMAER_mean_dic.items():
        CheckMAER_mean_dic[Lmk2] = sum(CheckMAER_mean_dic[Lmk2])/len(CheckMAER_mean_dic[Lmk2])
    for Lmk3,Lmv3 in CheckMAEF_mean_dic.items():
        CheckMAEF_mean_dic[Lmk3] = sum(CheckMAEF_mean_dic[Lmk3])/len(CheckMAEF_mean_dic[Lmk3])
    for Lmk4,Lmv4 in CheckMAEFR_mean_dic.items():
        CheckMAEFR_mean_dic[Lmk4] = sum(CheckMAEFR_mean_dic[Lmk4])/len(CheckMAEFR_mean_dic[Lmk4])

    p_sample.append(CheckMAE_mean_dic)
    p_sample_name.append(2**s)
    f_sample.append(CheckMAER_mean_dic)
    p_sampleR.append(CheckMAEF_mean_dic)
    f_sampleR.append(CheckMAEFR_mean_dic)
    p_sample_dic = create_dict(p_sample_name, p_sample)
    f_sample_dic = create_dict(p_sample_name, f_sample)
    p_sampleR_dic = create_dict(p_sample_name, p_sampleR)
    f_sampleR_dic = create_dict(p_sample_name, f_sampleR)

Checkq = pd.DataFrame.from_dict(p_sample_dic)
Checkf = pd.DataFrame.from_dict(f_sample_dic)
CheckR = pd.DataFrame.from_dict(p_sampleR_dic)
CheckfR = pd.DataFrame.from_dict(f_sampleR_dic)
```

The Sobol matrix gets sliced in a way that every sample is twice bigger than the
previous `qamples` list elements. The first one has only two rows. From
iteration to iteration the number gets increased as 2 to the power of s, up to
2,048 rows for the last one. The `qamples` matrix gets in turn sliced in 50
parts of equivalent size. The rationale behind this operation is generating an
adequate numbers of runs whose output can be averaged to compensate for
fluctuations.

Two sample matrices are generated by slicing down the larger
matrix in two parts of equal length. The code line `sample_Matrices_dic` creates
a dictionary by appending the name to the matrices in the order they have been
generated (a, b). This operation can be ideally replicated in the case one
wishes to compute more sample matrices for the estimators algorithm.

The mixed
matrices (i.e. ab1, ab6, ba2, etc.) are generated by scrambling the columns of
the sample matrices. The first *if* excludes counting the same matrix twice, the
second lower level *for* loop replicates the matrices the number of parameters
one has and for each scrambles the relative column. Finally, the name is
appended dependent the original matrix, the matrix whose column has been used
for the scrambling and the number of the column. The label is eventually
associated to the mixed matrices in the same way as done for the sample
matrices. The sample and mixed matrices are zipped together into a
`matrices_dic`.

The functions are applied to the sets of matrices and the
result of the operations are stored into dictionaries again. Now each item in
the dictionaries `f_MM_dic` and `f_matrices_dic` is a vector rather than a
matrix (f(a), f(b), f(ab4), etc.)

For each function, the sample matrices are
selected ('if len(mk)==3:'). From the ensemble of the scramble matrices, those
having the same function and starting letter are selected for the total
sensivity index accounting. While a different starting letter is the condition
for the first-order estimator (fk1[2]!=mk[2]). The selection is based on the
names (keys). The dictionary is filtered according to these criteria and the
Jansen estimator-to-variance ratio is computed for the higher order (`Check` and
`CheckR`) and the Sobol estimator-to-variance ratio for the first order matrices
(`Check3` and `Check3R`).

Finally, the Mean Absolute Errors, the difference
between the analytical value and the estimator figure, are computed and the
values appended in a dictionary.

These figures are then averaged on the
different parameters (from six figures to one) as well as runs and stored in a
dictionary `CheckMAE_dic` and `CheckMAEF_dic`.

Which is in turn appended onto
the final dictionary wrapping up all the experiments performed (up to the power
of 2 'p' initially defined). This latter is eventually converted into a
convenient *pandas* dataframe from which the trends can be easily plotted or
statistically inference on the figures can be produced.

```
for ind, row in Checkq.iterrows():
 for indR, row in CheckR.iterrows():
  if ind == indR:
   x_vals = Checkq.columns.values
   y1 = Checkq.loc[ind]
   y2 = CheckR.loc[indR]
   for i6 in range(0, len(x_vals), 1):
    plt.loglog(x_vals[i6:i6+2], y1[i6:i6+2], c = "b", marker = "o", label = 'QR' if i6 == 0 else '')
    plt.loglog(x_vals[i6:i6+2], y2[i6:i6+2], c = "r", marker = "x", label ='R' if i6 == 0 else '')
    plt.xlim(0,8500)
   plt.title('Jansen' + '_ST_' +str(ind[-2:]))
   plt.legend()
   plt.show()
```

A series of plots is finally produced whose trend is compared against the sample
size across functions in order to evaluate the convergence rate of the higher-
order sensitivity indices. The convergence paces appear to be slower for C-type
functions. For these latter, the convergence pace of Quasi-random-based
functions against random-based functions appears to be similar and there would
be no advantage in using a low-discrepancy sequence.

```
for ind, row in Checkf.iterrows():
 for indR, row in CheckfR.iterrows():
  if ind == indR:
   x_vals = Checkf.columns.values
   y1 = Checkf.loc[ind]
   y2 = CheckfR.loc[indR]
   for i6 in range(0, len(x_vals), 1):
    plt.loglog(x_vals[i6:i6+2], y1[i6:i6+2], c = "b", marker = "o", label = 'QR' if i6 == 0 else '')
    plt.loglog(x_vals[i6:i6+2], y2[i6:i6+2], c = "r", marker = "x", label ='R' if i6 == 0 else '')
    plt.xlim(0,8500)
   plt.title('Sobol' + '_S_' +str(ind[-2:]))
   plt.legend()
   plt.show()
```

The same operation can be repeated for first-order indices. The inference on the
trends is analogous to high-order indices.

## Conclusions

The convergence pace
is an aspect an analyst should have clear in mind when designing the experiment
to account for the sensitivity indices: complicated models with plenty of
relevant higher-order interactions would call for larger sample sizes and
therefore longer and more computational demanding experiments to adequately
estimate the sensitivity indices. Low-discrepancy sequences can play a role in
easing the correct computation of sensitivity indices as long as high-order
terms are not dominant for most of the input parameters.


```python
import sys
sys.path.append('..')
import torch
import numpy as np
import matplotlib.pyplot as plt
from sympy import simplify_logic

from lens.utils.base import validate_network
from lens.utils.relu_nn import get_reduced_model, prune_features
from lens import logic
import lens

torch.manual_seed(0)
np.random.seed(0)
```

```python
# XOR problem
x_train = torch.tensor([
    [0, 0],
    [0, 1],
    [1, 0],
    [1, 1],
], dtype=torch.float)
y_train = torch.tensor([0, 1, 1, 0], dtype=torch.long)
n_classes = len(torch.unique(y_train))

x_test = torch.tensor([
    [0, 0.95],
    [0, 0.9],
    [0.05, 1],
    [0.1, 0.8],
    [0.45, 1],
    [0, 0.35],
    [0.95, 0.9],
    [0.75, 0.2],
    [0.75, 0.15],
], dtype=torch.float)
y_test = torch.tensor([1, 1, 1, 1, 1, 0, 0, 1, 1], dtype=torch.long)
```

```python
layers = [
    torch.nn.Linear(x_train.size(1), 10 * n_classes),
    torch.nn.ReLU(),
    lens.nn.XLinear(10, 4, n_classes),
    torch.nn.ReLU(),
    lens.nn.XLinear(4, 1, n_classes),
    torch.nn.Sigmoid(),
]
model = torch.nn.Sequential(*layers)

optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
loss_form = torch.nn.CrossEntropyLoss()
model.train()
need_pruning = True
for epoch in range(400):
    # forward pass
    optimizer.zero_grad()
    y_pred = model(x_train)
    # Compute Loss
    loss = loss_form(y_pred, y_train)

    for module in model.children():
        if isinstance(module, torch.nn.Linear):
            loss += 0.001 * torch.norm(module.weight, 1)

    # backward pass
    loss.backward()
    optimizer.step()

    if epoch > 200 and need_pruning:
        prune_features(model, n_classes)
        need_pruning = False

    # compute accuracy
    if epoch % 100 == 0:
        y_pred_d = torch.argmax(y_pred, dim=1)
        accuracy = y_pred_d.eq(y_train).sum().item() / y_train.size(0)
        print(f'Epoch {epoch}: train accuracy: {accuracy:.4f}')
```

    Epoch 0: train accuracy: 0.5000
    Epoch 100: train accuracy: 1.0000
    Epoch 200: train accuracy: 1.0000
    Epoch 300: train accuracy: 1.0000

# Decision boundaries

```python
def plot_decision_bundaries(model, x, h=0.1, cmap='BrBG', torch_model=True, target_class=0):
    x1_min, x1_max = x[:, 0].min() - 1, x[:, 0].max() + 1
    x2_min, x2_max = x[:, 1].min() - 1, x[:, 1].max() + 1
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, h),
                           np.arange(x2_min, x2_max, h))
    if torch_model:
        xx = torch.FloatTensor(np.c_[xx1.ravel(), xx2.ravel()])
        Z = model(xx).detach().numpy()[:, target_class]
    else:
        xx = np.c_[xx1.ravel(), xx2.ravel()]
        Z = model.predict_proba(xx)[:, 1]
    Z = Z.reshape(xx1.shape)
    plt.contourf(xx1, xx2, Z, alpha=0.2, cmap=cmap)
    return
```

## True decision boundaries

```python
xnp = x_train.detach().numpy()
ynp = y_train.detach().numpy().ravel()
cmap='BrBG'

plt.figure(figsize=[8, 4])
plt.subplot(121)
plt.title(f'Decision boundaries class 0')
plot_decision_bundaries(model, x_train, h=0.01, cmap=cmap)
plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
plt.xlim([-0.03, 1.03])
plt.ylim([-0.03, 1.03])
plt.subplot(122)
plt.title(f'Decision boundaries class 1')
plot_decision_bundaries(model, x_train, h=0.01, cmap=cmap, target_class=1)
plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
plt.xlim([-0.03, 1.03])
plt.ylim([-0.03, 1.03])
plt.show()
```

## Local decision boundaries

```python
for sample_id in range(len(y_train)):
    xin = x_train[sample_id]
    model_reduced = get_reduced_model(model, xin)
    for module in model_reduced.children():
        if isinstance(module, torch.nn.Linear):
            wa = module.weight.detach().numpy()
            break
    output = model_reduced(xin)
    target_class = torch.argmax(output).item()
    explanation = logic.relu_nn.explain_local(model, x_train, y_train,
                                           xin, concept_names=['f1', 'f2'])

    model_class = target_class if y_train[sample_id] == 1 else 1-target_class

    plt.figure(figsize=[16, 4])
    plt.subplot(141)
    plt.title(f'True decision boundary class 0')
    plot_decision_bundaries(model, x_train, h=0.01, cmap=cmap, target_class=target_class)
    plt.scatter(xin[0], xin[1], c='k', marker='x', s=100)
    c = plt.Circle((xin[0], xin[1]), radius=0.2, edgecolor='k', fill=False, linestyle='--')
    plt.gca().add_artist(c)
    plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
    plt.xlim([-0.5, 1.5])
    plt.ylim([-0.5, 1.5])
    plt.subplot(142)
    plt.title(f'True decision boundary class 1')
    plot_decision_bundaries(model, x_train, h=0.01, cmap=cmap, target_class=1-target_class)
    plt.scatter(xin[0], xin[1], c='k', marker='x', s=100)
    c = plt.Circle((xin[0], xin[1]), radius=0.2, edgecolor='k', fill=False, linestyle='--')
    plt.gca().add_artist(c)
    plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
    plt.xlim([-0.5, 1.5])
    plt.ylim([-0.5, 1.5])
    plt.subplot(143)
    plt.title(f'IN={xin.detach().numpy()} - OUT={output.detach().numpy()}\nW={wa}\nExplanation: {explanation}')
    plot_decision_bundaries(model_reduced, x_train, target_class=model_class)
    plt.scatter(xin[0], xin[1], c='k', marker='x', s=100)
    c = plt.Circle((xin[0], xin[1]), radius=0.2, edgecolor='k', fill=False, linestyle='--')
    plt.gca().add_artist(c)
    plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
    plt.xlim([-0.5, 1.5])
    plt.ylim([-0.5, 1.5])
    plt.subplot(144)
    plt.title(f'IN={xin.detach().numpy()} - OUT={output.detach().numpy()}\nW={wa}\nExplanation: {explanation}')
    plot_decision_bundaries(model_reduced, x_train, target_class=1-model_class)
    plt.scatter(xin[0], xin[1], c='k', marker='x', s=100)
    c = plt.Circle((xin[0], xin[1]), radius=0.2, edgecolor='k', fill=False, linestyle='--')
    plt.gca().add_artist(c)
    plt.scatter(xnp[:, 0], xnp[:, 1], c=ynp, cmap=cmap)
    plt.xlim([-0.5, 1.5])
    plt.ylim([-0.5, 1.5])
    plt.show()
```

# Combine local explanations

```python
explanation, predictions, counter = logic.relu_nn.combine_local_explanations(model, x_train, y_train,
                                                                         target_class=0,
                                                                          concept_names=['x1', 'x2'])
explanation
```

    '(x1 & x2) | (~x1 & ~x2)'

```python
explanation, predictions, counter = logic.relu_nn.combine_local_explanations(model, x_train, y_train,
                                                                         target_class=1,
                                                                          concept_names=['x1', 'x2'])
explanation
```


# Optimizer tweaks

```python
%load_ext autoreload
%autoreload 2

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

%matplotlib inline
```

```python
#export
from exp.nb_08 import *
```

```python
listify??
```

## Imagenette data

We grab the data from the previous notebook.

```python
path = datasets.untar_data(datasets.URLs.IMAGENETTE_160)
```

```python
tfms = [make_rgb, ResizeFixed(128), to_byte_tensor, to_float_tensor]
bs=128

il = ImageList.from_files(path, tfms=tfms)
sd = SplitData.split_by_func(il, partial(grandparent_splitter, valid_name='val'))
ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())
data = ll.to_databunch(bs, c_in=3, c_out=10, num_workers=4) # RGB 3 in_channels, 10 output channels, 10 object classes
```

Then a model:

```python
nfs = [32,64,128,256]
```

```python
cbfs = [partial(AvgStatsCallback,accuracy), CudaCallback,
        partial(BatchTransformXCallback, norm_imagenette)]
```

This is the baseline of training with vanilla SGD.

get_learn_run? get_learn_run(nfs, data, lr, layer, cbs=None, opt_func=None, **kwargs)

```python
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs)
```

learn object has data, loss_func, opt, model

```python
# learn.model
learn.loss_func
```

    <function torch.nn.functional.cross_entropy(input, target, weight=None, size_average=None, ignore_index=-100, reduce=None, reduction='mean')>

```python
run.fit(1, learn)
```

    train: [1.7890693163971614, tensor(0.3789, device='cuda:0')]
    valid: [1.8528590087890624, tensor(0.4120, device='cuda:0')]

## Refining the optimizer

In PyTorch, the base optimizer in `torch.optim` is just a dictionary that stores the hyper-parameters and references to the parameters of the model we want to train in parameter groups (different groups can have different learning rates/momentum/weight decay... which is what lets us do discriminative learning rates).

It contains a method `step` that will update our parameters with the gradients and a method `zero_grad` to detach and zero the gradients of all our parameters.

We build the equivalent from scratch, only ours will be more flexible. In our implementation, the step function loops over all the parameters to execute the step using stepper functions that we have to provide when initializing the optimizer.

```python
class Optimizer():
    def __init__(self, params, steppers, **defaults):
        # might be a generator
        self.param_groups = list(params)
        # ensure params is a list of lists
        if not isinstance(self.param_groups[0], list): self.param_groups = [self.param_groups]
        self.hypers = [{**defaults} for p in self.param_groups]
        self.steppers = listify(steppers)

    def grad_params(self):
        return [(p,hyper) for pg,hyper in zip(self.param_groups,self.hypers)
            for p in pg if p.grad is not None]

    def zero_grad(self):
        for p,hyper in self.grad_params():
            p.grad.detach_()
            p.grad.zero_()

    def step(self):
        for p,hyper in self.grad_params(): compose(p, self.steppers, **hyper)
```

```python
compose??
```

To do basic SGD, this what a step looks like:

```python
a.add??
```

```python
a = tensor([1,2,3])
a.add_(2, 10) # add the product of the 2x10 to a
```

    tensor([21, 22, 23])

```python
#export
def sgd_step(p, lr, **kwargs):
    p.data.add_(-lr, p.grad.data)
    return p
```

```python
opt_func = partial(Optimizer, steppers=[sgd_step])
```

Now that we have changed the optimizer, we will need to adjust the callbacks that were using properties from the PyTorch optimizer: in particular the hyper-parameters are in the list of dictionaries `opt.hypers` (PyTorch has everything in the the list of param groups).

```python
#export
class Recorder(Callback):
    def begin_fit(self): self.lrs,self.losses = [],[]

    def after_batch(self):
        if not self.in_train: return
        self.lrs.append(self.opt.hypers[-1]['lr'])
        self.losses.append(self.loss.detach().cpu())

    def plot_lr  (self): plt.plot(self.lrs)
    def plot_loss(self): plt.plot(self.losses)

    def plot(self, skip_last=0):
        losses = [o.item() for o in self.losses]
        n = len(losses)-skip_last
        plt.xscale('log')
        plt.plot(self.lrs[:n], losses[:n])

class ParamScheduler(Callback):
    _order=1
    def __init__(self, pname, sched_funcs):
        self.pname,self.sched_funcs = pname,listify(sched_funcs)

    def begin_batch(self):
        if not self.in_train: return
        fs = self.sched_funcs
        if len(fs)==1: fs = fs*len(self.opt.param_groups)
        pos = self.n_epochs/self.epochs
        for f,h in zip(fs,self.opt.hypers): h[self.pname] = f(pos)

class LR_Find(Callback):
    _order=1
    def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
        self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr
        self.best_loss = 1e9

    def begin_batch(self):
        if not self.in_train: return
        pos = self.n_iter/self.max_iter
        lr = self.min_lr * (self.max_lr/self.min_lr) ** pos
        for pg in self.opt.hypers: pg['lr'] = lr

    def after_step(self):
        if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
            raise CancelTrainException()
        if self.loss < self.best_loss: self.best_loss = self.loss
```

So let's check we didn't break anything and that recorder and param scheduler work properly.

```python
sched = combine_scheds([0.3, 0.7], [sched_cos(0.3, 0.6), sched_cos(0.6, 0.2)])
```

```python
cbfs = [partial(AvgStatsCallback,accuracy),
        CudaCallback, Recorder,
        partial(ParamScheduler, 'lr', sched)]
```

```python
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs, opt_func=opt_func)
```

```python
%time run.fit(1, learn)
```

    train: [1.7651078626686831, tensor(0.3931, device='cuda:0')]
    valid: [1.3783167724609375, tensor(0.5440, device='cuda:0')]
    CPU times: user 4.74 s, sys: 1.37 s, total: 6.11 s
    Wall time: 11.2 s

```python
run.recorder.plot_loss()
```

```python
run.recorder.plot_lr()
```

## Weight decay

By letting our model learn high parameters, it might fit all the data points in the training set with an over-complex function that has very sharp changes, which will lead to overfitting.

Weight decay comes from the idea of L2 regularization, which consists in adding to your loss function the sum of all the weights squared. Why do that? Because when we compute the gradients, it will add a contribution to them that will encourage the weights to be as small as possible.

Limiting our weights from growing too much is going to hinder the training of the model, but it will yield to a state where it generalizes better. Going back to the theory a little bit, weight decay (or just `wd`) is a parameter that controls that sum of squares we add to our loss:
``` python
loss_with_wd = loss + (wd/2) * (weights**2).sum()
```

In practice though, it would be very inefficient (and maybe numerically unstable) to compute that big sum and add it to the loss. If you remember a little bit of high school math, the derivative of `p**2` with respect to `p` is `2*p`. So adding that big sum to our loss is exactly the same as doing:
``` python
weight.grad += wd * weight
```

for every weight in our model, which in the case of vanilla SGD is equivalent to updating the parameters with:
``` python
weight = weight - lr*(weight.grad + wd*weight)
```

This technique is called "weight decay", as each weight is decayed by a factor `lr * wd`, as it's shown in this last formula.

This only works for standard SGD, as we have seen that with momentum, RMSProp and Adam, the update has some additional formulas around the gradient. In those cases, the formula that comes from L2 regularization:
``` python
weight.grad += wd * weight
```
is different than weight decay
``` python
new_weight = weight - lr * weight.grad - lr * wd * weight
```

Most libraries use the first one, but as it was pointed out in [Decoupled Weight Regularization](https://arxiv.org/pdf/1711.05101.pdf) by Ilya Loshchilov and Frank Hutter, it is better to use the second one with the Adam optimizer, which is why fastai made it its default.

Weight decay is subtracting `lr*wd*weight` from the weights. We need this function to have an attribute `_defaults` so that we are sure there is an hyper-parameter of the same name in our `Optimizer`.

```python
#export
def weight_decay(p, lr, wd, **kwargs):
    p.data.mul_(1 - lr*wd)
    return p
weight_decay._defaults = dict(wd=0.)
```

L2 regularization is adding `wd*weight` to the gradients.

```python
#export
def l2_reg(p, lr, wd, **kwargs):
    p.grad.data.add_(wd, p.data)
    return p
l2_reg._defaults = dict(wd=0.)
```

Let's allow steppers to add to our `defaults` (which are the default values of all the hyper-parameters). This helper function adds in `dest` the key/values it finds while going through `os` and applying `f` when they was no `key` of the same name.

```python
#export
def maybe_update(os, dest, f):
    for o in os:
        for k,v in f(o).items():
            if k not in dest: dest[k] = v

def get_defaults(d): return getattr(d,'_defaults',{})
```

This is the same as before, we just take the default values of the steppers when none are provided in the kwargs.

```python
#export
class Optimizer():
    def __init__(self, params, steppers, **defaults):
        self.steppers = listify(steppers)
        maybe_update(self.steppers, defaults, get_defaults)
        # might be a generator
        self.param_groups = list(params)
        # ensure params is a list of lists
        if not isinstance(self.param_groups[0], list): self.param_groups = [self.param_groups]
        self.hypers = [{**defaults} for p in self.param_groups]

    def grad_params(self):
        return [(p,hyper) for pg,hyper in zip(self.param_groups,self.hypers)
            for p in pg if p.grad is not None]

    def zero_grad(self):
        for p,hyper in self.grad_params():
            p.grad.detach_()
            p.grad.zero_()

    def step(self):
        for p,hyper in self.grad_params(): compose(p, self.steppers, **hyper)
```

```python
#export
sgd_opt = partial(Optimizer, steppers=[weight_decay, sgd_step])
```

```python
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs, opt_func=sgd_opt)
```

Before trying to train, let's check the behavior works as intended: when we don't provide a value for `wd`, we pull the corresponding default from `weight_decay`.

```python
model = learn.model
```

```python
opt = sgd_opt(model.parameters(), lr=0.1)
test_eq(opt.hypers[0]['wd'], 0.)
test_eq(opt.hypers[0]['lr'], 0.1)
```

But if we provide a value, it overrides the default.

```python
opt = sgd_opt(model.parameters(), lr=0.1, wd=1e-4)
test_eq(opt.hypers[0]['wd'], 1e-4)
test_eq(opt.hypers[0]['lr'], 0.1)
```

Now let's fit.

```python
cbfs = [partial(AvgStatsCallback,accuracy), CudaCallback]
```

```python
learn,run = get_learn_run(nfs, data, 0.3, conv_layer, cbs=cbfs, opt_func=partial(sgd_opt, wd=0.01))
```

```python
run.fit(1, learn)
```

    train: [1.8041583961047774, tensor(0.3763, device='cuda:0')]
    valid: [1.9106278076171874, tensor(0.3480, device='cuda:0')]

This is already better than the baseline!

## With momentum

Momentum requires to add some state. We need to save the moving average of the gradients to be able to do the step and store this inside the optimizer state. To do this, we introduce statistics. Statistics are object with two methods:
- `init_state`, that returns the initial state (a tensor of 0. for the moving average of gradients)
- `update`, that updates the state with the new gradient value

We also read the `_defaults` values of those objects, to allow them to provide default values to hyper-parameters.

```python
#export
class StatefulOptimizer(Optimizer):
    def __init__(self, params, steppers, stats=None, **defaults):
        self.stats = listify(stats)
        maybe_update(self.stats, defaults, get_defaults)
        super().__init__(params, steppers, **defaults)
        self.state = {}

    def step(self):
        for p,hyper in self.grad_params():
            if p not in self.state:
                #Create a state for p and call all the statistics to initialize it.
                self.state[p] = {}
                maybe_update(self.stats, self.state[p], lambda o: o.init_state(p))
            state = self.state[p]
            for stat in self.stats: state = stat.update(p, state, **hyper)
            compose(p, self.steppers, **state, **hyper)
            self.state[p] = state
```

```python
#export
class Stat():
    _defaults = {}
    def init_state(self, p): raise NotImplementedError
    def update(self, p, state, **kwargs): raise NotImplementedError
```

Here is an example of `Stat`:

```python
class AverageGrad(Stat):
    _defaults = dict(mom=0.9)

    def init_state(self, p): return {'grad_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, mom, **kwargs):
        state['grad_avg'].mul_(mom).add_(p.grad.data)
        return state
```

```python
a = tensor([[1,2,3], [4,5, 6]])
torch.zeros_like(a)
torch.zeros(3,4)
```

    tensor([[0, 0, 0],
            [0, 0, 0]])

    tensor([[0., 0., 0., 0.],
            [0., 0., 0., 0.],
            [0., 0., 0., 0.]])

Then we add the momentum step (instead of using the gradients to perform the step, we use the average).

```python
#export
def momentum_step(p, lr, grad_avg, **kwargs):
    p.data.add_(-lr, grad_avg)
    return p
```

```python
sgd_mom_opt = partial(StatefulOptimizer, steppers=[momentum_step,weight_decay],
                  stats=AverageGrad(), wd=0.01)
```

```python
learn,run = get_learn_run(nfs, data, 0.3, conv_layer, cbs=cbfs, opt_func=sgd_mom_opt)
```

```python
run.fit(1, learn)
```

    train: [1.749207631698852, tensor(0.3990, device='cuda:0')]
    valid: [1.9519072265625, tensor(0.3440, device='cuda:0')]

### Momentum experiments

What does momentum do to the gradients exactly? Let's do some plots to find out!

```python
x = torch.linspace(-4, 4, 200)
y = torch.randn(200) + 0.3
betas = [0.5, 0.7, 0.9, 0.99]
x.mean();y.mean()
```

    tensor(-5.4836e-08)

    tensor(0.2762)

```python
def plot_mom(f):
    _,axs = plt.subplots(2,2, figsize=(12,8))
    for beta,ax in zip(betas, axs.flatten()):
        ax.plot(y, linestyle='None', marker='.')
        avg,res = None,[]
        for i,yi in enumerate(y):
            avg,p = f(avg, beta, yi, i)
            res.append(p)
        ax.plot(res, color='red')
        ax.set_title(f'beta={beta}')
```

This is the regular momentum.

```python
def mom1(avg, beta, yi, i):
    if avg is None: avg=yi
    res = beta*avg + yi
    return res,res
plot_mom(mom1)
```

As we can see, with a too high value, it may go way too high with no way to change its course.

Another way to smooth noisy data is to do an exponentially weighted moving average. In this case, there is a dampening of (1-beta) in front of the new value, which is less trusted than the current average. We'll define `lin_comb` (*linear combination*) to make this easier (note that in the lesson this was named `ewma`).

```python
#export
def lin_comb(v1, v2, beta): return beta*v1 + (1-beta)*v2
```

```python
def mom2(avg, beta, yi, i):
    if avg is None: avg=yi
    avg = lin_comb(avg, yi, beta)
    return avg, avg
plot_mom(mom2)
```

We can see it gets to a zero-constant when the data is purely random. If the data has a certain shape, it will get that shape (with some delay for high beta).

```python
y = 1 - (x/3) ** 2 + torch.randn(200) * 0.1
```

```python
y[0]=0.5
```

```python
plot_mom(mom2)
```

Debiasing is here to correct the wrong information we may have in the very first batch. The debias term corresponds to the sum of the coefficient in our moving average. At the time step i, our average is:

$\begin{align*}
avg_{i} &= \beta\ avg_{i-1} + (1-\beta)\ v_{i} = \beta\ (\beta\ avg_{i-2} + (1-\beta)\ v_{i-1}) + (1-\beta)\ v_{i} \\
&= \beta^{2}\ avg_{i-2} + (1-\beta)\ \beta\ v_{i-1} + (1-\beta)\ v_{i} \\
&= \beta^{3}\ avg_{i-3} + (1-\beta)\ \beta^{2}\ v_{i-2} + (1-\beta)\ \beta\ v_{i-1} + (1-\beta)\ v_{i} \\
&\vdots \\
&= (1-\beta)\ \beta^{i}\ v_{0} + (1-\beta)\ \beta^{i-1}\ v_{1} + \cdots + (1-\beta)\ \beta^{2}\ v_{i-2} + (1-\beta)\ \beta\  v_{i-1} + (1-\beta)\ v_{i}
\end{align*}$

and so the sum of the coefficients is

$\begin{align*}
S &=(1-\beta)\ \beta^{i} + (1-\beta)\ \beta^{i-1} + \cdots + (1-\beta)\ \beta^{2} + (1-\beta)\ \beta + (1-\beta) \\
&= (\beta^{i} - \beta^{i+1}) + (\beta^{i-1} - \beta^{i}) + \cdots + (\beta^{2} - \beta^{3}) + (\beta - \beta^{2}) + (1-\beta) \\
&= 1 - \beta^{i+1}
\end{align*}$

since all the other terms cancel out each other.

By dividing by this term, we make our moving average a true average (in the sense that all the coefficients we used for the average sum up to 1).

```python
def mom3(avg, beta, yi, i):
    if avg is None: avg=0
    avg = lin_comb(avg, yi, beta)
    return avg, avg/(1-beta**(i+1))
plot_mom(mom3)
```

## Adam and friends

In Adam, we use the gradient averages but with dampening (not like in SGD with momentum), so let's add this to the `AverageGrad` class.

```python
#export
class AverageGrad(Stat):
    _defaults = dict(mom=0.9)

    def __init__(self, dampening:bool=False): self.dampening=dampening
    def init_state(self, p): return {'grad_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, mom, **kwargs):
        state['mom_damp'] = 1-mom if self.dampening else 1.
        state['grad_avg'].mul_(mom).add_(state['mom_damp'], p.grad.data)
        return state
```

We also need to track the moving average of the gradients squared.

```python
# a.addcmul??
a = tensor([[1., 2., 3,], [4., 5, 6]])
a.addcmul(.2, tensor([1.]),tensor([2.]))
```

    tensor([[1.4000, 2.4000, 3.4000],
            [4.4000, 5.4000, 6.4000]])

```python
#export
class AverageSqrGrad(Stat):
    _defaults = dict(sqr_mom=0.99)

    def __init__(self, dampening:bool=True): self.dampening=dampening
    def init_state(self, p): return {'sqr_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, sqr_mom, **kwargs):
        state['sqr_damp'] = 1-sqr_mom if self.dampening else 1.
        state['sqr_avg'].mul_(sqr_mom).addcmul_(state['sqr_damp'], p.grad.data, p.grad.data)
        return state
```

We will also need the number of steps done during training for the debiasing.

```python
#export
class StepCount(Stat):
    def init_state(self, p): return {'step': 0}
    def update(self, p, state, **kwargs):
        state['step'] += 1
        return state
```

This helper function computes the debias term. If we dampening, `damp = 1 - mom` and we get the same result as before. If we don't use dampening, (`damp = 1`) we will need to divide by `1 - mom` because that term is missing everywhere.

```python
#export
def debias(mom, damp, step): return damp * (1 - mom**step) / (1-mom)
```

Then the Adam step is just the following:

```python
#export
def adam_step(p, lr, mom, mom_damp, step, sqr_mom, sqr_damp, grad_avg, sqr_avg, eps, **kwargs):
    debias1 = debias(mom,     mom_damp, step)
    debias2 = debias(sqr_mom, sqr_damp, step)
    p.data.addcdiv_(-lr / debias1, grad_avg, (sqr_avg/debias2).sqrt() + eps)
    return p
adam_step._defaults = dict(eps=1e-5)
```

```python
#export
def adam_opt(xtra_step=None, **kwargs):
    return partial(StatefulOptimizer, steppers=[adam_step,weight_decay]+listify(xtra_step),
                   stats=[AverageGrad(dampening=True), AverageSqrGrad(), StepCount()], **kwargs)
```

```python
learn,run = get_learn_run(nfs, data, 0.001, conv_layer, cbs=cbfs, opt_func=adam_opt())
```

```python
run.fit(3, learn)
```

    train: [1.7600864500930666, tensor(0.3937, device='cuda:0')]
    valid: [1.445366455078125, tensor(0.5060, device='cuda:0')]
    train: [1.2675379839944936, tensor(0.5829, device='cuda:0')]
    valid: [1.181259033203125, tensor(0.6120, device='cuda:0')]
    train: [0.9869430397617108, tensor(0.6824, device='cuda:0')]
    valid: [1.04019091796875, tensor(0.6540, device='cuda:0')]

## LAMB

It's then super easy to implement a new optimizer. This is LAMB from a [very recent paper](https://arxiv.org/pdf/1904.00962.pdf):

$\begin{align}
g_{t}^{l} &= \nabla L(w_{t-1}^{l}, x_{t}) \\
m_{t}^{l} &= \beta_{1} m_{t-1}^{l} + (1-\beta_{1}) g_{t}^{l} \\
v_{t}^{l} &= \beta_{2} v_{t-1}^{l} + (1-\beta_{2}) g_{t}^{l} \odot g_{t}^{l} \\
m_{t}^{l} &= m_{t}^{l} / (1 - \beta_{1}^{t}) \\
v_{t}^{l} &= v_{t}^{l} / (1 - \beta_{2}^{t}) \\
r_{1} &= \|w_{t-1}^{l}\|_{2} \\
s_{t}^{l} &= \frac{m_{t}^{l}}{\sqrt{v_{t}^{l} + \epsilon}} + \lambda w_{t-1}^{l} \\
r_{2} &= \| s_{t}^{l} \|_{2} \\
\eta^{l} &= \eta * r_{1}/r_{2} \\
w_{t}^{l} &= w_{t}^{l-1} - \eta_{l} * s_{t}^{l} \\
\end{align}$

```python
def lamb_step(p, lr, mom, mom_damp, step, sqr_mom, sqr_damp, grad_avg, sqr_avg, eps, wd, **kwargs):
    debias1 = debias(mom,     mom_damp, step)
    debias2 = debias(sqr_mom, sqr_damp, step)
    r1 = p.data.pow(2).mean().sqrt()
    step = (grad_avg/debias1) / ((sqr_avg/debias2).sqrt()+eps) + wd*p.data
    r2 = step.pow(2).mean().sqrt()
    p.data.add_(-lr * min(r1/r2,10), step)
    return p
lamb_step._defaults = dict(eps=1e-6, wd=0.)
```

```python
dict(a=3, b='c')
```

    {'a': 3, 'b': 'c'}

```python
lamb = partial(StatefulOptimizer, steppers=lamb_step, stats=[AverageGrad(dampening=True), AverageSqrGrad(), StepCount()])
```

```python
learn,run = get_learn_run(nfs, data, 0.003, conv_layer, cbs=cbfs, opt_func=lamb)
```

```python
run.fit(3, learn)
```

    train: [1.8728660132619823, tensor(0.3419, device='cuda:0')]
    valid: [1.484662841796875, tensor(0.4860, device='cuda:0')]
    train: [1.3553239394291918, tensor(0.5506, device='cuda:0')]
    valid: [1.537447021484375, tensor(0.4840, device='cuda:0')]
    train: [1.0624497101364976, tensor(0.6537, device='cuda:0')]
    valid: [1.11101806640625, tensor(0.6460, device='cuda:0')]

Other recent variants of optimizers:
- [Large Batch Training of Convolutional Networks](https://arxiv.org/abs/1708.03888) (LARS also uses weight statistics, not just gradient statistics. Can you add that to this class?)
- [Adafactor: Adaptive Learning Rates with Sublinear Memory Cost](https://arxiv.org/abs/1804.04235) (Adafactor combines stats over multiple sets of axes)
- [Adaptive Gradient Methods with Dynamic Bound of Learning Rate](https://arxiv.org/abs/1902.09843)

## Export

```python
!python notebook2script.py 09_optimizers.ipynb
```

```python

```


```python
import numpy as np
import math
from math import exp
from IPython.display import display
from sympy import *
init_printing()
import matplotlib.pyplot as plt
%matplotlib notebook
```

```python
chi = IndexedBase('chi')  # Atomic orbital
c = IndexedBase('c')    # MO coefficients
psi = IndexedBase('psi')  # Molecular orbital
i = Symbol('i', integer=True)   # index of atomic orbitals
j = Symbol('j', integer=True)   # index of molecular orbital
N = Symbol('N', integer=True)   # Number of atomic orbitals

mo_expr = Sum(c[i,j] * chi[i], (i,1,N))
Eq(psi[j], mo_expr)
```

## Molecular Orbitals
The GaussianOrbitals notebook went into detail about the individual basis functions on each atomic site.  These orbitals get combined to for molecular orbitals - a linear combination of orbitals from both the same atom and different atoms.

The code to evaluate basis functions has been placed in 'gaussian_orbitals.py', and the code to read the basis and MO information from the QMCPACK XML input file is in 'read_qmcpack.py'

```python
# Contains
# - CG_basis (named tuple) for storing basis contraction coefficients
# - GTO class - symbolic form for basis functions (and derivatives) and methods for evaluating them
# - GTO_centers - multiple atoms of different kinds at different locations

import gaussian_orbitals
reload(gaussian_orbitals)
from gaussian_orbitals import GTO, CG_basis, GTO_centers

from read_qmcpack import parse_qmc_wf, read_structure_file
```

```python
# For Neon with DEF2-SVP
ne_basis_set, ne_MO_matrix = parse_qmc_wf('ne_def2_svp.wfnoj.xml',['Ne'])
for cg in ne_basis_set:
    print cg

ne_MO_matrix
```

    MO coeff size =  14
    Ne

    array([[ 0.990133, -0.031233,  0.009072,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [ 0.322026,  0.643777,  0.460567, -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      , -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      ],
           [-0.      , -0.      , -0.      ,  0.697249, -0.      , -0.      ,
            -0.454527, -0.      , -0.      , -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.697249,
             0.      ,  0.      , -0.454527,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [-0.      , -0.      , -0.      , -0.      ,  0.697249, -0.      ,
            -0.      , -0.454527, -0.      , -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      ],
           [ 0.327583,  1.608118, -1.575531,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.906086,
             0.      ,  0.      ,  1.049073,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.906086,  0.      ,
             0.      ,  1.049073,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.906086,  0.      ,  0.      ,
             1.049073,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  1.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             1.      ,  0.      ,  0.      ],
           [ 0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  0.      ,  0.      ,  0.      ,  0.      ,  0.      ,
             0.      ,  1.      ,  0.      ],
           [-0.      , -0.      , -0.      , -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      , -0.480634, -0.519119,  0.999753,
            -0.      , -0.      , -0.      ],
           [-0.      , -0.      , -0.      , -0.      , -0.      , -0.      ,
            -0.      , -0.      , -0.      ,  0.876921, -0.854702, -0.02222 ,
            -0.      , -0.      , -0.      ]])

```python
ne_gto = GTO(ne_basis_set['Ne'])
ne_one_eval = ne_gto.eval_v(0.2, 0.3, 0.1)
print ne_one_eval
```

    [-0.46242075697109086, 0.7402974041672985, 0.35013343898398275, -0.746944254700568, -1.120416382050852, -0.373472127350284, 0.09424566278803664, 0.14136849418205494, 0.04712283139401832, 0.15370224555863643, 0.3458300525069319, 0.03842556138965911, 0.39933014781747894, 0.13311004927249298, 0.19966507390873944]

```python
%%time
rvals = np.arange(-4.0, 4.0, .1)
ne_atomic_orbs = np.zeros( (len(ne_one_eval), rvals.shape[0]))
print ne_atomic_orbs.shape
for i,x in enumerate(rvals):
    ne_atomic_orbs[:,i] = ne_gto.eval_v(x, 0.0, 0.0)
```

    (15, 80)
    CPU times: user 50.2 ms, sys: 4.53 ms, total: 54.7 ms
    Wall time: 37.8 ms

```python
plt.plot(rvals, ne_atomic_orbs[0,:], label="AO 1")
plt.plot(rvals, ne_atomic_orbs[1,:], label="AO 2")
plt.legend()
```

```python
%%time
n = 50
xsp = np.linspace(-1, 1, n)
xyzvals = np.meshgrid(xsp, xsp, xsp)
xyzvals = np.zeros( (n,n,n))
for i,x in enumerate(xsp):
    for j,y in enumerate(xsp):
        for k,z in enumerate(xsp):
            xyzvals[i,j,k] = ne_gto.eval_v(x,y,z)[1]
```

    CPU times: user 39.5 s, sys: 185 ms, total: 39.6 s
    Wall time: 39.2 s

```python
print xyzvals
```

    [[[0.01537756 0.01713686 0.01901144 ... 0.01901144 0.01713686 0.01537756]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      ...
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01537756 0.01713686 0.01901144 ... 0.01901144 0.01713686 0.01537756]]

     [[0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01909744 0.02128232 0.02361036 ... 0.02361036 0.02128232 0.01909744]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      ...
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.01909744 0.02128232 0.02361036 ... 0.02361036 0.02128232 0.01909744]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]]

     [[0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.02350404 0.02619307 0.02905829 ... 0.02905829 0.02619307 0.02350404]
      ...
      [0.02350404 0.02619307 0.02905829 ... 0.02905829 0.02619307 0.02350404]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]]

     ...

     [[0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.02350404 0.02619307 0.02905829 ... 0.02905829 0.02619307 0.02350404]
      ...
      [0.02350404 0.02619307 0.02905829 ... 0.02905829 0.02619307 0.02350404]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]]

     [[0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01909744 0.02128232 0.02361036 ... 0.02361036 0.02128232 0.01909744]
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      ...
      [0.02118648 0.02361036 0.02619307 ... 0.02619307 0.02361036 0.02118648]
      [0.01909744 0.02128232 0.02361036 ... 0.02361036 0.02128232 0.01909744]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]]

     [[0.01537756 0.01713686 0.01901144 ... 0.01901144 0.01713686 0.01537756]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      ...
      [0.01901144 0.02118648 0.02350404 ... 0.02350404 0.02118648 0.01901144]
      [0.01713686 0.01909744 0.02118648 ... 0.02118648 0.01909744 0.01713686]
      [0.01537756 0.01713686 0.01901144 ... 0.01901144 0.01713686 0.01537756]]]

```python
# Use ipyvolume to render in 3D
import ipyvolume as ipy
```

```python
ipy.quickvolshow(xyzvals,level=[0.1,0.2,0.9])
```

    VkJveChjaGlsZHJlbj0oVkJveChjaGlsZHJlbj0oSEJveChjaGlsZHJlbj0oTGFiZWwodmFsdWU9dSdsZXZlbHM6JyksIEZsb2F0U2xpZGVyKHZhbHVlPTAuMSwgbWF4PTEuMCwgc3RlcD0wLjDigKY=

```python
ne_mo = np.dot(ne_MO_matrix, ne_atomic_orbs)
```

```python
ne_mo.shape
```

```python
plt.plot(rvals, ne_mo[0,:], label="MO 1")
plt.plot(rvals, ne_mo[1,:], label="MO 2")
plt.legend()
```

```python
# For HCN
hcn_pos_list, hcn_elements = read_structure_file("hcn.structure.xml")
hcn_basis_sets, hcn_MO_matrix = parse_qmc_wf("hcn.wfnoj.xml", hcn_elements)

hcn_gtos = GTO_centers(hcn_pos_list, hcn_elements, hcn_basis_sets)
print hcn_pos_list
```

    MO coeff size =  33
    [[-1.09604107  0.          0.        ]
     [ 1.09604107  0.          0.        ]
     [ 3.10859925  0.          0.        ]]

```python

rvals = np.arange(-5.0, 5.0, .2)

hcn_one_eval = hcn_gtos.eval_v(0.2, 0.3, 0.1)

hcn_atomic_orbs = np.zeros( (len(hcn_one_eval), rvals.shape[0]))
print hcn_atomic_orbs.shape
for i,x in enumerate(rvals):
    hcn_atomic_orbs[:,i] = hcn_gtos.eval_v(x, 0.0, 0.0)
```

    (35, 50)

```python
hcn_mo = np.dot(hcn_MO_matrix, hcn_atomic_orbs)
```

```python
fig1,ax1 = plt.subplots()
#ax1.plot(rvals, hcn_atomic_orbs[0,:], label="AO 1")
#ax1.plot(rvals, hcn_atomic_orbs[1,:], label="AO 2")
ax1.plot(rvals, hcn_mo[0,:], label="MO 1")
plt.legend()

```

```python

```