# Tutorial: Geometric view of data

---
# Objectives

In this notebook we'll explore how multivariate data can be represented in different orthonormal bases (dimensions). In other words, how to change the dimensions defining a set of data. As a start, the cartesian coordinates are an example of a two-dimensional orthogonal basis. This notebook will help you build intuition that will be helpful in understanding the use of PCA in research that you will be reading about.

Overview:
 - Explore correlated multivariate data.
 - Define and visualize an arbitrary orthonormal basis.
 - Project data from one basis (cartesian) onto another basis (arbitrary).

---
# Setup

```python
# @title Imports, Settings, Functions
# @markdown Execute this code cell to set up the notebook
import numpy as np
import matplotlib.pyplot as plt

import ipywidgets as widgets  # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")

def plot_data(X):
  """
  Plots bivariate data. Includes a plot of each random variable, and a scatter
  plot of their joint activity. The title indicates the sample correlation
  calculated from the data.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable

  Returns:
    Nothing.
  """

  fig = plt.figure(figsize=[10, 6])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(X[:, 0], color='k')
  plt.ylabel('Neuron 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(X[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(X[:, 1], color='k')
  plt.xlabel('Sample Number')
  plt.ylabel('Neuron 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(X[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(X[:, 0], X[:, 1], '.', markerfacecolor=[.5, .5, .5],
           markeredgewidth=0)
  ax3.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(X[:, 0], X[:, 1])[0, 1]))
  plt.show()

def plot_basis_vectors(X, W):
  """
  Plots bivariate data as well as new basis vectors.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable
    W (numpy array of floats) :   Square matrix representing new orthonormal
                                  basis each column represents a basis vector

  Returns:
    Nothing.
  """

  plt.figure(figsize=[4, 4])
  plt.plot(X[:, 0], X[:, 1], '.', color=[.5, .5, .5], label='Data')
  plt.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
           label='Basis vector 1')
  plt.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
           label='Basis vector 2')
  plt.legend()
  plt.show()

def plot_data_new_basis(Y):
  """
  Plots bivariate data after transformation to new bases.
  Similar to plot_data but with colors corresponding to projections onto
  basis 1 (red) and basis 2 (blue). The title indicates the sample correlation
  calculated from the data.

  Note that samples are re-sorted in ascending order for the first
  random variable.

  Args:
    Y (numpy array of floats):   Data matrix in new basis each column
                                 corresponds to a different random variable

  Returns:
    Nothing.
  """
  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(Y[:, 0], 'r')
  plt.xlabel
  plt.ylabel('Projection \n basis vector 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(Y[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(Y[:, 1], 'b')
  plt.xlabel('Sample number')
  plt.ylabel('Projection \n basis vector 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(Y[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(Y[:, 0], Y[:, 1], '.', color=[.5, .5, .5])
  ax3.axis('equal')
  plt.xlabel('Projection basis vector 1')
  plt.ylabel('Projection basis vector 2')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(Y[:, 0], Y[:, 1])[0, 1]))
  plt.show()

def get_data(cov_matrix):
  """
  Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian.

  Note that samples are sorted in ascending order for the first random variable

  Args:
    cov_matrix (numpy array of floats): desired covariance matrix

  Returns:
    (numpy array of floats) : samples from the bivariate Gaussian, with each
                              column corresponding to a different random
                              variable
  """

  mean = np.array([0, 0])
  X = np.random.multivariate_normal(mean, cov_matrix, size=1000)
  indices_for_sorting = np.argsort(X[:, 0])
  # X = X[indices_for_sorting, :]

  return X

def calculate_cov_matrix(var_1, var_2, corr_coef):
  """
  Calculates the covariance matrix based on the variances and correlation
  coefficient.
  Args:
    var_1 (scalar)          : variance of the first random variable
    var_2 (scalar)          : variance of the second random variable
    corr_coef (scalar)      : correlation coefficient
  Returns:
    (numpy array of floats) : covariance matrix
  """

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

def define_orthonormal_basis(u):
  """
  Calculates an orthonormal basis given an arbitrary vector u.
  Args:
    u (numpy array of floats) : arbitrary 2-dimensional vector used for new
                                basis
  Returns:
    (numpy array of floats)   : new orthonormal basis
                                columns correspond to basis vectors
  """

  # Normalize vector u
  u = u / np.sqrt(u[0] ** 2 + u[1] ** 2)

  # Calculate vector w that is orthogonal to w
  w = np.array([-u[1], u[0]])

  # Put in matrix form
  W = np.column_stack([u, w])

  return W

def change_of_basis(X, W):
  """
  Projects data onto new basis W.
  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors
  Returns:
    (numpy array of floats)    : Data matrix expressed in new basis
  """

  # Project data onto new basis described by W
  Y = X @ W

  return Y

```

---
# Section 1: (Correlated) multivariate data

This notebook provides code to draw random samples from a zero-mean bivariate normal distribution with a specified covariance matrix. Throughout this tutorial, we'll imagine these samples represent the activity (firing rates) of two recorded neurons on different trials.

 Click here if you are interested in detials about the math behind multivariate normal distributions

To gain intuition, we will first use a simple model to generate multivariate data. Specifically, we will draw random samples from a *bivariate normal distribution*. This is an extension of the one-dimensional normal distribution to two dimensions, in which each $x_i$ is marginally normal with mean $\mu_i$ and variance $\sigma_i^2$:

\begin{align}
x_i \sim \mathcal{N}(\mu_i,\sigma_i^2).
\end{align}

Additionally, the joint distribution for $x_1$ and $x_2$ has a specified correlation coefficient $\rho$. Recall that the correlation coefficient is a normalized version of the covariance, and ranges between -1 and +1:

\begin{align}
\rho = \frac{\text{cov}(x_1,x_2)}{\sqrt{\sigma_1^2 \sigma_2^2}}.
\end{align}

For simplicity, we will assume that the mean of each variable has already been subtracted, so that $\mu_i=0$ for both $i=1$ and $i=2$. The remaining parameters can be summarized in the covariance matrix, which for two dimensions has the following form:

\begin{align}
{\bf \Sigma} =
\begin{pmatrix}
 \text{var}(x_1) & \text{cov}(x_1,x_2) \\
 \text{cov}(x_1,x_2) &\text{var}(x_2)
\end{pmatrix}.
\end{align}

In general, $\bf \Sigma$ is a symmetric matrix with the variances $\text{var}(x_i) = \sigma_i^2$ on the diagonal, and the covariances on the off-diagonal. Later, we will see that the covariance matrix plays a key role in PCA.

The covariance can be found by rearranging the equation above:

\begin{align}
\text{cov}(x_1,x_2) = \rho \sqrt{\sigma_1^2 \sigma_2^2}.
\end{align}

## Interactive Demo 1: Effect of correlation on data in two dimensions

Executing the code cell below will enable you to can change the correlation coefficient between data in two dimensions via an interactive slider. You should get a feel for how changing the correlation coefficient affects the geometry of the simulated data.

1.   What effect do negative correlation coefficient values have?
2.   What correlation coefficient results in a circular data cloud?

```python
# @markdown Execute this cell to enable widget

def _calculate_cov_matrix(var_1, var_2, corr_coef):

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

@widgets.interact(corr_coef = widgets.FloatSlider(value=.2, min=-1, max=1, step=0.1))
def visualize_correlated_data(corr_coef=0):
  variance_1 = 1
  variance_2 = 1

  # Compute covariance matrix
  cov_matrix = _calculate_cov_matrix(variance_1, variance_2, corr_coef)

  # Generate data with this covariance matrix
  X = get_data(cov_matrix)

  # Visualize
  plot_data(X)
```

---
# Section 2: Define an orthonormal basis (a different set of dimensions)

Data can be represented in many ways using different bases. We will be using "orthonormal" bases (orthogonal with length 1).

 Click here if you are interested in some detail about the math

We will define a new orthonormal basis of vectors ${\bf u} = [u_1,u_2]$ and ${\bf w} = [w_1,w_2]$. Two vectors are orthonormal if:

1.   They are orthogonal (i.e., their dot product is zero):
\begin{align}
{\bf u\cdot w} = u_1 w_1 + u_2 w_2 = 0
\end{align}
2.   They have unit length:
\begin{align}
||{\bf u} || = ||{\bf w} || = 1
\end{align}

In two dimensions, it is easy to make an arbitrary orthonormal basis. All we need is a random vector ${\bf u}$, which we have normalized. If we now define the second basis vector to be ${\bf w} = [-u_2,u_1]$, we can check that both conditions are satisfied:
\begin{align}
{\bf u\cdot w} = - u_1 u_2 + u_2 u_1 = 0
\end{align}
and
\begin{align}
{|| {\bf w} ||} = \sqrt{(-u_2)^2 + u_1^2} = \sqrt{u_1^2 + u_2^2} = 1,
\end{align}
where we used the fact that ${\bf u}$ is normalized. So, with an arbitrary input vector, we can define an orthonormal basis, which we will write in matrix by stacking the basis vectors horizontally:

\begin{align}
{{\bf W} } =
\begin{pmatrix}
 u_1 & w_1 \\
 u_2 & w_2
\end{pmatrix}.
\end{align}

```python
# @markdown Execute this cell to plot two orthonormal bases,
# @markdown with the first dimension based on a vector defined by `[x,y]`

# @widgets.interact(corr_coef = widgets.FloatSlider(value=.2, min=-1, max=1, step=0.1))
def visualize_orthonormal_bases(x=0,y=3):
  u = [x,y]

  # Get orthonomal basis
  W = define_orthonormal_basis(u)

  # Visualize
  with plt.xkcd():
    fig,ax  = plt.subplots(figsize=[6, 6])
    # ax = plt.subplot(1)
    # plt.figure(figsize=[6, 6])
    ax.axis('equal')
    ax.plot([0, u[0]], [0, u[1]], color='k', linewidth=6,
              label='Original vector')
    ax.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
              label='Basis vector 1')
    ax.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
              label='Basis vector 2')
    ax.legend()
    plt.show()

_ = widgets.interact(visualize_orthonormal_bases, x=(-3,3,0.2), y=(-3,3,0.2))
```

---
# Section 3: Project data onto new basis

Finally, we will express bivariate (2-dimensional) data ($\bf X$) in a new 2-dimensional basis defined as in Section 2. We can project the data into our new basis using ***matrix multiplication*** :

\begin{align}
{\bf Y = X W}.
\end{align}

We will explore the geometry of the transformed data $\bf Y$ as we vary the choice of basis.

## Interactive Demo 3: Play with the basis vectors
To see what happens to the correlation between dimensions as we change the basis vectors, run the cell below.

The parameter corr_coeff controls the correlation between the two original dimensions.

The parameter $\theta$ controls the angle of the first basis vector (red, $\bf u$) in degrees. The second basis vector will be orthogonal to the first. Use the slider to rotate the basis vectors (the new dimensions).

1.   What happens to the projected data as you rotate the basis?
2.   How does the correlation coefficient change? How does the variance of the projection onto each basis vector change?
3.   Are you able to find a basis in which the projected data is **uncorrelated**?

```python
# @markdown Execute this cell to enable the widget

def refresh(corr_coef=0.5,theta=0):
  # corr_coef=0.5
  variance_1 = 1
  variance_2 = 1

  # Compute covariance matrix
  cov_matrix = _calculate_cov_matrix(variance_1, variance_2, corr_coef)

  # Generate data with this covariance matrix
  X = get_data(cov_matrix)
  u = [1, np.tan(theta * np.pi / 180)]
  W = define_orthonormal_basis(u)
  Y = change_of_basis(X, W)
  # plot_basis_vectors(X, W)

  # plot_data_new_basis(Y)
  fig = plt.figure(figsize=[20, 10])
  gs = fig.add_gridspec(4, 4)

  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(X[:, 0], color='k')
  plt.ylabel('Neuron 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(X[:, 0])))
  ax1.set_xticklabels([])

  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(X[:, 1], color='k')
  plt.xlabel('Sample Number')
  plt.ylabel('Neuron 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(X[:, 1])))

  ax3 = fig.add_subplot(gs[0:2, 1])
  ax3.plot(X[:, 0], X[:, 1], '.', markerfacecolor=[.5, .5, .5],
           markeredgewidth=0)
  ax3.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
            label='Basis vector 1')
  ax3.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
            label='Basis vector 2')
  ax3.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(X[:, 0], X[:, 1])[0, 1]))

  ax4 = fig.add_subplot(gs[2, 0])
  ax4.plot(Y[:, 0], 'r')
  plt.xlabel
  plt.ylabel('Projection \n basis vector 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(Y[:, 0])))
  ax4.set_xticklabels([])
  ax5 = fig.add_subplot(gs[3, 0])
  ax5.plot(Y[:, 1], 'b')
  plt.xlabel('Sample number')
  plt.ylabel('Projection \n basis vector 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(Y[:, 1])))
  ax6 = fig.add_subplot(gs[2:4, 1])
  ax6.plot(Y[:, 0], Y[:, 1], '.', color=[.5, .5, .5])
  ax6.axis('equal')
  plt.xlabel('Projection basis vector 1')
  plt.ylabel('Projection basis vector 2')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(Y[:, 0], Y[:, 1])[0, 1]))
  plt.show()

_ = widgets.interact(refresh, corr_coef=(-1,1,0.1), theta=(-90, 90, 5))
```

---
# Summary

- In this tutorial, we learned that multivariate data can be visualized as a cloud of points in a high-dimensional vector space. The geometry of this cloud is shaped by the *covariance matrix*.

- Multivariate data can be represented in a new orthonormal basis using the dot product (matrix multiplication). These new basis vectors correspond to specific mixtures of the original variables - *for example, in neuroscience, they could represent different ratios of activation  across a population of neurons*.

- The projected data (after transforming into the new basis) will generally have a different geometry from the original data. In particular, taking basis vectors that are aligned with the spread of cloud of points decorrelates the data.

* These concepts - covariance, projections, and orthonormal bases - are key for understanding PCA, which is a foundational computational tool in a wide variety of neuroscience research.

---
# Notation

\begin{align}
x_i &\quad \text{data point for dimension } i\\
\mu_i &\quad \text{mean along dimension } i\\
\sigma_i^2 &\quad \text{variance along dimension } i \\
\bf u, \bf w &\quad \text{orthonormal basis vectors}\\
\rho &\quad \text{correlation coefficient}\\
\bf \Sigma &\quad \text{covariance matrix}\\
\bf X &\quad \text{original data matrix}\\
\bf W &\quad \text{projection matrix}\\
\bf Y &\quad \text{transformed data}\\
\end{align}

---
This tutorial was written by Krista Perks for BIOL358 Motor Systems taught at Wesleyan University. Based on content from **Neuromatch Academy 2020: Week 1, Day 5: Dimensionality Reduction** by Alex Cayco Gajic, John Murray


```python
import os
import time
import handcalcs.render
import numpy as np
import pandas as pd
import pandas_profiling
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.graph_objects as go
import plotly.express as px
from IPython.display import display
from autoviz.AutoViz_Class import AutoViz_Class
from sklearn.preprocessing import FunctionTransformer,  OneHotEncoder, MinMaxScaler
from sklearn.impute import SimpleImputer
from sklearn.decomposition import PCA
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split, GridSearchCV, KFold, cross_val_score
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from pycaret.classification import *
from pycaret.datasets import get_data
%matplotlib inline
```

```python
def load_data(name='test'):
    path = os.path.join('../data', name + '.csv')
    data = pd.read_csv(path)
    return data
```

```python
train_data = load_data('train')
test_data = load_data('test')
df_all = train_data.append(test_data, ignore_index=True)
```

```python
display(df_all.head(40))
```

```python
display(df_all.describe())
```

```python
display(df_all.info())
```

```python
df_all.isnull().info()
```

```python
print(df_all.columns)
```

```python
sns.distplot(df_all['Age'], color='black')
plt.show()
age_cat = pd.cut(df_all['Age'], bins=[0, 10, 20, 30, 40, 50, 60, 70, 80], labels=[
                 '0-10', '10-20', '20-30', '30-40', '40-50', '50-60', '60-70', '70-80'])
fig, axes = plt.subplots(1, 2, figsize=(10, 6))
sns.countplot(age_cat[df_all['Sex'] == 'female'], color='black', ax=axes[0])
sns.countplot(age_cat[(df_all['Survived'] == 1) & (
    df_all['Sex'] == 'female')], color='pink', ax=axes[0]).set_title('Female')
sns.countplot(age_cat[df_all['Sex'] == 'male'], color='black', ax=axes[1])
sns.countplot(age_cat[(df_all['Survived'] == 1) & (
    df_all['Sex'] == 'male')], color='blue', ax=axes[1]).set_title('Male')
plt.show()
```

```python
sns.countplot(df_all['Pclass'])
plt.show()
```

```python
grouped = df_all.groupby(['Sex', 'Pclass'])
# display(grouped['Age'].median())
ax = grouped['Age'].median().plot(kind='bar', color='black')
ax.set(ylabel='Median Age')
plt.show()
```

```python
df_all.drop('PassengerId', axis=1, inplace=True)
df_all.drop('Ticket', axis=1, inplace=True)
df_all.drop('Name', axis=1, inplace=True)
df_all.loc[df_all['Cabin'].isnull(), 'Cabin'] = 0
df_all.loc[df_all['Cabin'] != 0, 'Cabin'] = 1
df_all['Embarked'].fillna(
    df_all['Embarked'].value_counts().index[0], inplace=True)
df_all['Fare'].fillna(df_all['Fare'].median(), inplace=True)
df_all['Age'] = grouped['Age'].apply(lambda x: x.fillna(x.median()))
df_all['Sex'] = df_all['Sex'].map({'male': 0, 'female': 1})
df_all = pd.get_dummies(df_all, columns=['Embarked'])
```

```python
display(df_all.head(4))
report = pandas_profiling.ProfileReport(df_all)
display(report)
```

```python
AV = AutoViz_Class()
report_av = AV.AutoViz('../data/train.csv')
```

```python
scaler = MinMaxScaler()
X = df_all.drop('Survived', axis=1).iloc[:891].values
y = (df_all['Survived'].iloc[:891].values).astype(int)
X = scaler.fit_transform(X)
X_test = df_all.drop('Survived', axis=1).iloc[891:].values
X_test = scaler.transform(X_test)
y_test = (df_all['Survived'].iloc[891:].values).astype(int)
```

```python
lr = LogisticRegression()
lr.fit(X, y)
Y_pred = lr.predict(X_test)
lr.score(X, y)
```

```python
svc = SVC()
svc.fit(X, y)
Y_pred = svc.predict(X_test)
svc.score(X, y)
```

```python
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X, y)
Y_pred = knn.predict(X_test)
knn.score(X, y)
```

```python
gaussian = GaussianNB()
gaussian.fit(X, y)
Y_pred = gaussian.predict(X_test)
gaussian.score(X, y)
```

```python
perceptron = Perceptron()
perceptron.fit(X, y)
Y_pred = perceptron.predict(X_test)
perceptron.score(X, y)
```

```python
linear_svc = LinearSVC()
linear_svc.fit(X, y)
Y_pred = linear_svc.predict(X_test)
linear_svc.score(X, y)
```

```python
sgd = SGDClassifier()
sgd.fit(X, y)
Y_pred = sgd.predict(X_test)
sgd.score(X, y)
```

```python
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X, y)
Y_pred = decision_tree.predict(X_test)
decision_tree.score(X, y)
```

```python
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X, y)
Y_pred = random_forest.predict(X_test)
random_forest.score(X, y)
```

```python

```

## Logistic Regression
\begin{equation}
P\left[Y=y\ \big|\ x;\omega\right]\approx\sigma(\omega^Tx)
\end{equation}
where
\begin{equation}
\sigma(t)=\frac{1}{1+\exp^{-t}}
\end{equation}
$\omega$ can be obtained using maximum likelihood estimation, i.e. minimizing the negative log-likelihood:
\begin{equation}
J(\omega) = -\frac{1}{m}\sum\limits_{i=1}^{n}y_i\log(\sigma(\omega^Tx_i))+(1-y_i)\log(1-\sigma(\omega^Tx_i))
\end{equation}

```python
def logistic_regression(X, y, alpha=1e-3, num_iter=30, random_state=1):
    np.random.seed(random_state)
    d, m = X.shape
    K = np.max(y)+1  # 0~c-1 => 1~c
    w = np.random.randn(d, K)

    def softmax(x):
        s = np.exp(x)/np.sum(np.exp(x))
        return s

    def one_hot(y, k):
        """
        y=[0,1,2,1]
        k=3
        return:
        [[1. 0. 0.]
         [0. 1. 0.]
         [0. 0. 1.]
         [0. 1. 0.]]
        """
        y_one_hot = np.eye(k)[y]
        return y_one_hot

    def h(x, w):
        p = softmax(w.T@x)
        return p

    def cost(pred, y):
        c = np.sum(-one_hot(y, K).T*np.log(pred))
        return c

    def grad(w, x, y):
        Y = one_hot(y, K).T
        b = h(x, w)-Y
        b = np.reshape(b, (-1, 1))
        x = x.reshape((-1, 1))
        g = x@b.T
        return g
    for i in range(num_iter):
        for j in np.random.permutation(m):
            gradient = grad(w, X[:, j], y[j])
            w -= alpha*gradient
    return w
```

## Least Square Ridge Classifier
$X\in R^{m\times d},Y\in R^{m\times K}$
#### closed form solution
\begin{equation}
\begin{aligned}
J(\omega) &= \|X\omega-Y\|_2^2+\lambda\|\omega\|_F^2 \\
&=(X\omega-Y)^T\cdot(X\omega-Y)+\lambda\omega^T\omega \\
&=(\omega^TX^T-Y^T)\cdot(X\omega-Y)+\lambda\omega^T\omega \\
&=\omega^TX^TX\omega-\omega^TX^TY-Y^TX\omega+Y^TY+\lambda\omega^T\omega
\end{aligned}
\end{equation}
To minimize: $\frac{\partial J}{\partial\omega}=0$
\begin{equation}
\begin{aligned}
&\frac{\partial J}{\partial\omega}=2X^TX\omega-2X^TY+2\lambda\omega=0 \\
\implies & 2(X^TX+\lambda I)\omega=2X^TY \\
\implies & \omega=(X^TX+\lambda I)^{-1}X^TY
\end{aligned}
\end{equation}

```python
def ridge_classifier(X, y, lambd=1e-4):
    d, m = X.shape
    k = np.max(y)+1
    w = np.linalg.inv(X@X.T+lambd*np.eye(d))@X@np.eye(k)[y]
    return w
```

```python
def error(X, y, w):
    m = np.shape(y)
    y_pred = w.T @ X
    y_pred = np.argmax(y_pred, axis=0)
    err = np.sum(y_pred == y) / m
    return err
```

```python
scores_lr = []
scores_ls = []
fold = 1

for tr, val in KFold(n_splits=5, random_state=42).split(X, y):
    X_train = X[tr]
    X_val = X[val]
    y_train = y[tr]
    y_val = y[val]
    best_W_LR = logistic_regression(
        X_train.T, y_train, alpha=1e-3, num_iter=300, random_state=42)
    val_acc_LR = error(X_val.T, y_val, best_W_LR)
    scores_lr.append(val_acc_LR)
    print(f'Validation acc LR: Fold {fold}:', val_acc_LR)
    W_LS = ridge_classifier(X_train.T, y_train, lambd=1e-4)
    val_acc_LS = error(X_val.T, y_val, W_LS)
    scores_ls.append(val_acc_LS)
    print(f'Validation acc LS: Fold {fold}:', val_acc_LS)
    fold += 1

print('-------------------------------')
print("Accuracy Logistic Regression: %0.2f (+/- %0.2f)" %
      (np.mean(scores_lr), np.std(scores_lr) * 2))
print("Accuracy Least Squares Ridge: %0.2f (+/- %0.2f)" %
      (np.mean(scores_ls), np.std(scores_ls) * 2))
```

```python
def test_clfs(clfs):
    for clf in clfs:
        start = time()
        clf = clf(random_state=0)
        scores = cross_val_score(clf, X, y, cv=5)
        print(str(clf), 'results:')
        print('Accuracy')
```

```python
data = train_data.drop('Name',axis=1).drop('Ticket',axis=1).drop('PassengerId',axis=1)
clf = setup(data=data,target='Survived')
top3 = compare_models(n_select=3,exclude=['catboost'])
```

```python
tuned_top3 = [tune_model(i) for i in top3]
```

```python
bagged_tuned_top3 = [ensemble_model(i, method = 'Bagging') for i in tuned_top3]
```

```python
blender = blend_models(estimator_list = top3)
```

```python
stacker = stack_models(estimator_list = top3[1:], meta_model = top3[0])
```

```python
best_model = automl(optimize = 'Accuracy')
```

```python
save_model(best_model, 'model')
```

```python
plot_model(best_model,plot='boundary')
```

```python
evaluate_model(best_model)
```

```python
predict_model(best_model)
```

```python
model = finalize_model(best_model)
```

```python
y_test_pred = predict_model(best_model, data=test_data)[['PassengerId','Label']]
y_test_pred['Survived']=y_test_pred['Label']
y_test_pred.drop('Label',axis=1,inplace=True)
print(y_test_pred)
y_test_pred.to_csv('../data/my_submission.csv',header=True,index=None,encoding='utf-8')
```

```python

```


#### [**NICOLAS CACHANOSKY**](http://www.ncachanosky.com) | Department of Economics | Metropolitan State University of Denver | ncachano@msudenver.edu

# LABOR MARKET
---

This note illustrates how to code a labor market in Python. The purpose of the note is to walk through Python applications, not to offer a detailed discussion of the labor market or to show best coding practices. The note also assumes familiarity with the neoclassical labor market model and a beginner experience with Python.

For a more complete and detailed discussion of Python applications see the material in [Quant Econ](https://quantecon.org/).

---

## TABLE OF CONTENTS
1. [Labor demand](#1.-LABOR-DEMAND)
2. [Labor supply](#2.-LABOR-SUPPLY)
3. [Equilibrium](#3.-EQUILIBRIUM)

## 1. LABOR DEMAND

Labor demand $\left(N^D\right)$ comes from a representative firm maximazing its profits $(\pi)$. Assume output $(Q)$ follows a Cobb-Douglas production function with Hicks-Neutral technology $(A)$, and that $P$ is the market price of the firm's output. Further, assume that $w$ and $r$ are the prices of labor $(N)$ and capital $(K)$ respectively. Then, firm's profit is (where $\alpha \in (0, 1)$):

\begin{align}
    \pi &= P \cdot Q(K, N) - wN - rK \\
    \pi &= P \cdot \left(A \cdot K^{\alpha} N^{1-\alpha} \right) - wN - rK
\end{align}

With capital and technology as given in the short-run, the firm maximizes its profits by changin the amount of labor. The firm demands labor (that has decreasing marginal returns) up the points of its marginal productivity. It can be seen that labor demand has an hyperbolic shape with respect to real wages $(w/P)$.

\begin{align}
    \frac{\partial \pi}{\partial N} &= P \cdot (1-\alpha) \, A \left(\frac{K}{N}\right)^{\alpha} - w= 0   \\
    N^D &= K \cdot \left[\frac{(1-\alpha)A}{(w/P)}\right]^{1/\alpha}
\end{align}

The following code plots labor demand and shifts produced by changes in $k$ (in blue), $A$ (in red), and in $\alpha$ (in green). The first part of the code imports the required packages. The second part defines the parameters and vectors to be used. The third part of the code builds the labor demand function. The fourth section calculates labor demand and the effects of shocks (1) capital $(\Delta K = 20)$, (2) productivity $(\Delta A = 20)$, and (3) output elasticity of capital $(\Delta \alpha = 20)$. The fifth part of the code plots labor demand and the shock effects.

```python
"1|IMPORT PACKAGES"
import numpy as np               # Package for scientific computing with Python
import matplotlib.pyplot as plt  # Matplotlib is a 2D plotting library

"2|DEFINE PARAMETERS AND ARRAYS"
# Parameters
size = 50                        # Real wage domain
K = 20                           # Capital stock
A = 20                           # Technology
alpha = 0.6                      # Output elasticity of capital
# Arrays
rW = np.arange(1, size)          # Real wage

"3|LABOR DEMAND FUNCTION"
def Ndemand(A, K, rW, alpha):
    Nd = K * ((1-alpha)*A/rW)**(1/alpha)
    return Nd

"4|CALCULATE LABOR DEMAND AND SHOCK EFFECTS"
D_K = 20    # Shock to K
D_A = 20    # Shock to A
D_a = 0.2   # Shock to alpha

Nd   = Ndemand(A    , K    , rW, alpha)
Nd_K = Ndemand(A    , K+D_K, rW, alpha)
Nd_A = Ndemand(A+D_A, K    , rW, alpha)
Nd_a = Ndemand(A    , K    , rW, alpha+D_a)

"5|PLOT LABOR DEMAND AND SHOCK EFFECTS"
xmax_v = np.zeros(4)
xmax_v[0] = np.max(Nd)
xmax_v[1] = np.max(Nd_K)
xmax_v[2] = np.max(Nd_A)
xmax_v[3] = np.max(Nd_a)
xmax = np.max(xmax_v)

v = [0, 30, 0, size]                            # Set the axes range
fig, ax = plt.subplots(figsize=(10, 8))
ax.set(title="LABOR DEMAND", xlabel=r'Nd', ylabel=r'w/P')
ax.grid()
ax.plot(Nd  , rW, "k-", label="Labor demand", linewidth=3)
ax.plot(Nd_K, rW, "b-", label="Capital shock")
ax.plot(Nd_A, rW, "r-", label="Productivity shock")
ax.plot(Nd_a, rW, "g-", label="Output elasticity of K shock")
ax.yaxis.set_major_locator(plt.NullLocator())   # Hide ticks
ax.xaxis.set_major_locator(plt.NullLocator())   # Hide ticks
ax.legend()
plt.axis(v)                                     # Use 'v' as the axes range
plt.show()
```

## 2. LABOR SUPPLY

Labor supply is dervied from the consumer maximizing a constrained utility function. The consumer receives utility from consumption $(C)$ and leisure time $(L)$. While the profit function of the firm has an internal maximum, the utility function is strictly increasing on $C$ and $L$. Therefore, the utility maximization problem includes (1) a binding constrain and (2) the right mix of $C$ and $L$ that will depend on their relative prices.

Assume a Cobb-Douglas utility function where $\beta$ is the consumption elasticity of utilility.

\begin{equation}
    U(C, L) = C^{\beta} L^{1-\beta}
\end{equation}

The individual faces the folllowing budget constraint:

\begin{align}
    C &= \left(\frac{w}{P} \right) (24 - L) + I                                        \\
    C &= \underbrace{\left[I + 24 \left(\frac{w}{P} \right) \right]}_\text{intercept} - \underbrace{\left( \frac{w}{P} \right)}_\text{slope}L
\end{align}

where $24$ is the amount of hours the individual can work in given day and $I$ is other (non-labor) income.

Before deriving labor supply $N^S$ we can plot the indifference curve between consumption and leisure with the budget constraint. "Solving" for $C$ for a given level of utility:

\begin{equation}
    C = \left( \frac{\bar{U}}{L^{1-\beta}} \right)^{(1/\beta)}
\end{equation}

---
We can now maximize the utility with the budget constraint using a Lagrangian $\left(\Im\right)$:
\begin{equation}
    \max_{\{C, L\}} \Im = C^{\beta} L^{1-\beta} + \lambda \left[C - I - \frac{w}{P} (24-L) \right]
\end{equation}

The FOC for $\Im$:
\begin{cases}
    \Im_{L} = (1 - \beta) \left( \frac{C}{L} \right)^{\beta} - \lambda = 0                             \\
    \Im_{C} = \beta \left( \frac{L}{C} \right)^{1-\beta} - \lambda \left(\frac{w}{P} \right) = 0       \\
    \Im_{\lambda} = C - I - \left(\frac{w}{P}\right) (24-L) = 0
\end{cases}

From the firt two FOCs we get the known relationship $\frac{U_{L}}{U_{C}} = \frac{w/P}{1}$

Solving for $C$ in terms of $L$ yields $C = \frac{\beta}{1-\beta} \left(\frac{w}{P}\right)L$.
Pluging this result in the third FOC and solving for $L$ yields $L^{*} = (1-\beta) \left[\frac{I + 24 (w/P)}{(w/P)} \right]$. With $L^*$ we can now get $C^* = \beta \left[I + 24 (w/P) \right]$. Next we plug-in $C^*$ and $L^*$ into the utility function.

\begin{align}
    U(C^{*}, L^{*})^* &= \left(C^*\right)^{\beta} \left(L^*\right)^{1-\beta}      \\
    U(C^{*}, L^{*})^* &= \left[\beta(I + 24 (w/P)\right]^{\beta} \left[(1-\beta) \frac{I+24(w/P)}{(w/P)} \right]^{1-\beta}
\end{align}

Note that if $I=0$ then $L^*$ and $C^*$ are fixed quantities that depend on the value of $\beta$.

Using the lagrangian method also allows to find the "optimal" value of $\lambda$ or the "shadow price":

\begin{align}
    \lambda^* &= (1-\beta) \cdot \left(\frac{C^*}{L^*} \right)^{\beta}                                            \\
    \lambda^* &= (1-\beta) \cdot \left[\frac{\beta \left(I+24(w/P)\right)w}{(1-\beta)(I+24(w/P)} \right]^{\beta}  \\
    \lambda^* &= (1-\beta) \cdot \left(\frac{\beta}{1-\beta} \frac{w}{P} \right)^{\beta}
\end{align}

Now we can use this information to plot the indifference curve with `matplotlib`. Note that the code calculates $U^*$, $L^*$, and $C^*$ and uses these values in the graph. The first part of the code imports the required packages. The second part of the code defines needed parameters and vectors. The third part of the code calculates $U^*$, $L^*$, $C^*$, and builds the functions for the indifference curve and the budget constraint. The fourth part of the code builds the plot.

```python
"1|IMPORT PACKAGES"
import numpy as np               # Package for scientific computing with Python
import matplotlib.pyplot as plt  # Matplotlib is a 2D plotting library

"2|DEFINE PARAMETERS AND ARRAYS"
T = 25                           # Available hours to work
beta = 0.7                       # Utility elasticity of consumption
I = 50                           # Non-labor income
L = np.arange(1, T)                 # Array of labor hours from 0 to T
rW = 25                          # Real wage

"3|CALCULATE OPTIMAL VALUES AND DEFINE FUNCTIONS"
Ustar = (beta*(I+24*rW))**beta * ((1-beta)*(I+24*rW)/rW)**(1-beta)
Lstar = (1-beta)*((I+24*rW)/rW)
Cstar = beta*(I+24*rW)

def C_indiff(U, L, beta):        # Create consumption function
    C_indiff = (U/L**(1-beta))**(1/beta)
    return C_indiff

def Budget(I, rW, L):            # Create budget constraint
    Budget = (I + 24*rW) - rW*L
    return Budget

B = Budget(I, rW, L)             # Budget constraint
C = C_indiff(Ustar, L, beta)     # Indifference curve

"4|PLOT THE INDIFFERENCE CURVE AND THE BUDDGET CONSTRAINT"
y_max = 2*Budget(I, rW, 0)

v = [0, T, 0, y_max]                               # Set the axes range
fig, ax = plt.subplots(figsize=(10, 8))
ax.set(title="INDIFFERENCE CURVE", xlabel="Leisure", ylabel="Real income")
ax.grid()
ax.plot(L, C, "g-", label="Indifference curve")
ax.plot(L, B, "k-", label="Budget constraint")
plt.axvline(x=T-1  , ymin=0, ymax=I/y_max, color='k') # Add non-labor income
plt.axvline(x=Lstar, ymin=0, ymax = Cstar/y_max, ls=':', color='k') # Lstar
plt.axhline(y=Cstar, xmin=0, xmax = Lstar/T    , ls=':', color='k') # Cstar
plt.plot(Lstar, Cstar, 'bo')                                        # Point
plt.text(0.1      ,  Cstar+5, np.round(Cstar, 1), color="k")
plt.text(Lstar+0.2,  10     , np.round(Lstar, 1), color="k")
ax.legend()
plt.axis(v)                                         # Use 'v' as the axes range
plt.show()
```

---
Labor supply $N^S$ is the number of hours **not spent** in leisure. Note that $N^S$ decreases with $I$ and increases with $(w/P)$.

\begin{align}
    N^S &= 24 - L^*                                                  \\
    N^S &= 24 - (1-\beta) \left[\frac{I + 24 (w/P)}{(w/P)} \right]
\end{align}

The following code shows labor supply (in black) and shocks to non-labor income $\Delta I = 25$ (in blue) and to consumption elasticity of ulility $\Delta \beta = 0.10$ (in red). Note that in this construction $N^S$ does not bend-backwards.

```python
"1|IMPORT PACKAGES"
import numpy as np               # Package for scientific computing with Python
import matplotlib.pyplot as plt  # Matplotlib is a 2D plotting library

"2|DEFINE PARAMETERS AND ARRAYS"
size = 50
T = 25                           # Available hours to work
beta = 0.6                       # Utility elasticity of consumption
I = 50                           # Non-labor income
rW   = np.arange(1, size)        # Vector of real wages

"3|LABOR SUPPLY"
def Lsupply(rW, beta, I):
    Lsupply = 24 - (1-beta)*((24*rW + I)/rW)
    return Lsupply

D_I = 25     # Shock to non-income labor
D_b = 0.10   # Shock to beta

Ns   = Lsupply(rW, beta    , I)
Ns_b = Lsupply(rW, beta+D_b, I)
Ns_I = Lsupply(rW, beta    , I+D_I)

"4|PLOT LABOR SUPPLY"
y_max = np.max(Ns)

v = [0, T, 0, y_max]                               # Set the axes range
fig, ax = plt.subplots(figsize=(10, 8))
ax.set(title="LABOR SUPPLY", xlabel="Work Hs.", ylabel=r'(w/P)')
ax.grid()
ax.plot(Ns  , rW, "k", label="Labor supply", linewidth=3)
ax.plot(Ns_I, rW, "b", label="Non-labor income shock")
ax.plot(Ns_b, rW, "r", label="Consumption elasticy of utility shock")
ax.yaxis.set_major_locator(plt.NullLocator())      # Hide ticks
ax.xaxis.set_major_locator(plt.NullLocator())      # Hide ticks
ax.legend()
plt.axis(v)                                        # Use 'v' as the axes range
plt.show()
```

## 3. EQUILIBRIUM

We can now calculate the equilibrium condition, the value of $(w/P)_0$ which makes $N^D\left(\left(\frac{w}{P}\right)_0\right) = N^S\left(\left(\frac{w}{P}\right)_0\right)$. Then, we can define a function $\Theta$ equal to zero at $\left(\frac{w}{P}\right)_0$:

\begin{align}
    \Theta \left[ \left(\frac{w}{P}\right)_0\right] &= 0 = N^D \left[\left(\frac{w}{P} \right)_0\right] - N^S \left[\left(\frac{w}{P}\right)_0\right] \\
    \Theta \left[ \left(\frac{w}{P}\right)_0\right] &= 0 = \underbrace{\left[ K \cdot \left[\frac{(1-\alpha)A}{(w/P)}\right]^{1/\alpha} \right]}_{N^D} - \underbrace{\left[ 24  (1-\beta) \left[\frac{I + 24 (w/P)}{(w/P)} \right] \right]}_{N^S}
\end{align}

We can ask Python to calculate the value (root) of $\left( \frac{w}{P} \right)$ that makes $\Theta = 0$. For this we need the "root" function from the `SciPy` library. The cade has four sections. Section 1 imports the required packages. Section 2 defines the paramters and arrays. Section 3 find the dequilibrium values. And section 4 plots the results.

```python
"1|IMPORT PACKAGES"
import numpy as np               # Package for scientific computing with Python
import matplotlib.pyplot as plt  # Matplotlib is a 2D plotting library
from scipy.optimize import root  # Package to find the roots of a function

"2|DEFINE PARAMETERS AND ARRAYS"
size = 50
T = 24                           # Available hours to work
# Demand parameters
K = 20                           # Capital stock
A = 20                           # Total factor productivity
alpha = 0.6                      # Output elasticity of capital
# Supply parameters
I = 50                           # Non-labor income
beta = 0.6                       # Utility elasticity of consumption
# Arrays
rW = np.arange(1, size)             # Real wage

"3|OPTIMIZATION PROBLEM: FIND EQUILIBRIUM VALUES"
def Ndemand(A, K, rW, alpha):
    Nd = K * ((1-alpha)*A/rW)**(1/alpha)
    return Nd

def Nsupply(rW, beta, I):
    Lsupply = T - (1-beta)*((24*rW + I)/rW)
    return Lsupply

def Eq_Wage(rW):
    Eq_Wage = Ndemand(A, K, rW, alpha) - Nsupply(rW, beta, I)
    return Eq_Wage

rW_0 = 10                               # Initial value (guess)
rW_star = root(Eq_Wage, rW_0)           # Equilibrium: Wage
N_star = Nsupply(rW_star.x, beta, I)    # Equilibrium: Labor

"4|PLOT LABOR MARKET EQUILIBRIUM"
Nd = Ndemand(A, K, rW, alpha)
Ns = Nsupply(rW, beta, I)

y_max = rW_star.x*2
v = [0, T, 0, y_max]                               # Set the axes range
fig, ax = plt.subplots(figsize=(10, 8))
ax.set(title="LABOR SUPPLY", xlabel="Work Hs.", ylabel=r'(w/P)')
ax.plot(Ns[1:T], rW[1:T], "k", label="Labor supply")
ax.plot(Nd[1:T], rW[1:T], "k", label="Labor demand")
plt.plot(N_star, rW_star.x, 'bo')
plt.axvline(x=N_star   , ymin=0, ymax=rW_star.x/y_max, ls=':', color='k')
plt.axhline(y=rW_star.x, xmin=0, xmax=N_star/T       , ls=':', color='k')
plt.text(5 , 20, "Labor demand")
plt.text(19,  9, "Labor supply")
plt.text(0.2       , rW_star.x+0.5, np.round(rW_star.x, 1))
plt.text(N_star+0.3, 0.3          , np.round(N_star, 1))
plt.axis(v)                                         # Use 'v' as the axes range
plt.show()
```


# Lockout: Quick User's Guide

A synthetic dataset was generated and partitioned into training, validation, and testing subsets and it is used througout this guide.

## Create DataLoaders
Change this section according to your particular problem to build training, validation, and testing DataLoaders.

```python
from lockout.pytorch_utils import load_data_reg, normalize_xy, make_DataLoaders
from lockout.utils import read_list

# Read Data
_, x_train, x_valid, x_test, _, y_train, y_valid, y_test = load_data_reg('dataset/')

# Normalize Data
xtrain, xvalid, xtest, ytrain, yvalid, ytest = normalize_xy(x_train, x_valid, x_test,
                                                            y_train, y_valid, y_test)

# Create DataLoaders
dl_train, dl_valid, dl_test = make_DataLoaders(xtrain, xvalid, xtest, ytrain, yvalid, ytest)
```

## Neural Network Architecture
Change this section to use a different architecture.

```python
from lockout.pytorch_utils import FCNN

# Number of Input features
n_features  = 100

# Nodes per layer:
# - len(layer_sizes) = number of layers in the network
# - layer_sizes[i]   = number of nodes in the ith layer
layer_sizes = [10, 1]

# Instantiate Neural Network (with ReLUs)
model_init = FCNN(n_features, layer_sizes)
model_init
```

    FCNN(
      (linear_layers): ModuleList(
        (0): Linear(in_features=100, out_features=10, bias=True)
        (1): Linear(in_features=10, out_features=1, bias=True)
      )
      (relu): ReLU(inplace=True)
    )

## Unconstrained Training
Change this section to use different hyperparameters. This problem is a regression task.

```python
from lockout import Lockout

# Learning rate
lr = 1e-2

# Loss function id:
# - loss_type = 1: Mean Squared Error
# - loss_type = 2: Mean Cross Entropy Loss
loss_type = 1

# Optimizer id:
# - optim_id = 1: Stochastic Gradient Descend
# - optim_id = 2: Adam
optim_id = 1

# Instantiate Lockout
lockout_forward = Lockout(model_init,
                          lr=lr,
                          loss_type=loss_type,
                          optim_id=optim_id)

# Train Neural Network Without Regularization
lockout_forward.train(dl_train, dl_valid,
                      train_how="unconstrained",
                      epochs=10000,              # Max number of epochs
                      early_stopping=20,         # Epoch used for convergence condition
                      tol_loss=1e-6,             # Tolerance in training loss used for convergence condition
                      reset_weights=True)        # Whether or not to reset weights before training
```

    Epochs:  53%|█████▎    | 5314/10000 [01:37<01:25, 54.74it/s]

    Early stopping = True
    Last iteration = 5315
    Best validation at iteration = 348

```python
from lockout.pytorch_utils import save_model

# Retrieve and Save Unconstrained Model
model_forward_unconstrained = lockout_forward.model_last
save_model(model_forward_unconstrained, 'model_forward_unconstrained.pth')

# Retrieve and Save Model At Validation Minimum
model_forward_best = lockout_forward.model_best_valid
save_model(model_forward_best, 'model_forward_best.pth')
```

```python
# Path Data
df0 = lockout_forward.path_data
df0.head()
```

      iteration
      train_loss
      valid_loss
      train_accu
      valid_accu

      0
      1
      1.118341
      1.188415
      -0.118341
      -0.105266

      1
      2
      1.111329
      1.182360
      -0.111329
      -0.099634

      2
      3
      1.104635
      1.176597
      -0.104635
      -0.094275

      3
      4
      1.098198
      1.171101
      -0.098198
      -0.089162

      4
      5
      1.091971
      1.165862
      -0.091972
      -0.084290

```python
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(9,6))

axes.plot(df0["iteration"], df0["train_loss"], label="Training", linewidth=4)
axes.plot(df0["iteration"], df0["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
plt.savefig("loss_vs_iter_forward.pdf", bbox_inches='tight')
plt.show()
```

## Lockout Training: Option 1
Within this option, the network is first trained until the regularization path is found (path 1). Then, the constraint value $t_0$ is iteratively decreased (path 2) with a step size
\begin{equation}
    \Delta t_0 =\frac{{t_0}_{\mathrm{final}} - {t_0}_{\mathrm{initial}}}{\mathrm{epochs}}.
\end{equation}
A small $\Delta t_0$ is necessary to stay on the regularization path.

```python
from lockout import Lockout

# List of tuples (or dictionary) of the form [(layer_name, regul_id)] where:
# - layer_name: layer name in the input model (string)
# - regul_id = 1: L1 regularization
# - regul_id = 2: Log regularization
regul_type = [('linear_layers.0.weight', 1)]

# List of tuples (or dictionary) of the form [(layer_name, path_flg)] where:
# - path_flg = True: the constraint t0 will be iteratively changed in this layer
# - path_flg = False: the constraint t0 will be kept constant in this layer
regul_path = [('linear_layers.0.weight', True)]

# Instantiate Lockout
lockout_option1 = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2,
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path)

# Train Neural Network With Lockout
lockout_option1.train(dl_train, dl_valid,
                      train_how="decrease_t0",
                      epochs=5000,              # Max number of epochs used in Path 1
                      epochs2=20000,            # Max number of epochs used in Path 2
                      early_stopping=20,
                      tol_loss=1e-5)
```

    Epochs1:  39%|███▉      | 1970/5000 [00:39<01:00, 50.05it/s]

    Path 1: Early stopping = True
            Last iteration = 1971

    Epochs2: 100%|██████████| 20000/20000 [06:49<00:00, 48.79it/s]

    Best validation at iteration = 19840

```python
# Save Model At Validation Minimum
model_lockout_option1 = lockout_option1.model_best_valid
save_model(model_lockout_option1, 'model_lockout_option1.pth')
```

```python
# Path Data
df1 = lockout_option1.path_data
df1.head()
```

      iteration
      sparcity__linear_layers.0.weight
      t0_calc__linear_layers.0.weight
      t0_used__linear_layers.0.weight
      train_loss
      valid_loss
      train_accu
      valid_accu

      0
      1
      1.0
      54.207119
      54.207119
      0.326455
      0.728214
      0.673545
      0.322737

      1
      2
      1.0
      54.207310
      54.207119
      0.325617
      0.728189
      0.674384
      0.322760

      2
      3
      1.0
      54.207184
      54.207119
      0.324770
      0.728236
      0.675230
      0.322716

      3
      4
      1.0
      54.207195
      54.207119
      0.323921
      0.728168
      0.676079
      0.322780

      4
      5
      1.0
      54.207092
      54.207119
      0.323038
      0.728198
      0.676962
      0.322752

```python
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(8,6))

axes.plot(df1["iteration"], df1["train_loss"], label="Training", linewidth=4)
axes.plot(df1["iteration"], df1["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
plt.savefig("loss_vs_iter_lockout1.pdf", bbox_inches='tight')
plt.show()
```

```python
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(16,7))

axes[0].plot(df0["iteration"], df0["train_loss"], label="Training", linewidth=4)
axes[0].plot(df0["iteration"], df0["valid_loss"], label="Validation", linewidth=4)
axes[0].legend(fontsize=16)
axes[0].set_xlabel("iteration", fontsize=16)
axes[0].set_ylabel("Loss Function", fontsize=16)
axes[0].tick_params(axis='both', which='major', labelsize=14)
axes[0].set_title("Unconstrained", fontsize=16)
axes[0].grid(True, zorder=2)
axes[0].set_ylim(0,1.3)

axes[1].plot(df1["iteration"], df1["train_loss"], label="Training", linewidth=4)
axes[1].plot(df1["iteration"], df1["valid_loss"], label="Validation", linewidth=4)
axes[1].legend(fontsize=16)
axes[1].set_xlabel("iteration", fontsize=16)
axes[1].set_ylabel("Loss Function", fontsize=16)
axes[1].tick_params(axis='both', which='major', labelsize=14)
axes[1].set_title("Lockout", fontsize=16)
axes[1].grid(True, zorder=2)
axes[1].set_ylim(0,1.3)

plt.tight_layout()
plt.show()
```

```python
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _  = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout1, _ = dataset_r2(dl_test, model_lockout_option1, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout1))
```

    Test R2 (unconstrained) = 0.655
    Test R2 (lockout)       = 0.972

```python
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option1, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout1.pdf', bbox_inches='tight')
plt.show()
```

## Lockout Training: Option 2
Within this option, a discrete set $t_0$ values are sampled. The set of $t_0$s can be either:

a) Entered externally as a 1D torch tensor:

```python
from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]
regul_path = [('linear_layers.0.weight', True)]

# List of tuples (or dictionary) of the form [(layer_name, t0_sampled)] where:
# - layer_name: layer name in the input model (string)
# - t0_sampled: 1D tensor with the constraint values t0 to be sampled in the layer
t0_sampled = torch.from_numpy(np.geomspace(53.504620, 1e-3, num=100, endpoint=True))
t0_grid    = {'linear_layers.0.weight': t0_sampled}

# Instantiate Lockout
lockout_option2a = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2,
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path,
                          t0_grid=t0_grid)

# Train Neural Network With Lockout
lockout_option2a.train(dl_train, dl_valid,
                      train_how="sampling_t0",
                      epochs=5000,             # Max number of epochs used for first t0
                      epochs2=200,             # Max number of epochs used for the rest of the t0 values
                      early_stopping=20,
                      tol_loss=1e-4)
```

    n = 1

    Epochs1:  19%|█▉        | 968/5000 [00:18<01:19, 50.95it/s]

    Early stopping = True (969)

    n = 2

    Epochs1:  72%|███████▎  | 145/200 [00:02<00:01, 49.56it/s]

    Early stopping = True (146)

    n = 3

    Epochs1:  90%|█████████ | 180/200 [00:03<00:00, 53.58it/s]

    Early stopping = True (181)

    n = 4

    Epochs1:  70%|██████▉   | 139/200 [00:02<00:01, 48.92it/s]

    Early stopping = True (140)

    n = 5

    Epochs1:  86%|████████▌ | 172/200 [00:03<00:00, 49.86it/s]

    Early stopping = True (173)

    n = 6

    Epochs1:  68%|██████▊   | 137/200 [00:02<00:01, 49.29it/s]

    Early stopping = True (138)

    n = 7

    Epochs1:  72%|███████▎  | 145/200 [00:02<00:01, 52.37it/s]

    Early stopping = True (146)

    n = 8

    Epochs1:  86%|████████▋ | 173/200 [00:03<00:00, 50.39it/s]

    Early stopping = True (174)

    n = 9

    Epochs1:  81%|████████  | 162/200 [00:03<00:00, 49.43it/s]

    Early stopping = True (163)

    n = 10

    Epochs1:  80%|████████  | 160/200 [00:03<00:00, 49.88it/s]

    Early stopping = True (161)

    n = 11

    Epochs1:  72%|███████▎  | 145/200 [00:02<00:01, 53.02it/s]

    Early stopping = True (146)

    n = 12

    Epochs1:  96%|█████████▌| 191/200 [00:03<00:00, 50.31it/s]

    Early stopping = True (192)

    n = 13

    Epochs1:  59%|█████▉    | 118/200 [00:02<00:01, 48.09it/s]

    Early stopping = True (119)

    n = 14

    Epochs1:  76%|███████▋  | 153/200 [00:03<00:00, 49.63it/s]

    Early stopping = True (154)

    n = 15

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.22it/s]

    Early stopping = False (200)

    n = 16

    Epochs1:  44%|████▎     | 87/200 [00:01<00:02, 53.06it/s]

    Early stopping = True (88)

    n = 17

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.81it/s]

    Early stopping = False (200)

    n = 18

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.77it/s]

    Early stopping = False (200)

    n = 19

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.16it/s]

    Early stopping = False (200)

    n = 20

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.86it/s]

    Early stopping = False (200)

    n = 21

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.88it/s]

    Early stopping = False (200)

    n = 22

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.55it/s]

    Early stopping = False (200)

    n = 23

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.35it/s]

    Early stopping = False (200)

    n = 24

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.89it/s]

    Early stopping = False (200)

    n = 25

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.00it/s]

    Early stopping = False (200)

    n = 26

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.71it/s]

    Early stopping = False (200)

    n = 27

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.05it/s]

    Early stopping = False (200)

    n = 28

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.66it/s]

    Early stopping = False (200)

    n = 29

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.06it/s]

    Early stopping = False (200)

    n = 30

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.74it/s]

    Early stopping = False (200)

    n = 31

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 52.81it/s]

    Early stopping = False (200)

    n = 32

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.87it/s]

    Early stopping = False (200)

    n = 33

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.31it/s]

    Early stopping = False (200)

    n = 34

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.51it/s]

    Early stopping = False (200)

    n = 35

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.31it/s]

    Early stopping = False (200)

    n = 36

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.49it/s]

    Early stopping = False (200)

    n = 37

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.37it/s]

    Early stopping = False (200)

    n = 38

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.32it/s]

    Early stopping = False (200)

    n = 39

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.21it/s]

    Early stopping = False (200)

    n = 40

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.56it/s]

    Early stopping = False (200)

    n = 41

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.62it/s]

    Early stopping = False (200)

    n = 42

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.70it/s]

    Early stopping = False (200)

    n = 43

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.90it/s]

    Early stopping = False (200)

    n = 44

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.26it/s]

    Early stopping = False (200)

    n = 45

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.69it/s]

    Early stopping = False (200)

    n = 46

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.06it/s]

    Early stopping = False (200)

    n = 47

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.30it/s]

    Early stopping = False (200)

    n = 48

    Epochs1:  32%|███▏      | 64/200 [00:01<00:02, 52.85it/s]

    Early stopping = True (65)

    n = 49

    Epochs1:  26%|██▌       | 52/200 [00:00<00:02, 53.03it/s]

    Early stopping = True (53)

    n = 50

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.09it/s]

    Early stopping = False (200)

    n = 51

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.06it/s]

    Early stopping = False (200)

    n = 52

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.90it/s]

    Early stopping = False (200)

    n = 53

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.13it/s]

    Early stopping = False (200)

    n = 54

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.99it/s]

    Early stopping = False (200)

    n = 55

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 49.48it/s]

    Early stopping = False (200)

    n = 56

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 49.56it/s]

    Early stopping = False (200)

    n = 57

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.33it/s]

    Early stopping = False (200)

    n = 58

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.20it/s]

    Early stopping = False (200)

    n = 59

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.86it/s]

    Early stopping = False (200)

    n = 60

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 53.05it/s]

    Early stopping = False (200)

    n = 61

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.00it/s]

    Early stopping = False (200)

    n = 62

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 53.90it/s]

    Early stopping = False (200)

    n = 63

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.83it/s]

    Early stopping = False (200)

    n = 64

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.04it/s]

    Early stopping = False (200)

    n = 65

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.98it/s]

    Early stopping = False (200)

    n = 66

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.14it/s]

    Early stopping = False (200)

    n = 67

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.22it/s]

    Early stopping = False (200)

    n = 68

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 49.96it/s]

    Early stopping = False (200)

    n = 69

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.37it/s]

    Early stopping = False (200)

    n = 70

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.72it/s]

    Early stopping = False (200)

    n = 71

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.58it/s]

    Early stopping = False (200)

    n = 72

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.86it/s]

    Early stopping = False (200)

    n = 73

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.04it/s]

    Early stopping = False (200)

    n = 74

    Epochs1:  11%|█         | 22/200 [00:00<00:03, 50.16it/s]

    Early stopping = True (23)

    n = 75

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 49.79it/s]

    Early stopping = True (22)

    n = 76

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.71it/s]

    Early stopping = True (22)

    n = 77

    Epochs1:  10%|█         | 21/200 [00:00<00:05, 34.40it/s]

    Early stopping = True (22)

    n = 78

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.72it/s]

    Early stopping = True (22)

    n = 79

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 49.92it/s]

    Early stopping = True (22)

    n = 80

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.04it/s]

    Early stopping = True (22)

    n = 81

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.48it/s]

    Early stopping = True (22)

    n = 82

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.38it/s]

    Early stopping = True (22)

    n = 83

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.37it/s]

    Early stopping = True (22)

    n = 84

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.59it/s]

    Early stopping = True (22)

    n = 85

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.59it/s]

    Early stopping = True (22)

    n = 86

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 49.69it/s]

    Early stopping = True (22)

    n = 87

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 49.89it/s]

    Early stopping = True (22)

    n = 88

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.45it/s]

    Early stopping = True (22)

    n = 89

    Epochs1:  10%|█         | 21/200 [00:00<00:05, 33.96it/s]

    Early stopping = True (22)

    n = 90

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.47it/s]

    Early stopping = True (22)

    n = 91

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.20it/s]

    Early stopping = True (22)

    n = 92

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 48.71it/s]

    Early stopping = True (22)

    n = 93

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.21it/s]

    Early stopping = True (22)

    n = 94

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.17it/s]

    Early stopping = True (22)

    n = 95

    Epochs1:  10%|█         | 21/200 [00:00<00:05, 33.69it/s]

    Early stopping = True (22)

    n = 96

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 51.05it/s]

    Early stopping = True (22)

    n = 97

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.29it/s]

    Early stopping = True (22)

    n = 98

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.29it/s]

    Early stopping = True (22)

    n = 99

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.38it/s]

    Early stopping = True (22)

    n = 100

    Epochs1:  10%|█         | 21/200 [00:00<00:03, 50.45it/s]

    Early stopping = True (22)

    Best validation at iteration = 47

```python
# Save Model At Validation Minimum
model_lockout_option2a = lockout_option2a.model_best_valid
save_model(model_lockout_option2a, 'model_lockout_option2a.pth')
```

```python
# Path Data
df2a = lockout_option2a.path_data
df2a.head()
```

      iteration
      sparcity__linear_layers.0.weight
      t0_calc__linear_layers.0.weight
      t0_used__linear_layers.0.weight
      train_loss
      valid_loss
      train_accu
      valid_accu

      0
      969
      1.0
      53.504848
      53.504620
      0.036792
      0.882386
      0.963208
      0.179351

      1
      146
      1.0
      47.933159
      47.932464
      0.038469
      0.869678
      0.961531
      0.191170

      2
      181
      1.0
      42.942211
      42.940612
      0.038247
      0.862440
      0.961753
      0.197902

      3
      140
      1.0
      38.469421
      38.468628
      0.043386
      0.850439
      0.956614
      0.209064

      4
      173
      1.0
      34.464169
      34.462372
      0.046387
      0.840710
      0.953613
      0.218112

```python
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option2a, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout2a.pdf', bbox_inches='tight')
plt.show()
```

```python
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _   = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout2a, _ = dataset_r2(dl_test, model_lockout_option2a, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout2a))
```

    Test R2 (unconstrained) = 0.655
    Test R2 (lockout)       = 0.953

b) Generated internally, in which case the $t_0$ points are linearly sampled

```python
from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]
regul_path = [('linear_layers.0.weight', True)]

# List of tuples (or dictionary) of the form [(layer_name, t0_number)] where:
# - layer_name: layer name in the input model (string)
# - t0_number: number of constraint values t0 to be linearly sampled (integer)
t0_points    = {'linear_layers.0.weight': 200}

# Instantiate Lockout
lockout_option2b = Lockout(lockout_forward.model_best_valid,  # initial network
                          lr=1e-2,
                          loss_type=1,
                          regul_type=regul_type,
                          regul_path=regul_path,
                          t0_points=t0_points)

# Train Neural Network With Lockout
lockout_option2b.train(dl_train, dl_valid,
                      train_how="sampling_t0",
                      epochs=5000,             # Max number of epochs used for first t0
                      epochs2=200,             # Max number of epochs used for the rest of the t0 values
                      early_stopping=20,
                      tol_loss=1e-4)
```

    n = 1

    Epochs1:  20%|█▉        | 979/5000 [00:19<01:18, 51.33it/s]

    Early stopping = True (980)

    n = 2

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.66it/s]

    Early stopping = True (24)

    n = 3

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.83it/s]

    Early stopping = True (24)

    n = 4

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.10it/s]

    Early stopping = True (24)

    n = 5

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.00it/s]

    Early stopping = True (25)

    n = 6

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.16it/s]

    Early stopping = True (24)

    n = 7

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.55it/s]

    Early stopping = True (24)

    n = 8

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.13it/s]

    Early stopping = True (24)

    n = 9

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.89it/s]

    Early stopping = True (24)

    n = 10

    Epochs1:  12%|█▏        | 23/200 [00:00<00:04, 35.69it/s]

    Early stopping = True (24)

    n = 11

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.98it/s]

    Early stopping = True (24)

    n = 12

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.60it/s]

    Early stopping = True (24)

    n = 13

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.60it/s]

    Early stopping = True (24)

    n = 14

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.52it/s]

    Early stopping = True (24)

    n = 15

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.69it/s]

    Early stopping = True (24)

    n = 16

    Epochs1:  12%|█▏        | 23/200 [00:00<00:04, 35.41it/s]

    Early stopping = True (24)

    n = 17

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.48it/s]

    Early stopping = True (24)

    n = 18

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.23it/s]

    Early stopping = True (24)

    n = 19

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.74it/s]

    Early stopping = True (24)

    n = 20

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.92it/s]

    Early stopping = True (24)

    n = 21

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.11it/s]

    Early stopping = True (24)

    n = 22

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.88it/s]

    Early stopping = True (24)

    n = 23

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.52it/s]

    Early stopping = True (24)

    n = 24

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.20it/s]

    Early stopping = True (24)

    n = 25

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.87it/s]

    Early stopping = True (24)

    n = 26

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.93it/s]

    Early stopping = True (24)

    n = 27

    Epochs1:  12%|█▏        | 23/200 [00:00<00:05, 33.91it/s]

    Early stopping = True (24)

    n = 28

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.24it/s]

    Early stopping = True (24)

    n = 29

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.59it/s]

    Early stopping = True (24)

    n = 30

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.37it/s]

    Early stopping = True (24)

    n = 31

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.75it/s]

    Early stopping = True (24)

    n = 32

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.59it/s]

    Early stopping = True (24)

    n = 33

    Epochs1:  12%|█▏        | 23/200 [00:00<00:04, 35.57it/s]

    Early stopping = True (24)

    n = 34

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.04it/s]

    Early stopping = True (24)

    n = 35

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.96it/s]

    Early stopping = True (24)

    n = 36

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.62it/s]

    Early stopping = True (24)

    n = 37

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.27it/s]

    Early stopping = True (24)

    n = 38

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.94it/s]

    Early stopping = True (24)

    n = 39

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.32it/s]

    Early stopping = True (24)

    n = 40

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.22it/s]

    Early stopping = True (24)

    n = 41

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.23it/s]

    Early stopping = True (24)

    n = 42

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.10it/s]

    Early stopping = True (24)

    n = 43

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.08it/s]

    Early stopping = True (24)

    n = 44

    Epochs1:  12%|█▏        | 23/200 [00:00<00:04, 35.57it/s]

    Early stopping = True (24)

    n = 45

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.31it/s]

    Early stopping = True (24)

    n = 46

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.89it/s]

    Early stopping = True (24)

    n = 47

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.32it/s]

    Early stopping = True (24)

    n = 48

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.84it/s]

    Early stopping = True (24)

    n = 49

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.52it/s]

    Early stopping = True (24)

    n = 50

    Epochs1:  12%|█▏        | 23/200 [00:00<00:05, 35.03it/s]

    Early stopping = True (24)

    n = 51

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.45it/s]

    Early stopping = True (24)

    n = 52

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.92it/s]

    Early stopping = True (24)

    n = 53

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.60it/s]

    Early stopping = True (24)

    n = 54

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.78it/s]

    Early stopping = True (24)

    n = 55

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.90it/s]

    Early stopping = True (24)

    n = 56

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 51.09it/s]

    Early stopping = True (24)

    n = 57

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.81it/s]

    Early stopping = True (24)

    n = 58

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.41it/s]

    Early stopping = True (24)

    n = 59

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.76it/s]

    Early stopping = True (24)

    n = 60

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 49.62it/s]

    Early stopping = True (24)

    n = 61

    Epochs1:  12%|█▏        | 23/200 [00:00<00:05, 35.17it/s]

    Early stopping = True (24)

    n = 62

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.89it/s]

    Early stopping = True (24)

    n = 63

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.44it/s]

    Early stopping = True (24)

    n = 64

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.65it/s]

    Early stopping = True (25)

    n = 65

    Epochs1:  12%|█▏        | 23/200 [00:00<00:03, 50.61it/s]

    Early stopping = True (24)

    n = 66

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.80it/s]

    Early stopping = True (25)

    n = 67

    Epochs1:  12%|█▏        | 24/200 [00:00<00:04, 35.67it/s]

    Early stopping = True (25)

    n = 68

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.34it/s]

    Early stopping = True (25)

    n = 69

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 49.39it/s]

    Early stopping = True (25)

    n = 70

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.32it/s]

    Early stopping = True (25)

    n = 71

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.93it/s]

    Early stopping = True (25)

    n = 72

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.69it/s]

    Early stopping = True (25)

    n = 73

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.26it/s]

    Early stopping = True (25)

    n = 74

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.06it/s]

    Early stopping = True (25)

    n = 75

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.10it/s]

    Early stopping = True (25)

    n = 76

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.33it/s]

    Early stopping = True (25)

    n = 77

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.96it/s]

    Early stopping = True (25)

    n = 78

    Epochs1:  12%|█▏        | 24/200 [00:00<00:04, 35.88it/s]

    Early stopping = True (25)

    n = 79

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.03it/s]

    Early stopping = True (25)

    n = 80

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.45it/s]

    Early stopping = True (25)

    n = 81

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.92it/s]

    Early stopping = True (25)

    n = 82

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.00it/s]

    Early stopping = True (25)

    n = 83

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.87it/s]

    Early stopping = True (25)

    n = 84

    Epochs1:  12%|█▏        | 24/200 [00:00<00:04, 35.85it/s]

    Early stopping = True (25)

    n = 85

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.11it/s]

    Early stopping = True (25)

    n = 86

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.89it/s]

    Early stopping = True (25)

    n = 87

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.89it/s]

    Early stopping = True (25)

    n = 88

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.68it/s]

    Early stopping = True (25)

    n = 89

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.32it/s]

    Early stopping = True (25)

    n = 90

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.08it/s]

    Early stopping = True (25)

    n = 91

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.82it/s]

    Early stopping = True (25)

    n = 92

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 51.08it/s]

    Early stopping = True (26)

    n = 93

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.91it/s]

    Early stopping = True (25)

    n = 94

    Epochs1:  12%|█▎        | 25/200 [00:00<00:04, 36.45it/s]

    Early stopping = True (26)

    n = 95

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.68it/s]

    Early stopping = True (25)

    n = 96

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.48it/s]

    Early stopping = True (25)

    n = 97

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.65it/s]

    Early stopping = True (25)

    n = 98

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.40it/s]

    Early stopping = True (25)

    n = 99

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.74it/s]

    Early stopping = True (25)

    n = 100

    Epochs1:  12%|█▏        | 24/200 [00:00<00:04, 35.69it/s]

    Early stopping = True (25)

    n = 101

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.06it/s]

    Early stopping = True (25)

    n = 102

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.61it/s]

    Early stopping = True (25)

    n = 103

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.52it/s]

    Early stopping = True (25)

    n = 104

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.21it/s]

    Early stopping = True (25)

    n = 105

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.90it/s]

    Early stopping = True (25)

    n = 106

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.83it/s]

    Early stopping = True (25)

    n = 107

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 47.57it/s]

    Early stopping = True (25)

    n = 108

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 49.74it/s]

    Early stopping = True (25)

    n = 109

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.91it/s]

    Early stopping = True (25)

    n = 110

    Epochs1:  12%|█▏        | 24/200 [00:00<00:05, 34.01it/s]

    Early stopping = True (25)

    n = 111

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 50.33it/s]

    Early stopping = True (26)

    n = 112

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 49.94it/s]

    Early stopping = True (26)

    n = 113

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 49.78it/s]

    Early stopping = True (25)

    n = 114

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 49.76it/s]

    Early stopping = True (25)

    n = 115

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 49.70it/s]

    Early stopping = True (25)

    n = 116

    Epochs1:  12%|█▏        | 24/200 [00:00<00:05, 35.18it/s]

    Early stopping = True (25)

    n = 117

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 50.12it/s]

    Early stopping = True (26)

    n = 118

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 50.32it/s]

    Early stopping = True (25)

    n = 119

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 50.56it/s]

    Early stopping = True (26)

    n = 120

    Epochs1:  12%|█▏        | 24/200 [00:00<00:03, 51.19it/s]

    Early stopping = True (25)

    n = 121

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.29it/s]

    Early stopping = True (27)

    n = 122

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.32it/s]

    Early stopping = True (27)

    n = 123

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.67it/s]

    Early stopping = True (27)

    n = 124

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.18it/s]

    Early stopping = True (27)

    n = 125

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.61it/s]

    Early stopping = True (27)

    n = 126

    Epochs1:  13%|█▎        | 26/200 [00:00<00:04, 36.93it/s]

    Early stopping = True (27)

    n = 127

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 50.07it/s]

    Early stopping = True (27)

    n = 128

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 49.74it/s]

    Early stopping = True (27)

    n = 129

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 50.49it/s]

    Early stopping = True (27)

    n = 130

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 49.24it/s]

    Early stopping = True (27)

    n = 131

    Epochs1:  14%|█▎        | 27/200 [00:00<00:04, 37.29it/s]

    Early stopping = True (28)

    n = 132

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 50.57it/s]

    Early stopping = True (27)

    n = 133

    Epochs1:  14%|█▎        | 27/200 [00:00<00:03, 50.31it/s]

    Early stopping = True (28)

    n = 134

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.11it/s]

    Early stopping = True (27)

    n = 135

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 50.45it/s]

    Early stopping = True (27)

    n = 136

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.37it/s]

    Early stopping = True (27)

    n = 137

    Epochs1:  16%|█▌        | 32/200 [00:00<00:03, 51.16it/s]

    Early stopping = True (33)

    n = 138

    Epochs1:  12%|█▎        | 25/200 [00:00<00:03, 50.44it/s]

    Early stopping = True (26)

    n = 139

    Epochs1:  14%|█▍        | 28/200 [00:00<00:03, 51.16it/s]

    Early stopping = True (29)

    n = 140

    Epochs1:  20%|██        | 41/200 [00:01<00:03, 39.90it/s]

    Early stopping = True (42)

    n = 141

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 49.42it/s]

    Early stopping = True (27)

    n = 142

    Epochs1:  14%|█▎        | 27/200 [00:00<00:03, 49.73it/s]

    Early stopping = True (28)

    n = 143

    Epochs1:  19%|█▉        | 38/200 [00:00<00:03, 51.64it/s]

    Early stopping = True (39)

    n = 144

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 51.33it/s]

    Early stopping = True (27)

    n = 145

    Epochs1:  16%|█▋        | 33/200 [00:00<00:04, 39.12it/s]

    Early stopping = True (34)

    n = 146

    Epochs1:  18%|█▊        | 35/200 [00:00<00:03, 50.61it/s]

    Early stopping = True (36)

    n = 147

    Epochs1:  13%|█▎        | 26/200 [00:00<00:03, 50.25it/s]

    Early stopping = True (27)

    n = 148

    Epochs1:  15%|█▌        | 30/200 [00:00<00:03, 51.64it/s]

    Early stopping = True (31)

    n = 149

    Epochs1:  14%|█▎        | 27/200 [00:00<00:03, 50.72it/s]

    Early stopping = True (28)

    n = 150

    Epochs1:  27%|██▋       | 54/200 [00:01<00:02, 52.01it/s]

    Early stopping = True (55)

    n = 151

    Epochs1:  28%|██▊       | 56/200 [00:01<00:02, 52.46it/s]

    Early stopping = True (57)

    n = 152

    Epochs1:  14%|█▍        | 29/200 [00:00<00:04, 37.96it/s]

    Early stopping = True (30)

    n = 153

    Epochs1:  48%|████▊     | 96/200 [00:01<00:01, 52.64it/s]

    Early stopping = True (97)

    n = 154

    Epochs1:  14%|█▍        | 28/200 [00:00<00:03, 51.41it/s]

    Early stopping = True (29)

    n = 155

    Epochs1:  16%|█▋        | 33/200 [00:00<00:04, 39.34it/s]

    Early stopping = True (34)

    n = 156

    Epochs1:  26%|██▌       | 52/200 [00:00<00:02, 52.23it/s]

    Early stopping = True (53)

    n = 157

    Epochs1:  16%|█▋        | 33/200 [00:00<00:03, 51.01it/s]

    Early stopping = True (34)

    n = 158

    Epochs1:  38%|███▊      | 77/200 [00:01<00:02, 52.35it/s]

    Early stopping = True (78)

    n = 159

    Epochs1:  24%|██▍       | 48/200 [00:00<00:02, 50.72it/s]

    Early stopping = True (49)

    n = 160

    Epochs1:  78%|███████▊  | 157/200 [00:03<00:00, 49.62it/s]

    Early stopping = True (158)

    n = 161

    Epochs1:  85%|████████▌ | 170/200 [00:03<00:00, 50.58it/s]

    Early stopping = True (171)

    n = 162

    Epochs1:  16%|█▋        | 33/200 [00:00<00:03, 50.89it/s]

    Early stopping = True (34)

    n = 163

    Epochs1:  25%|██▌       | 50/200 [00:00<00:02, 52.41it/s]

    Early stopping = True (51)

    n = 164

    Epochs1:  40%|████      | 81/200 [00:01<00:02, 46.23it/s]

    Early stopping = True (82)

    n = 165

    Epochs1:  36%|███▌      | 71/200 [00:01<00:02, 52.18it/s]

    Early stopping = True (72)

    n = 166

    Epochs1:  18%|█▊        | 37/200 [00:00<00:03, 40.80it/s]

    Early stopping = True (38)

    n = 167

    Epochs1:  98%|█████████▊| 196/200 [00:03<00:00, 53.91it/s]

    Early stopping = True (197)

    n = 168

    Epochs1:  16%|█▌        | 32/200 [00:00<00:03, 52.06it/s]

    Early stopping = True (33)

    n = 169

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.95it/s]

    Early stopping = False (200)

    n = 170

    Epochs1:  70%|███████   | 141/200 [00:02<00:01, 53.73it/s]

    Early stopping = True (142)

    n = 171

    Epochs1:  34%|███▍      | 68/200 [00:01<00:02, 53.40it/s]

    Early stopping = True (69)

    n = 172

    Epochs1:  80%|████████  | 161/200 [00:03<00:00, 50.41it/s]

    Early stopping = True (162)

    n = 173

    Epochs1:  67%|██████▋   | 134/200 [00:02<00:01, 49.63it/s]

    Early stopping = True (135)

    n = 174

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.56it/s]

    Early stopping = False (200)

    n = 175

    Epochs1:  19%|█▉        | 38/200 [00:00<00:03, 51.77it/s]

    Early stopping = True (39)

    n = 176

    Epochs1:  31%|███       | 62/200 [00:01<00:02, 52.83it/s]

    Early stopping = True (63)

    n = 177

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.50it/s]

    Early stopping = False (200)

    n = 178

    Epochs1:  60%|█████▉    | 119/200 [00:02<00:01, 49.26it/s]

    Early stopping = True (120)

    n = 179

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.58it/s]

    Early stopping = False (200)

    n = 180

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.58it/s]

    Early stopping = False (200)

    n = 181

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.33it/s]

    Early stopping = False (200)

    n = 182

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.62it/s]

    Early stopping = False (200)

    n = 183

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.17it/s]

    Early stopping = False (200)

    n = 184

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 53.38it/s]

    Early stopping = False (200)

    n = 185

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.96it/s]

    Early stopping = False (200)

    n = 186

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 54.07it/s]

    Early stopping = False (200)

    n = 187

    Epochs1: 100%|██████████| 200/200 [00:04<00:00, 48.92it/s]

    Early stopping = False (200)

    n = 188

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 53.97it/s]

    Early stopping = False (200)

    n = 189

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.34it/s]

    Early stopping = False (200)

    n = 190

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.30it/s]

    Early stopping = False (200)

    n = 191

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.33it/s]

    Early stopping = False (200)

    n = 192

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.07it/s]

    Early stopping = False (200)

    n = 193

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.87it/s]

    Early stopping = False (200)

    n = 194

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.85it/s]

    Early stopping = False (200)

    n = 195

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.84it/s]

    Early stopping = False (200)

    n = 196

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.91it/s]

    Early stopping = False (200)

    n = 197

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.10it/s]

    Early stopping = False (200)

    n = 198

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.03it/s]

    Early stopping = False (200)

    n = 199

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 50.67it/s]

    Early stopping = False (200)

    n = 200

    Epochs1: 100%|██████████| 200/200 [00:03<00:00, 51.08it/s]

    Early stopping = False (200)

    Best validation at iteration = 198

```python
# Path Data
df2b = lockout_option2b.path_data
df2b.head()
```

      iteration
      sparcity__linear_layers.0.weight
      t0_calc__linear_layers.0.weight
      t0_used__linear_layers.0.weight
      train_loss
      valid_loss
      train_accu
      valid_accu

      0
      980
      1.0
      54.207325
      54.207119
      0.034422
      0.891790
      0.965578
      0.170606

      1
      24
      1.0
      53.935425
      53.934721
      0.033113
      0.893095
      0.966887
      0.169392

      2
      24
      1.0
      53.663013
      53.662324
      0.031916
      0.895570
      0.968084
      0.167090

      3
      24
      1.0
      53.390324
      53.389926
      0.030784
      0.897732
      0.969216
      0.165079

      4
      25
      1.0
      53.118298
      53.117529
      0.029654
      0.898822
      0.970346
      0.164066

```python
# Save Model At Validation Minimum
model_lockout_option2b = lockout_option2b.model_best_valid
save_model(model_lockout_option2b, 'model_lockout_option2b.pth')
```

```python
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _   = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout2b, _ = dataset_r2(dl_test, model_lockout_option2b, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout2b))
```

    Test R2 (unconstrained) = 0.655
    Test R2 (lockout)       = 0.956

```python
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option2b, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout2b.pdf', bbox_inches='tight')
plt.show()
```

## Lockout Training: Option 3
Within this option, the neural network is trained with a single (constant) value of $t_0$ until the regularization path is found.

```python
from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]

# List of tuples (or dictionary) of the form [(layer_name, t0_value)] where:
# - layer_name: layer name in the input model (string)
# - t0_value: constraint value t0 used in the layer (tensor)
t0 = {'linear_layers.0.weight': torch.tensor(0.4)}

# Instantiate Lockout
lockout_option3 = Lockout(model_lockout_option2b,  # initial network
                          lr=1e-3,
                          loss_type=1,
                          regul_type=regul_type,
                          t0=t0)

# Train Neural Network With Lockout
lockout_option3.train(dl_train, dl_valid,
                      train_how="constant_t0",
                      epochs=10000,              # Max number of epochs
                      early_stopping=20,
                      tol_loss=1e-5)
```

    Epochs1:  11%|█         | 1070/10000 [00:20<02:52, 51.66it/s]

    Early stopping = True
    Last iteration = 1071
    Best validation at iteration = 936

```python
# Path Data
df3 = lockout_option3.path_data
df3.tail()
```

      iteration
      sparcity__linear_layers.0.weight
      t0_calc__linear_layers.0.weight
      t0_used__linear_layers.0.weight
      train_loss
      valid_loss
      train_accu
      valid_accu

      1066
      1067
      0.07
      0.4
      0.4
      0.501023
      0.565949
      0.498977
      0.473649

      1067
      1068
      0.07
      0.4
      0.4
      0.501015
      0.565923
      0.498985
      0.473673

      1068
      1069
      0.07
      0.4
      0.4
      0.501021
      0.565942
      0.498979
      0.473655

      1069
      1070
      0.07
      0.4
      0.4
      0.501016
      0.566030
      0.498984
      0.473573

      1070
      1071
      0.07
      0.4
      0.4
      0.501018
      0.566230
      0.498982
      0.473387

```python
# Save Model At Regularization Path
from lockout.pytorch_utils import save_model

model_lockout_option3 = lockout_option3.model_last
save_model(model_lockout_option3, 'model_lockout_option3.pth')
```

```python
# Compute Test Accuracy
import torch
from lockout.pytorch_utils import dataset_r2

device = torch.device('cpu')
r2_test_forward, _  = dataset_r2(dl_test, model_forward_best, device)
r2_test_lockout3, _ = dataset_r2(dl_test, model_lockout_option3, device)
print("Test R2 (unconstrained) = {:.3f}".format(r2_test_forward))
print("Test R2 (lockout)       = {:.3f}".format(r2_test_lockout3))
```

    Test R2 (unconstrained) = 0.655
    Test R2 (lockout)       = 0.954

```python
# Graph Feature Importance
import matplotlib.pyplot as plt
import numpy as np
from lockout.pytorch_utils import get_features_importance

importance = get_features_importance(model_lockout_option3, 'linear_layers.0.weight')

fig, axes = plt.subplots(figsize=(9,6))
x_pos = np.arange(len(importance))
axes.bar(x_pos, importance, zorder=2)
axes.set_xticks(x_pos)
axes.set_xticklabels(importance.index, rotation='vertical')
axes.set_xlim(-1,len(x_pos))
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_ylabel('Importance', fontsize=16)
axes.set_xlabel('feature', fontsize=16)
axes.set_title('Lockout', fontsize=16)
axes.grid(True, zorder=1)

plt.tight_layout()
plt.savefig('feature_importance_lockout3.pdf', bbox_inches='tight')
plt.show()
```

## Lockout Training: Option 4
Within this option, the neural network is trained until the regularization path is found. The contraint value used, $t_0$ is computed from the input model.

```python
from lockout import Lockout

regul_type = [('linear_layers.0.weight', 1)]

# Instantiate Lockout
lockout_option4 = Lockout(model_forward_best,  # initial network
                          lr=1e-2,
                          loss_type=1,
                          regul_type=regul_type)

# Train Neural Network With Lockout
lockout_option4.train(dl_train, dl_valid,
                      train_how="until_path",
                      epochs=10000,            # Max number of epochs
                      early_stopping=20,
                      tol_loss=1e-5)
```

    Epochs1:  20%|██        | 2011/10000 [00:38<02:32, 52.46it/s]

    Early stopping = True
    Last iteration = 2012
    Best validation at iteration = 4

```python
# Path Data
df4 = lockout_option4.path_data
df4.tail()
```

      iteration
      sparcity__linear_layers.0.weight
      t0_calc__linear_layers.0.weight
      t0_used__linear_layers.0.weight
      train_loss
      valid_loss
      train_accu
      valid_accu

      2007
      2008
      1.0
      54.208427
      54.207119
      0.002655
      0.994821
      0.997345
      0.074784

      2008
      2009
      1.0
      54.208115
      54.207119
      0.002647
      0.994602
      0.997353
      0.074987

      2009
      2010
      1.0
      54.208317
      54.207119
      0.002641
      0.994861
      0.997359
      0.074746

      2010
      2011
      1.0
      54.207520
      54.207119
      0.002634
      0.994811
      0.997366
      0.074793

      2011
      2012
      1.0
      54.208824
      54.207119
      0.002625
      0.995240
      0.997375
      0.074394

```python
# Plot loss vs iteration
import matplotlib.pyplot as plt
import numpy as np

fig, axes = plt.subplots(figsize=(8,6))

axes.plot(df4["iteration"], df4["train_loss"], label="Training", linewidth=4)
axes.plot(df4["iteration"], df4["valid_loss"], label="Validation", linewidth=4)
axes.legend(fontsize=16)
axes.set_xlabel("iteration", fontsize=16)
axes.set_ylabel("Loss Function", fontsize=16)
axes.tick_params(axis='both', which='major', labelsize=14)
axes.set_title("Unconstrained", fontsize=16)
axes.grid(True, zorder=2)

plt.tight_layout()
# plt.savefig("loss_vs_iter_lockout4.pdf", bbox_inches='tight')
plt.show()
```

```python

```

```python

```