$\Large{\text{Kernel machines}}$

Let us first generate a synthetic data set to illustrate the deficiency of linear support vector machines.

We shall generate $n$ samples where each sample is 2-dimensional of the form $x^i = (x^i_1,x^i_2), i \in \{1,2,\ldots,n\}$. For each sample $x^i$ we have the following labeling scheme:

$
\begin{align}
y^i = \begin{cases}
        +1 \text{ if } (x^i_1-1)^2 + (x^i_2-1)^2 < 1 \\
        -1 \text{ else. }
      \end{cases}
\end{align}
$

Hence the circle $(x^i_1-1)^2 + (x^i_2-1)^2 = 1$ (or equivalently $(x^i_1)^2 -2x^i_1 + (x^i_2)^2 -2x^i_2 = -1)$  forms a separating surface for the data.

```python
import numpy as np
import matplotlib.pyplot as plt
num_samples = 100

#seed the random number generator
np.random.seed(100)

X = []
y = []

radius = 1.

center_point = np.array([1.,1.])
center_x1 = center_point[0]
center_x2 = center_point[1]

#generate points within circle (x1-1)^2 + (x2-1)^2 = 1
for i in range(int(num_samples/2)):
  toss = np.random.rand()
  x1 = np.random.uniform(low=0, high=2)
  x2 = np.random.uniform(low=0, high=2)
  while (x1-center_x1)**2+(x2-center_x2)**2>radius:
    x1 = np.random.uniform(low=0, high=2)
    x2 = np.random.uniform(low=0, high=2)

  #print('x1:',x1,'x2:',x2,(x1-center_x1)**2+(x2-center_x2)**2)
  X.append([x1,x2])
  y.append([1])

#generate points outside circle (x1-1)^2 + (x2-1)^2 = 1
for i in range(int(num_samples/2)):
  toss = np.random.rand()
  x1 = np.random.uniform(low=0, high=2)
  x2 = np.random.uniform(low=0, high=2)
  while (x1-center_x1)**2+(x2-center_x2)**2<=radius:
    x1 = np.random.uniform(low=0, high=2)
    x2 = np.random.uniform(low=0, high=2)

  #print('x1:',x1,'x2:',x2,(x1-center_x1)**2+(x2-center_x2)**2)
  X.append([x1,x2])
  y.append([-1])

X = np.array(X)
y = np.array(y)

#print('X:',X)
#print('y:',y)

#plot the circle
angle = np.linspace(0,2*np.pi,150)

x1 = center_x1+radius * np.cos( angle )
x2 = center_x2+radius * np.sin( angle )

figure, axes = plt.subplots(1)

axes.plot(x1,x2)
axes.set_aspect(1)

plt.scatter(X[:int(num_samples/2),0],X[:int(num_samples/2),1], marker='o', color='red')
plt.scatter(X[int(num_samples/2):,0],X[int(num_samples/2):,1], marker='s', color='green')

plt.title( 'Data with a non-linear separating surface' )
plt.show()
```

```python
#check the decision boundary obtained from Linear SVM
from sklearn.svm import LinearSVC #import linear SVM

clf = LinearSVC(C=1.0)
clf_model = clf.fit(X,y.ravel())

# create a mesh to plot in
h=0.02 #mesh step size
x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, h),
                     np.arange(x2_min, x2_max, h))

Z = clf_model.predict(np.c_[xx1.ravel(), xx2.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx1.shape)
plt.contourf(xx1, xx2, Z, cmap=plt.cm.coolwarm, alpha=0.8)

# Plot also the training points
#plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.coolwarm)
plt.scatter(X[:int(num_samples/2),0],X[:int(num_samples/2),1], marker='o', color='red')
plt.scatter(X[int(num_samples/2):,0],X[int(num_samples/2):,1], marker='s', color='green')

plt.xlabel('x1')
plt.ylabel('x2')
plt.xlim(xx1.min(), xx1.max())
plt.ylim(xx2.min(), xx2.max())
plt.xticks(())
plt.yticks(())
plt.title('Decision boundary from Linear SVM')
plt.show()

```

Let us now use the transformation $\Phi$ of the form to transform the features into a high-dimensional space hoping that the transformed data would be linearly separable.

We use the following transformation: For any $x=(x_1,x_2)$,

$
\begin{align}
\Phi(x) = \Phi( (x_1,x_2) ) = (x_1^2, x_2^2, \sqrt{2}x_1, \sqrt{2}x_2, 1).
\end{align}
$

Thus suppose in the high-dimensional space, if there is a linear separator of the form $(\mathbf{w},b) = ((w_1,w_2,w_3,w_4,w_5),b)$ such that $\mathbf{w}^\top \Phi (x) + b  = 0$ is the linear separator.

We shall check if indeed for the transformed space we would have a linear separator or not.

```python
#transform the data

X_transformed = []
for i in range(num_samples):
  x1=X[i,0]
  x2=X[i,1]
  X_transformed.append([x1**2,x2**2,np.sqrt(2)*x1,np.sqrt(2)*x2,1.])

X_transformed = np.array(X_transformed)
#print(X_transformed)

clf = LinearSVC(C=10., max_iter = 10000)
clf_model = clf.fit(X_transformed,y.ravel())

predicted_labels = clf_model.predict(X_transformed)
print(predicted_labels)

train_error = np.sum(0.5*np.abs(predicted_labels-y.ravel()))/len(y)*100.0
train_accuracy = 100.0-train_error

print('train accuracy:', train_accuracy)

```

    [ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
      1  1  1  1  1 -1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
      1  1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
     -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
     -1 -1 -1 -1]
    train accuracy: 99.0

Hence in the transformed space, we would be able to solve the hard-SVM problem:

$\begin{align}
\min_{\mathbf{w},b} & \frac{1}{2} \|\mathbf{w}\|_2^2 \nonumber \\
\text{such that } & y^i (\mathbf{w}^\top \Phi(x^i) + b) \geq 1 \ \forall i \in \{1,2,\ldots,n\}.
\end{align}
$

Then the variable $\mathbf{w}$ can be written as: $\mathbf{w} = \sum_{i=1}^{n} \lambda^i y^i \Phi(x^i)$, where $\lambda^i$ are called the Lagrange multipliers associated with the constraints in the optimization problem.

Hence during inference, for a test sample $\hat{x}$ we can use:
$
\begin{align}
\text{sign}(\mathbf{w}^\top \Phi(\hat{x}) + b) =  \text{sign} (\sum_{i=1}^{n} \lambda^i y^i \Phi(x^i)^\top \Phi(\hat{x}) + b).
\end{align}
$

Note that during inference of a test sample $\hat{x}$, we compute the dot product $\Phi(x^i)^\top \Phi(\hat{x})$ between each transformed training sample $\Phi(x^i)$ and transformed test sample $\Phi(\hat{x})$.

$\large{\text{Question:}}$ How do we find the transformation $\Phi$?

Suppose if we compute the dot product between two transformed points $\mathbf{u},\mathbf{z}$ using our previous choice for $\Phi$, we get:

$
\begin{align}
\Phi(\mathbf{u})^\top \Phi(\mathbf{z}) &= (u_1^2, u_2^2, \sqrt{2} u_1, \sqrt{2} u_2, 1)^\top  (z_1^2, z_2^2, \sqrt{2} z_1, \sqrt{2} z_2, 1) \\
&= u_1^2 z_1^2 + u_2^2 z_2^2 + 2 u_1 z_1 + 2 u_2 z_2 + 1 \\
&= (\mathbf{u}^\top \mathbf{z}+1)^2.
\end{align}
$

Hence the dot product between the transformed vectors in the transformed space can be represented using a transformation of the dot product between the original points in the ambient (or the original) space.

That is, we can represent $\Phi(\mathbf{u})^\top \Phi(\mathbf{z})$ as a function $K(\mathbf{u}, \mathbf{z}) = (\mathbf{u}^\top \mathbf{z}+1)^2$.

This function $K:\mathcal{X} \times \mathcal{X} \rightarrow \mathbb{R}$ is called a $\textbf{kernel}$ function.

Recall that $\mathcal{X}$ is the space of inputs.

$\large{\text{Kernel trick}}$

Fortunately, finding a suitable choice of $\Phi$ transformation can be by-passed if we can find a suitable kernel function $K$ such that the dot product $\Phi(\mathbf{u})^\top \Phi(\mathbf{z})$ can be represented as $K(\mathbf{u},\mathbf{z})$.

Indeed, this is possible using a result known as $\textbf{Mercer's theorem}$.

$\large{\text{Mercer's theorem}}$

$K$ satisfies:

$
\begin{align}
\int K(\mathbf{u}, \mathbf{z}) g(\mathbf{u}) g(\mathbf{z}) d\mathbf{u} d\mathbf{z} \geq 0
\end{align}
$

for any function $g$ satisfying $\int g^2(\mathbf{u}) d\mathbf{u} < \infty$, if and only if  $K$ corresponds to a unique transformation $\Phi$ such that $K( \mathbf{u}, \mathbf{z}) = \Phi(\mathbf{u})^\top \Phi(\mathbf{z})$.

Several possible choices of $K$ exist. Some examples are:

*   Polynomial kernel: $K(\mathbf{u}, \mathbf{z}) = (\mathbf{u}^\top \mathbf{z} + 1)^p$
*   Gaussian kernel (or) radial basis function (rbf) kernel: $K(\mathbf{u}, \mathbf{z}) = e^{-\frac{\|\mathbf{u}-\mathbf{z}\|^2}{2\sigma^2}}$

Let us try to use the RBF kernel on our original data and check.

```python
from sklearn.svm import SVC
clf = SVC(kernel='rbf',gamma=50,max_iter = 10000)
clf_model = clf.fit(X,y.ravel())

predicted_labels = clf_model.predict(X)
print(predicted_labels)

train_error = np.sum(0.5*np.abs(predicted_labels-y.ravel()))/len(y)*100.0
train_accuracy = 100.0-train_error

print('train accuracy:', train_accuracy)
```

    [ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
      1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
      1  1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
     -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
     -1 -1 -1 -1]
    train accuracy: 100.0

```python
# create a mesh to plot in
h=0.02 #mesh step size
x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, h),
                     np.arange(x2_min, x2_max, h))

Z = clf_model.predict(np.c_[xx1.ravel(), xx2.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx1.shape)
plt.contourf(xx1, xx2, Z, cmap=plt.cm.coolwarm, alpha=0.8)

# Plot also the training points
#plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.coolwarm)
plt.scatter(X[:int(num_samples/2),0],X[:int(num_samples/2),1], marker='o', color='red')
plt.scatter(X[int(num_samples/2):,0],X[int(num_samples/2):,1], marker='s', color='green')

plt.xlabel('x1')
plt.ylabel('x2')
plt.xlim(xx1.min(), xx1.max())
plt.ylim(xx2.min(), xx2.max())
plt.xticks(())
plt.yticks(())
plt.title('Decision boundary from SVM with RBF Kernel')
plt.show()

```

$\large{\text{Exercise}}$

1. Create a test data set $T$ for the synthetic data used in the discussion of kernel machines.
1. On the synthetic data generated, try RBF Kernel with $\gamma \in \{0.0001, 0.001, 0.1, 1, 25, 75, 100\}$. Plot the decision boundaries for training data obtained in each case. Explain your observations. Check the performance of kernel machine for each value of $\gamma$ on the test data set $T$ and plot the decision boundaries in the context of test data set. Indicate which $\gamma$ values give similar performance on both the train data and test data.
2. On the synthetic data generated, try polynomial kernel $K(\textbf{u},\textbf{v}) = (\textbf{u}^\top \textbf{v} + 1)^p$ with $p \in \{1,2,4,8,16,32,64,128\}$. Plot the decision boundaries for training data obtained in each case. Explain your observations. Check the performance of kernel machine for each value of $p$ on the test data set $T$ and plot the decision boundaries in the context of test data set. Indicate which $p$ values give similar performance on both the train data and test data.

```python

```


# Programming Assignment

## Naive Bayes and logistic regression

### Instructions

In this notebook, you will write code to develop a Naive Bayes classifier model to the Iris dataset using Distribution objects from TensorFlow Probability. You will also explore the connection between the Naive Bayes classifier and logistic regression.

Some code cells are provided you in the notebook. You should avoid editing provided code, and make sure to execute the cells in order to avoid unexpected errors. Some cells begin with the line:

`#### GRADED CELL ####`

Don't move or edit this first line - this is what the automatic grader looks for to recognise graded cells. These cells require you to write your own code to complete them, and are automatically graded when you submit the notebook. Don't edit the function name or signature provided in these cells, otherwise the automatic grader might not function properly.

### How to submit

Complete all the tasks you are asked for in the worksheet. When you have finished and are happy with your code, press the **Submit Assignment** button at the top of this notebook.

### Let's get started!

We'll start running some imports, and loading the dataset. Do not edit the existing imports in the following cell. If you would like to make further Tensorflow imports, you should add them here.

```python
#### PACKAGE IMPORTS ####

# Run this cell first to import all required packages. Do not make any imports elsewhere in the notebook
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
from sklearn import datasets, model_selection
%matplotlib inline

# If you would like to make further imports from TensorFlow or TensorFlow Probability, add them here

```

#### The Iris dataset

In this assignment, you will use the [Iris dataset](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html). It consists of 50 samples from each of three species of Iris (Iris setosa, Iris virginica and Iris versicolor). Four features were measured from each sample: the length and the width of the sepals and petals, in centimeters. For a reference, see the following papers:

- R. A. Fisher. "The use of multiple measurements in taxonomic problems". Annals of Eugenics. 7 (2): 179–188, 1936.

Your goal is to construct a Naive Bayes classifier model that predicts the correct class from the sepal length and sepal width features. Under certain assumptions about this classifier model, you will explore the relation to logistic regression.

#### Load and prepare the data

We will first read in the Iris dataset, and split the dataset into training and test sets.

```python
# Load the dataset

iris = datasets.load_iris()
```

```python
# Use only the first two features: sepal length and width

data = iris.data[:, :2]
targets = iris.target
```

```python
# Randomly shuffle the data and make train and test splits

x_train, x_test, y_train, y_test = model_selection.train_test_split(data, targets, test_size=0.2)
```

```python
# Plot the training data

labels = {0: 'Iris-Setosa', 1: 'Iris-Versicolour', 2: 'Iris-Virginica'}
label_colours = ['blue', 'orange', 'green']

def plot_data(x, y, labels, colours):
    for c in np.unique(y):
        inx = np.where(y == c)
        plt.scatter(x[inx, 0], x[inx, 1], label=labels[c], c=colours[c])
    plt.title("Training set")
    plt.xlabel("Sepal length (cm)")
    plt.ylabel("Sepal width (cm)")
    plt.legend()

plt.figure(figsize=(8, 5))
plot_data(x_train, y_train, labels, label_colours)
plt.show()
```

### Naive Bayes classifier

We will briefly review the Naive Bayes classifier model. The fundamental equation for this classifier is Bayes' rule:

$$
P(Y=y_k | X_1,\ldots,X_d) = \frac{P(X_1,\ldots,X_d | Y=y_k)P(Y=y_k)}{\sum_{k=1}^K P(X_1,\ldots,X_d | Y=y_k)P(Y=y_k)}
$$

In the above, $d$ is the number of features or dimensions in the inputs $X$ (in our case $d=2$), and $K$ is the number of classes (in our case $K=3$). The distribution $P(Y)$ is the class prior distribution, which is a discrete distribution over $K$ classes. The distribution $P(X | Y)$ is the class-conditional distribution over inputs.

The Naive Bayes classifier makes the assumption that the data features $X_i$ are conditionally independent give the class $Y$ (the 'naive' assumption). In this case, the class-conditional distribution decomposes as

$$
\begin{align}
P(X | Y=y_k) &= P(X_1,\ldots,X_d | Y=y_k)\\
&= \prod_{i=1}^d P(X_i | Y=y_k)
\end{align}
$$

This simplifying assumption means that we typically need to estimate far fewer parameters for each of the distributions $P(X_i | Y=y_k)$ instead of the full joint distribution $P(X | Y=y_k)$.

Once the class prior distribution and class-conditional densities are estimated, the Naive Bayes classifier model can then make a class prediction $\hat{Y}$ for a new data input $\tilde{X} := (\tilde{X}_1,\ldots,\tilde{X}_d)$ according to

$$
\begin{align}
\hat{Y} &= \text{argmax}_{y_k} P(Y=y_k | \tilde{X}_1,\ldots,\tilde{X}_d) \\
&= \text{argmax}_{y_k}\frac{P(\tilde{X}_1,\ldots,\tilde{X}_d | Y=y_k)P(Y=y_k)}{\sum_{k=1}^K P(\tilde{X}_1,\ldots,\tilde{X}_d | Y=y_k)P(Y=y_k)}\\
&= \text{argmax}_{y_k} P(\tilde{X}_1,\ldots,\tilde{X}_d | Y=y_k)P(Y=y_k)
\end{align}
$$

#### Define the class prior distribution

We will begin by defining the class prior distribution. To do this we will simply take the maximum likelihood estimate, given by

$$
P(Y=y_k) = \frac{\sum_{n=1}^N \delta(Y^{(n)}=y_k)}{N},
$$

where the superscript $(n)$ indicates the $n$-th dataset example, $\delta(Y^{(n)}=y_k) = 1$ if $Y^{(n)}=y_k$ and 0 otherwise, and $N$ is the total number of examples in the dataset. The above is simply the proportion of data examples belonging to class $k$.

You should now write a function that builds the prior distribution from the training data, and returns it as a `Categorical` Distribution object.

* The input to your function `y` will be a numpy array of shape `(num_samples,)`
* The entries in `y` will be integer labels $k=0, 1,\ldots, K-1$
* Your function should build and return the prior distribution as a `Categorical` distribution object
  * The probabilities for this distribution will be a length-$K$ vector, with entries corresponding to $P(Y = y_k)$ for $k=0,1,\ldots,K-1$
  * Your function should work for any value of $K\ge 1$
  * This Distribution will have an empty batch shape and empty event shape

```python
#### GRADED CELL ####

# Complete the following function.
# Make sure to not change the function name or arguments.

def get_prior(y):
    """
    This function takes training labels as a numpy array y of shape (num_samples,) as an input.
    Your function should
    This function should build a Categorical Distribution object with empty batch shape
    and event shape, with the probability of each class given as above.
    Your function should return the Distribution object.
    """

```

```python
# Run your function to get the prior

prior = get_prior(y_train)
# prior.prob([1]) + prior.prob([0]) + prior.prob([2])
```

```python
# Plot the prior distribution

labels = ['Iris-Setosa', 'Iris-Versicolour', 'Iris-Virginica']
plt.bar([0, 1, 2], prior.probs.numpy(), color=label_colours)
plt.xlabel("Class")
plt.ylabel("Prior probability")
plt.title("Class prior distribution")
plt.xticks([0, 1, 2], labels)
plt.show()
```

#### Define the class-conditional densities

We now turn to the definition of the class-conditional distributions $P(X_i | Y=y_k)$ for $i=0, 1$ and $k=0, 1, 2$. In our model, we will assume these distributions to be univariate Gaussian:

$$
\begin{align}
P(X_i | Y=y_k) &= N(X_i | \mu_{ik}, \sigma_{ik})\\
&= \frac{1}{\sqrt{2\pi\sigma_{ik}^2}} \exp\left\{-\frac{1}{2} \left(\frac{x - \mu_{ik}}{\sigma_{ik}}\right)^2\right\}
\end{align}
$$
with mean parameters $\mu_{ik}$ and standard deviation parameters $\sigma_{ik}$, twelve parameters in all. We will again estimate these parameters using maximum likelihood. In this case, the estimates are given by

$$
\begin{align}
\hat{\mu}_{ik} &= \frac{\sum_n X_i^{(n)} \delta(Y^{(n)}=y_k)}{\sum_n \delta(Y^{(n)}=y_k)} \\
\hat{\sigma}^2_{ik} &= \frac{\sum_n (X_i^{(n)} - \hat{\mu}_{ik})^2 \delta(Y^{(n)}=y_k)}{\sum_n \delta(Y^{(n)}=y_k)}
\end{align}
$$

Note that the above are just the means and variances of the sample data points for each class.

You should now write a function the computes the class-conditional Gaussian densities, using the maximum likelihood parameter estimates given above, and returns them in a single, batched `MultivariateNormalDiag` Distribution object.

* The inputs to the function are
  * a numpy array `x` of shape `(num_samples, num_features)` for the data inputs
  * a numpy array `y` of shape `(num_samples,)` for the target labels
* Your function should work for any number of classes $K\ge 1$ and any number of features $d\ge 1$

```python
#### GRADED CELL ####

# Complete the following function.
# Make sure to not change the function name or arguments.

def get_class_conditionals(x, y):
    """
    This function takes training data samples x and labels y as inputs.
    This function should build the class-conditional Gaussian distributions above.
    It should construct a batch of distributions for each feature and each class, using the
    parameter estimates above for the means and standard deviations.
    The batch shape of this distribution should be rank 2, where the first dimension corresponds
    to the number of classes and the second corresponds to the number of features.
    Your function should then return the Distribution object.
    """

```

```python
# Run your function to get the class-conditional distributions

class_conditionals = get_class_conditionals(x_train, y_train)
class_conditionals
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3] event_shape=[2] dtype=float32>

```python
# Random cell to verify some outputs

n_classes = np.unique(y_train)
n_features = x_train.shape[1]
print(n_classes)
means = np.zeros((n_classes.shape[0],n_features))
print(means)
variances = np.zeros((n_classes.shape[0],n_features))
for i, n_class in enumerate(n_classes):
    means[i] = x_train[y_train == n_class].mean(axis = 0, keepdims = True)
    variances[i] = x_train[y_train == n_class].var(axis = 0, keepdims = True)

print(np.sqrt(variances))
dist = tfd.MultivariateNormalDiag(loc = means.T,
                          scale_diag = variances.T)
dist
```

    [0 1 2]
    [[0. 0.]
     [0. 0.]
     [0. 0.]]
    [[0.35776423 0.3668604 ]
     [0.51815126 0.29051522]
     [0.56943066 0.32364794]]

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[2] event_shape=[3] dtype=float64>

```python
## Random cell to verify some outputs

locs = np.array([
    [[2],[3]],
    [[3],[4]],
    [[5],[6]]
], dtype = 'float32')

scale_diags = np.array([
    [[1],[2]],
    [[3],[4]],
    [[5],[6]]
], dtype = 'float32')

MVND = tfd.MultivariateNormalDiag(loc = locs,
                          scale_diag = scale_diags)

MVND
```

    <tfp.distributions.MultivariateNormalDiag 'MultivariateNormalDiag' batch_shape=[3, 2] event_shape=[1] dtype=float32>

We can visualise the class-conditional densities with contour plots by running the cell below. Notice how the contours of each distribution correspond to a Gaussian distribution with diagonal covariance matrix, since the model assumes that each feature is independent given the class.

```python
# Plot the training data with the class-conditional density contours

def get_meshgrid(x0_range, x1_range, num_points=100):
    x0 = np.linspace(x0_range[0], x0_range[1], num_points)
    x1 = np.linspace(x1_range[0], x1_range[1], num_points)
    return np.meshgrid(x0, x1)

def contour_plot(x0_range, x1_range, prob_fn, batch_shape, colours, levels=None, num_points=100):
    X0, X1 = get_meshgrid(x0_range, x1_range, num_points=num_points)
    Z = prob_fn(np.expand_dims(np.array([X0.ravel(), X1.ravel()]).T, 1))
    Z = np.array(Z).T.reshape(batch_shape, *X0.shape)
    for batch in np.arange(batch_shape):
        if levels:
            plt.contourf(X0, X1, Z[batch], alpha=0.2, colors=colours, levels=levels)
        else:
            plt.contour(X0, X1, Z[batch], colors=colours[batch], alpha=0.3)

plt.figure(figsize=(10, 6))
plot_data(x_train, y_train, labels, label_colours)
x0_min, x0_max = x_train[:, 0].min(), x_train[:, 0].max()
x1_min, x1_max = x_train[:, 1].min(), x_train[:, 1].max()
contour_plot((x0_min, x0_max), (x1_min, x1_max), class_conditionals.prob, 3, label_colours)
plt.title("Training set with class-conditional density contours")
plt.show()
```

#### Make predictions from the model

Now the prior and class-conditional distributions are defined, you can use them to compute the model's class probability predictions for an unknown test input $\tilde{X} = (\tilde{X}_1,\ldots,\tilde{X}_d)$, according to

$$
P(Y=y_k | \tilde{X}_1,\ldots,\tilde{X}_d) = \frac{P(\tilde{X}_1,\ldots,\tilde{X}_d | Y=y_k)P(Y=y_k)}{\sum_{k=1}^K P(\tilde{X}_1,\ldots,\tilde{X}_d | Y=y_k)P(Y=y_k)}
$$

The class prediction can then be taken as the class with the maximum probability:

$$
\hat{Y} = \text{argmax}_{y_k} P(Y=y_k | \tilde{X}_1,\ldots,\tilde{X}_d)
$$

You should now write a function to return the model's class probabilities for a given batch of test inputs of shape `(batch_shape, 2)`, where the `batch_shape` has rank at least one.

* The inputs to the function are the `prior` and `class_conditionals` distributions, and the inputs `x`
* Your function should use these distributions to compute the probabilities for each class $k$ as above
  * As before, your function should work for any number of classes $K\ge 1$
* It should then compute the prediction by taking the class with the highest probability
* The predictions should be returned in a numpy array of shape `(batch_shape)`

```python
#### GRADED CELL ####

# Complete the following function.
# Make sure to not change the function name or arguments.

def predict_class(prior, class_conditionals, x):
    """
    This function takes the prior distribution, class-conditional distribution, and
    a batch of inputs in a numpy array of shape (batch_shape, 2).
    This function should compute the class probabilities for each input in the batch, using
    the prior and class-conditional distributions, according to the above equation.
    Note that the batch_shape of x could have rank higher than one!
    Your function should then return the class predictions by taking the class with the
    maximum probability in a numpy array of shape (batch_shape,).
    """

```

```python
## Coding cell to test the codes

prior = get_prior(y_train)
prior_logs = np.log(prior.probs)
cond_probs = class_conditionals.log_prob(x_test[:,None,:])
joint_likelihood = tf.add(prior_logs, cond_probs)
norm_factor = tf.math.reduce_logsumexp(joint_likelihood, axis = -1, keepdims = True)
log_prob = joint_likelihood - norm_factor
tf.argmax(log_prob, axis = -1).numpy()
```

    array([1, 2, 0, 1, 2, 2, 0, 2, 2, 0, 0, 1, 1, 1, 0, 1, 0, 1, 2, 1, 0, 1,
           0, 1, 2, 2, 2, 2, 2, 0])

```python
# Get the class predictions

predictions = predict_class(prior, class_conditionals, x_test)
predictions
```

    array([1, 2, 0, 1, 2, 2, 0, 2, 2, 0, 0, 1, 1, 1, 0, 1, 0, 1, 2, 1, 0, 1,
           0, 1, 2, 2, 2, 2, 2, 0])

```python
# Evaluate the model accuracy on the test set

accuracy = accuracy_score(y_test, predictions)
print("Test accuracy: {:.4f}".format(accuracy))
```

    Test accuracy: 0.6667

```python
# Plot the model's decision regions

plt.figure(figsize=(10, 6))
plot_data(x_train, y_train, labels, label_colours)
x0_min, x0_max = x_train[:, 0].min(), x_train[:, 0].max()
x1_min, x1_max = x_train[:, 1].min(), x_train[:, 1].max()
contour_plot((x0_min, x0_max), (x1_min, x1_max),
             lambda x: predict_class(prior, class_conditionals, x),
             1, label_colours, levels=[-0.5, 0.5, 1.5, 2.5],
             num_points=500)
plt.title("Training set with decision regions")
plt.show()
```

### Binary classifier

We will now draw a connection between the Naive Bayes classifier and logistic regression.

First, we will update our model to be a binary classifier. In particular, the model will output the probability that a given input data sample belongs to the 'Iris-Setosa' class: $P(Y=y_0 | \tilde{X}_1,\ldots,\tilde{X}_d)$. The remaining two classes will be pooled together with the label $y_1$.

```python
# Redefine the dataset to have binary labels

y_train_binary = np.array(y_train)
y_train_binary[np.where(y_train_binary == 2)] = 1

y_test_binary = np.array(y_test)
y_test_binary[np.where(y_test_binary == 2)] = 1
```

```python
# Plot the training data

labels_binary = {0: 'Iris-Setosa', 1: 'Iris-Versicolour / Iris-Virginica'}
label_colours_binary = ['blue', 'red']

plt.figure(figsize=(8, 5))
plot_data(x_train, y_train_binary, labels_binary, label_colours_binary)
plt.show()
```

We will also make an extra modelling assumption that for each class $k$, the class-conditional distribution $P(X_i | Y=y_k)$ for each feature $i=0, 1$, has standard deviation $\sigma_i$, which is the same for each class $k$.

This means there are now six parameters in total: four for the means $\mu_{ik}$ and two for the standard deviations $\sigma_i$ ($i, k=0, 1$).

We will again use maximum likelihood to estimate these parameters. The prior distribution will be as before, with the class prior probabilities given by

$$
P(Y=y_k) = \frac{\sum_{n=1}^N \delta(Y^{(n)}=y_k)}{N},
$$

We will use your previous function `get_prior` to redefine the prior distribution.

```python
# Redefine the prior

prior_binary = get_prior(y_train_binary)
```

```python
# Plot the prior distribution

plt.bar([0, 1], prior_binary.probs.numpy(), color=label_colours_binary)
plt.xlabel("Class")
plt.ylabel("Prior probability")
plt.title("Class prior distribution")
plt.xticks([0, 1], labels_binary)
plt.show()
```

For the class-conditional densities, the maximum likelihood estimate for the means are again given by

$$
\hat{\mu}_{ik} = \frac{\sum_n X_i^{(n)} \delta(Y^{(n)}=y_k)}{\sum_n \delta(Y^{(n)}=y_k)} \\
$$

However, the estimate for the standard deviations $\sigma_i$ is updated. There is also a closed-form solution for the shared standard deviations, but we will instead learn these from the data.

You should now write a function that takes the training inputs and target labels as input, as well as an optimizer object, number of epochs and a TensorFlow Variable. This function should be written according to the following spec:

* The inputs to the function are:
  * a numpy array `x` of shape `(num_samples, num_features)` for the data inputs
  * a numpy array `y` of shape `(num_samples,)` for the target labels
  * a `tf.Variable` object `scales` of length 2 for the standard deviations $\sigma_i$
  * `optimiser`: an optimiser object
  * `epochs`: the number of epochs to run the training for
* The function should first compute the means $\mu_{ik}$ of the class-conditional Gaussians according to the above equation
* Then create a batched multivariate Gaussian distribution object using `MultivariateNormalDiag` with the means set to $\mu_{ik}$ and the scales set to `scales`
* Run a custom training loop for `epochs` number of epochs, in which:
  * the average per-example negative log likelihood for the whole dataset is computed as the loss
  * the gradient of the loss with respect to the `scales` variables is computed
  * the `scales` variables are updated by the `optimiser` object
* At each iteration, save the values of the `scales` variable and the loss
* The function should return a tuple of three objects:
   * a numpy array of shape `(epochs,)` of loss values
   * a numpy array of shape `(epochs, 2)` of values for the `scales` variable at each iteration
   * the final learned batched `MultivariateNormalDiag` distribution object

_NB: ideally, we would like to constrain the `scales` variable to have positive values. We are not doing that here, but in later weeks of the course you will learn how this can be implemented._

```python
#### GRADED CELL ####

# Complete the following function.
# Make sure to not change the function name or arguments.

def learn_stdevs(x, y, scales, optimiser, epochs):
    """
    This function takes the data inputs, targets, scales variable, optimiser and number of
    epochs as inputs.
    This function should set up and run a custom training loop according to the above
    specifications, by setting up the class conditional distributions as a MultivariateNormalDiag
    object, and updating the trainable variables (the scales) in a custom training loop.
    Your function should then return the a tuple of three elements: a numpy array of loss values
    during training, a numpy array of scales variables during training, and the final learned
    MultivariateNormalDiag distribution object.
    """
    train_loss_results, train_scales_results = [], []

    n_classes = np.unique(y)
    n_features = x.shape[1]

    means = np.zeros((n_classes.shape[0],n_features))
    for i, n_class in enumerate(n_classes):
        means[i] = x[y == n_class].mean(axis = 0, keepdims = True)
    means = tf.constant(means, dtype = tf.float32)
    MVND = tfd.MultivariateNormalDiag(loc = means, scale_diag = scales)
    x = tf.constant(x, dtype = tf.float32)

    def nll(x, y, distribution):
            predictions = - distribution.log_prob(x[:,None,:])
            p1 = tf.reduce_sum(predictions[y==0][:,0])
            p2 = tf.reduce_sum(predictions[y==1][:,1])
            return p1 + p2

    @tf.function
    def get_loss_and_grads(x,y, distribution):

        with tf.GradientTape() as tape:
            tape.watch(distribution.trainable_variables) # Watch whichever whose gradient is taken with respect to

            loss = nll(x,y, distribution) # Get the loss
            grads = tape.gradient(loss, distribution.trainable_variables) # Calculate the gradient

        return loss, grads

    for i in range(epochs):
        loss, grads = get_loss_and_grads(x,y, MVND)
        optimiser.apply_gradients(zip(grads, MVND.trainable_variables))

        scales_value = scales.value().numpy()
        train_loss_results.append(loss)
        train_scales_results.append(scales_value)
        print(f"Step {i:03d}: Loss: {loss:.3f}: Scales: {scales_value}.")

    return np.array(train_loss_results), np.array(train_scales_results), MVND
```

```python
## Coding cell to see if code works

def nll(x, y, distribution):
        predictions = - distribution.log_prob(x[:,None,:])
        p1 = tf.reduce_sum(predictions[y==0][:,0])
        p2 = tf.reduce_sum(predictions[y==1][:,1])
        return p1 + p2

@tf.function
def get_loss_and_grads(x,y, distribution):

    with tf.GradientTape() as tape:
        tape.watch(distribution.trainable_variables) # Watch whichever whose gradient is taken with respect to

        loss = nll(x,y, distribution) # Get the loss
        grads = tape.gradient(loss, distribution.trainable_variables) # Calculate the gradient

    return loss, grads

n_classes = np.unique(y_train_binary)
n_features = x_train.shape[1]
means = np.zeros((n_classes.shape[0],n_features))
for i, n_class in enumerate(n_classes):
    means[i] = x_train[y_train_binary == n_class].mean(axis = 0, keepdims = True)
means = tf.constant(means, dtype = tf.float32)
scales = tf.Variable([1., 1.])
distribution = tfd.MultivariateNormalDiag(loc = means, scale_diag = scales)
x = tf.constant(x_train, dtype = tf.float32)

for i in range(500):
    loss, grads = get_loss_and_grads(x,y_train_binary, distribution)
    opt.apply_gradients(zip(grads, distribution.trainable_variables))

    scales_value = scales.value().numpy()
#     train_loss_results.append(loss)
#     train_scales_results.append(scales_value)
    print(f"Step {i:03d}: Loss: {loss:.3f}: Scales: {scales_value}.")
```

```python
# Define the inputs to your function

scales = tf.Variable([1., 1.])
opt = tf.keras.optimizers.Adam(learning_rate=0.01)
epochs = 500
```

```python
# Run your function to learn the class-conditional standard deviations

nlls, scales_arr, class_conditionals_binary = learn_stdevs(x_train, y_train_binary, scales, opt, epochs)
```

    Step 000: Loss: 246.750: Scales: [0.99 0.99].
    Step 001: Loss: 244.870: Scales: [0.9799999 0.9799982].
    Step 002: Loss: 242.982: Scales: [0.9699997  0.96999323].
    Step 003: Loss: 241.085: Scales: [0.9599995 0.959984 ].
    Step 004: Loss: 239.180: Scales: [0.9499995 0.9499693].
    Step 005: Loss: 237.268: Scales: [0.94      0.9399478].
    Step 006: Loss: 235.347: Scales: [0.93000144 0.92991835].
    Step 007: Loss: 233.418: Scales: [0.9200043 0.9198798].
    Step 008: Loss: 231.481: Scales: [0.9100094 0.909831 ].
    Step 009: Loss: 229.537: Scales: [0.9000174 0.8997708].
    Step 010: Loss: 227.586: Scales: [0.8900294 0.889698 ].
    Step 011: Loss: 225.628: Scales: [0.8800465 0.8796117].
    Step 012: Loss: 223.663: Scales: [0.87007016 0.86951065].
    Step 013: Loss: 221.693: Scales: [0.8601019  0.85939395].
    Step 014: Loss: 219.716: Scales: [0.8501435 0.8492606].
    Step 015: Loss: 217.735: Scales: [0.840197  0.8391097].
    Step 016: Loss: 215.749: Scales: [0.83026475 0.82894033].
    Step 017: Loss: 213.759: Scales: [0.82034934 0.8187516 ].
    Step 018: Loss: 211.765: Scales: [0.8104538 0.8085427].
    Step 019: Loss: 209.770: Scales: [0.8005813 0.798313 ].
    Step 020: Loss: 207.773: Scales: [0.79073554 0.7880618 ].
    Step 021: Loss: 205.776: Scales: [0.7809207  0.77778846].
    Step 022: Loss: 203.779: Scales: [0.77114123 0.7674925 ].
    Step 023: Loss: 201.784: Scales: [0.76140225 0.75717336].
    Step 024: Loss: 199.793: Scales: [0.75170934 0.7468307 ].
    Step 025: Loss: 197.805: Scales: [0.74206877 0.73646426].
    Step 026: Loss: 195.824: Scales: [0.7324874 0.7260738].
    Step 027: Loss: 193.850: Scales: [0.72297275 0.71565926].
    Step 028: Loss: 191.885: Scales: [0.71353316 0.7052206 ].
    Step 029: Loss: 189.931: Scales: [0.7041779 0.694758 ].
    Step 030: Loss: 187.990: Scales: [0.6949171 0.6842717].
    Step 031: Loss: 186.064: Scales: [0.68576175 0.67376214].
    Step 032: Loss: 184.155: Scales: [0.676724   0.66322994].
    Step 033: Loss: 182.264: Scales: [0.6678172 0.6526758].
    Step 034: Loss: 180.395: Scales: [0.6590556  0.64210075].
    Step 035: Loss: 178.549: Scales: [0.650455   0.63150597].
    Step 036: Loss: 176.729: Scales: [0.6420323 0.6208929].
    Step 037: Loss: 174.936: Scales: [0.63380575 0.6102632 ].
    Step 038: Loss: 173.173: Scales: [0.62579477 0.5996191 ].
    Step 039: Loss: 171.441: Scales: [0.6180201 0.5889628].
    Step 040: Loss: 169.743: Scales: [0.6105036 0.5782972].
    Step 041: Loss: 168.081: Scales: [0.603268  0.5676255].
    Step 042: Loss: 166.455: Scales: [0.5963369 0.5569515].
    Step 043: Loss: 164.867: Scales: [0.5897342 0.5462796].
    Step 044: Loss: 163.319: Scales: [0.5834838  0.53561485].
    Step 045: Loss: 161.810: Scales: [0.5776093 0.524963 ].
    Step 046: Loss: 160.342: Scales: [0.5721331 0.5143308].
    Step 047: Loss: 158.916: Scales: [0.56707615 0.50372595].
    Step 048: Loss: 157.532: Scales: [0.5624568  0.49315727].
    Step 049: Loss: 156.191: Scales: [0.55829054 0.482635  ].
    Step 050: Loss: 154.895: Scales: [0.5545891  0.47217083].
    Step 051: Loss: 153.645: Scales: [0.5513598  0.46177828].
    Step 052: Loss: 152.443: Scales: [0.5486052  0.45147288].
    Step 053: Loss: 151.294: Scales: [0.5463228  0.44127247].
    Step 054: Loss: 150.200: Scales: [0.54450464 0.43119755].
    Step 055: Loss: 149.168: Scales: [0.5431374  0.42127168].
    Step 056: Loss: 148.203: Scales: [0.5422027 0.4115218].
    Step 057: Loss: 147.314: Scales: [0.54167753 0.40197864].
    Step 058: Loss: 146.507: Scales: [0.5415346  0.39267722].
    Step 059: Loss: 145.789: Scales: [0.5417431  0.38365704].
    Step 060: Loss: 145.170: Scales: [0.5422694 0.3749624].
    Step 061: Loss: 144.654: Scales: [0.5430778  0.36664233].
    Step 062: Loss: 144.245: Scales: [0.54413146 0.35875043].
    Step 063: Loss: 143.946: Scales: [0.5453927  0.35134405].
    Step 064: Loss: 143.753: Scales: [0.54682416 0.3444829 ].
    Step 065: Loss: 143.659: Scales: [0.5483892  0.33822718].
    Step 066: Loss: 143.654: Scales: [0.55005246 0.33263466].
    Step 067: Loss: 143.719: Scales: [0.55178034 0.3277574 ].
    Step 068: Loss: 143.836: Scales: [0.5535415  0.32363778].
    Step 069: Loss: 143.980: Scales: [0.5553068  0.32030493].
    Step 070: Loss: 144.130: Scales: [0.55705    0.31777135].
    Step 071: Loss: 144.264: Scales: [0.5587476 0.3160309].
    Step 072: Loss: 144.365: Scales: [0.560379  0.3150581].
    Step 073: Loss: 144.425: Scales: [0.56192666 0.31480917].
    Step 074: Loss: 144.437: Scales: [0.56337565 0.31522402].
    Step 075: Loss: 144.406: Scales: [0.56471413 0.3162297 ].
    Step 076: Loss: 144.337: Scales: [0.5659329  0.31774402].
    Step 077: Loss: 144.241: Scales: [0.56702536 0.31967944].
    Step 078: Loss: 144.128: Scales: [0.56798726 0.3219466 ].
    Step 079: Loss: 144.011: Scales: [0.5688167  0.32445747].
    Step 080: Loss: 143.897: Scales: [0.5695138  0.32712793].
    Step 081: Loss: 143.795: Scales: [0.57008046 0.32987973].
    Step 082: Loss: 143.709: Scales: [0.57052034 0.332642  ].
    Step 083: Loss: 143.641: Scales: [0.5708386  0.33535203].
    Step 084: Loss: 143.591: Scales: [0.5710414  0.33795598].
    Step 085: Loss: 143.559: Scales: [0.5711363 0.3404088].
    Step 086: Loss: 143.541: Scales: [0.5711315  0.34267417].
    Step 087: Loss: 143.536: Scales: [0.57103604 0.34472406].
    Step 088: Loss: 143.540: Scales: [0.5708594  0.34653822].
    Step 089: Loss: 143.551: Scales: [0.5706113  0.34810352].
    Step 090: Loss: 143.564: Scales: [0.57030183 0.34941334].
    Step 091: Loss: 143.579: Scales: [0.569941   0.35046673].
    Step 092: Loss: 143.592: Scales: [0.56953883 0.35126778].
    Step 093: Loss: 143.604: Scales: [0.569105   0.35182494].
    Step 094: Loss: 143.612: Scales: [0.56864905 0.35215026].
    Step 095: Loss: 143.616: Scales: [0.56817985 0.35225895].
    Step 096: Loss: 143.617: Scales: [0.5677058  0.35216865].
    Step 097: Loss: 143.615: Scales: [0.5672348 0.351899 ].
    Step 098: Loss: 143.609: Scales: [0.566774   0.35147113].
    Step 099: Loss: 143.601: Scales: [0.56632984 0.35090715].
    Step 100: Loss: 143.591: Scales: [0.5659079  0.35022986].
    Step 101: Loss: 143.580: Scales: [0.565513   0.34946218].
    Step 102: Loss: 143.569: Scales: [0.56514925 0.34862694].
    Step 103: Loss: 143.559: Scales: [0.56481975 0.34774643].
    Step 104: Loss: 143.549: Scales: [0.5645269 0.3468421].
    Step 105: Loss: 143.541: Scales: [0.5642723  0.34593424].
    Step 106: Loss: 143.534: Scales: [0.56405675 0.34504172].
    Step 107: Loss: 143.529: Scales: [0.5638804  0.34418175].
    Step 108: Loss: 143.526: Scales: [0.56374264 0.34336954].
    Step 109: Loss: 143.524: Scales: [0.56364244 0.34261832].
    Step 110: Loss: 143.524: Scales: [0.563578   0.34193894].
    Step 111: Loss: 143.524: Scales: [0.56354725 0.34134   ].
    Step 112: Loss: 143.526: Scales: [0.5635477  0.34082767].
    Step 113: Loss: 143.527: Scales: [0.56357634 0.3404058 ].
    Step 114: Loss: 143.529: Scales: [0.56363016 0.34007582].
    Step 115: Loss: 143.531: Scales: [0.5637059 0.3398371].
    Step 116: Loss: 143.532: Scales: [0.5638002  0.33968687].
    Step 117: Loss: 143.533: Scales: [0.56390965 0.3396206 ].
    Step 118: Loss: 143.533: Scales: [0.5640309  0.33963212].
    Step 119: Loss: 143.533: Scales: [0.5641606  0.33971402].
    Step 120: Loss: 143.532: Scales: [0.56429565 0.33985785].
    Step 121: Loss: 143.531: Scales: [0.5644331  0.34005442].
    Step 122: Loss: 143.530: Scales: [0.5645702 0.3402941].
    Step 123: Loss: 143.529: Scales: [0.5647045  0.34056705].
    Step 124: Loss: 143.527: Scales: [0.5648337 0.3408636].
    Step 125: Loss: 143.526: Scales: [0.56495595 0.3411743 ].
    Step 126: Loss: 143.525: Scales: [0.5650696  0.34149036].
    Step 127: Loss: 143.524: Scales: [0.5651734  0.34180355].
    Step 128: Loss: 143.524: Scales: [0.5652662  0.34210652].
    Step 129: Loss: 143.523: Scales: [0.5653474  0.34239286].
    Step 130: Loss: 143.523: Scales: [0.5654164  0.34265712].
    Step 131: Loss: 143.523: Scales: [0.56547314 0.34289494].
    Step 132: Loss: 143.523: Scales: [0.56551766 0.343103  ].
    Step 133: Loss: 143.523: Scales: [0.56555027 0.34327897].
    Step 134: Loss: 143.524: Scales: [0.56557137 0.34342158].
    Step 135: Loss: 143.524: Scales: [0.5655817 0.3435304].
    Step 136: Loss: 143.524: Scales: [0.56558204 0.3436059 ].
    Step 137: Loss: 143.524: Scales: [0.56557333 0.34364936].
    Step 138: Loss: 143.524: Scales: [0.5655565  0.34366268].
    Step 139: Loss: 143.524: Scales: [0.56553274 0.34364834].
    Step 140: Loss: 143.524: Scales: [0.5655031  0.34360927].
    Step 141: Loss: 143.524: Scales: [0.5654687  0.34354874].
    Step 142: Loss: 143.524: Scales: [0.56543076 0.34347025].
    Step 143: Loss: 143.524: Scales: [0.5653902  0.34337747].
    Step 144: Loss: 143.524: Scales: [0.5653482  0.34327406].
    Step 145: Loss: 143.523: Scales: [0.5653057  0.34316364].
    Step 146: Loss: 143.523: Scales: [0.56526357 0.3430497 ].
    Step 147: Loss: 143.523: Scales: [0.5652227 0.3429355].
    Step 148: Loss: 143.523: Scales: [0.5651837  0.34282398].
    Step 149: Loss: 143.523: Scales: [0.5651472  0.34271783].
    Step 150: Loss: 143.523: Scales: [0.5651138  0.34261927].
    Step 151: Loss: 143.523: Scales: [0.5650838  0.34253022].
    Step 152: Loss: 143.523: Scales: [0.5650575 0.3424521].
    Step 153: Loss: 143.523: Scales: [0.56503516 0.342386  ].
    Step 154: Loss: 143.523: Scales: [0.56501687 0.3423325 ].
    Step 155: Loss: 143.523: Scales: [0.56500256 0.3422919 ].
    Step 156: Loss: 143.523: Scales: [0.56499225 0.34226406].
    Step 157: Loss: 143.523: Scales: [0.5649857  0.34224853].
    Step 158: Loss: 143.523: Scales: [0.5649827  0.34224457].
    Step 159: Loss: 143.523: Scales: [0.56498307 0.3422512 ].
    Step 160: Loss: 143.523: Scales: [0.5649864  0.34226727].
    Step 161: Loss: 143.523: Scales: [0.56499237 0.34229144].
    Step 162: Loss: 143.523: Scales: [0.5650006 0.3423223].
    Step 163: Loss: 143.523: Scales: [0.56501067 0.34235832].
    Step 164: Loss: 143.523: Scales: [0.56502223 0.34239808].
    Step 165: Loss: 143.523: Scales: [0.56503487 0.3424401 ].
    Step 166: Loss: 143.523: Scales: [0.5650482 0.342483 ].
    Step 167: Loss: 143.523: Scales: [0.5650619  0.34252554].
    Step 168: Loss: 143.523: Scales: [0.5650757  0.34256652].
    Step 169: Loss: 143.523: Scales: [0.56508917 0.34260496].
    Step 170: Loss: 143.523: Scales: [0.5651021 0.34264  ].
    Step 171: Loss: 143.523: Scales: [0.56511426 0.342671  ].
    Step 172: Loss: 143.523: Scales: [0.5651254  0.34269747].
    Step 173: Loss: 143.523: Scales: [0.5651355  0.34271908].
    Step 174: Loss: 143.523: Scales: [0.5651443  0.34273568].
    Step 175: Loss: 143.523: Scales: [0.56515175 0.34274727].
    Step 176: Loss: 143.523: Scales: [0.5651579  0.34275398].
    Step 177: Loss: 143.523: Scales: [0.56516266 0.3427561 ].
    Step 178: Loss: 143.523: Scales: [0.56516606 0.34275398].
    Step 179: Loss: 143.523: Scales: [0.56516814 0.34274808].
    Step 180: Loss: 143.523: Scales: [0.56516904 0.34273896].
    Step 181: Loss: 143.523: Scales: [0.5651688  0.34272715].
    Step 182: Loss: 143.523: Scales: [0.56516755 0.34271327].
    Step 183: Loss: 143.523: Scales: [0.5651654  0.34269792].
    Step 184: Loss: 143.523: Scales: [0.5651625  0.34268168].
    Step 185: Loss: 143.523: Scales: [0.5651589 0.3426651].
    Step 186: Loss: 143.523: Scales: [0.56515485 0.34264874].
    Step 187: Loss: 143.523: Scales: [0.5651505  0.34263304].
    Step 188: Loss: 143.523: Scales: [0.5651459 0.3426184].
    Step 189: Loss: 143.523: Scales: [0.56514126 0.34260514].
    Step 190: Loss: 143.523: Scales: [0.5651367  0.34259355].
    Step 191: Loss: 143.523: Scales: [0.5651322 0.3425838].
    Step 192: Loss: 143.523: Scales: [0.56512797 0.34257603].
    Step 193: Loss: 143.523: Scales: [0.56512403 0.34257028].
    Step 194: Loss: 143.523: Scales: [0.56512046 0.34256652].
    Step 195: Loss: 143.523: Scales: [0.5651173  0.34256467].
    Step 196: Loss: 143.523: Scales: [0.5651146 0.3425646].
    Step 197: Loss: 143.523: Scales: [0.5651124 0.3425662].
    Step 198: Loss: 143.523: Scales: [0.5651107 0.3425692].
    Step 199: Loss: 143.523: Scales: [0.56510943 0.3425734 ].
    Step 200: Loss: 143.523: Scales: [0.56510866 0.34257856].
    Step 201: Loss: 143.523: Scales: [0.5651083 0.3425844].
    Step 202: Loss: 143.523: Scales: [0.56510836 0.34259072].
    Step 203: Loss: 143.523: Scales: [0.5651088  0.34259725].
    Step 204: Loss: 143.523: Scales: [0.56510955 0.34260377].
    Step 205: Loss: 143.523: Scales: [0.56511056 0.3426101 ].
    Step 206: Loss: 143.523: Scales: [0.5651118  0.34261602].
    Step 207: Loss: 143.523: Scales: [0.56511325 0.34262142].
    Step 208: Loss: 143.523: Scales: [0.5651148  0.34262615].
    Step 209: Loss: 143.523: Scales: [0.5651164  0.34263015].
    Step 210: Loss: 143.523: Scales: [0.565118   0.34263334].
    Step 211: Loss: 143.523: Scales: [0.5651196  0.34263572].
    Step 212: Loss: 143.523: Scales: [0.5651212  0.34263727].
    Step 213: Loss: 143.523: Scales: [0.5651226  0.34263805].
    Step 214: Loss: 143.523: Scales: [0.5651239  0.34263808].
    Step 215: Loss: 143.523: Scales: [0.5651251  0.34263745].
    Step 216: Loss: 143.523: Scales: [0.5651261  0.34263623].
    Step 217: Loss: 143.523: Scales: [0.565127   0.34263453].
    Step 218: Loss: 143.523: Scales: [0.56512773 0.34263244].
    Step 219: Loss: 143.523: Scales: [0.56512827 0.34263006].
    Step 220: Loss: 143.523: Scales: [0.5651286 0.3426275].
    Step 221: Loss: 143.523: Scales: [0.5651288  0.34262484].
    Step 222: Loss: 143.523: Scales: [0.56512886 0.34262222].
    Step 223: Loss: 143.523: Scales: [0.56512874 0.3426197 ].
    Step 224: Loss: 143.523: Scales: [0.5651285  0.34261733].
    Step 225: Loss: 143.523: Scales: [0.56512815 0.34261522].
    Step 226: Loss: 143.523: Scales: [0.56512773 0.34261337].
    Step 227: Loss: 143.523: Scales: [0.56512725 0.34261185].
    Step 228: Loss: 143.523: Scales: [0.5651267  0.34261066].
    Step 229: Loss: 143.523: Scales: [0.5651261 0.3426098].
    Step 230: Loss: 143.523: Scales: [0.5651255 0.3426093].
    Step 231: Loss: 143.523: Scales: [0.5651249  0.34260908].
    Step 232: Loss: 143.523: Scales: [0.5651244  0.34260917].
    Step 233: Loss: 143.523: Scales: [0.56512386 0.34260952].
    Step 234: Loss: 143.523: Scales: [0.5651234  0.34261012].
    Step 235: Loss: 143.523: Scales: [0.56512296 0.3426109 ].
    Step 236: Loss: 143.523: Scales: [0.5651226  0.34261182].
    Step 237: Loss: 143.523: Scales: [0.5651223  0.34261283].
    Step 238: Loss: 143.523: Scales: [0.56512207 0.3426139 ].
    Step 239: Loss: 143.523: Scales: [0.5651219 0.342615 ].
    Step 240: Loss: 143.523: Scales: [0.56512177 0.34261608].
    Step 241: Loss: 143.523: Scales: [0.5651217 0.3426171].
    Step 242: Loss: 143.523: Scales: [0.5651217  0.34261802].
    Step 243: Loss: 143.523: Scales: [0.56512177 0.34261882].
    Step 244: Loss: 143.523: Scales: [0.5651219 0.3426195].
    Step 245: Loss: 143.523: Scales: [0.565122   0.34262004].
    Step 246: Loss: 143.523: Scales: [0.5651222  0.34262043].
    Step 247: Loss: 143.523: Scales: [0.56512237 0.34262067].
    Step 248: Loss: 143.523: Scales: [0.56512254 0.3426208 ].
    Step 249: Loss: 143.523: Scales: [0.5651227 0.3426208].
    Step 250: Loss: 143.523: Scales: [0.5651229  0.34262067].
    Step 251: Loss: 143.523: Scales: [0.5651231  0.34262043].
    Step 252: Loss: 143.523: Scales: [0.56512326 0.3426201 ].
    Step 253: Loss: 143.523: Scales: [0.56512344 0.34261972].
    Step 254: Loss: 143.523: Scales: [0.5651236 0.3426193].
    Step 255: Loss: 143.523: Scales: [0.56512374 0.34261885].
    Step 256: Loss: 143.523: Scales: [0.56512386 0.3426184 ].
    Step 257: Loss: 143.523: Scales: [0.565124   0.34261796].
    Step 258: Loss: 143.523: Scales: [0.56512403 0.34261754].
    Step 259: Loss: 143.523: Scales: [0.5651241  0.34261715].
    Step 260: Loss: 143.523: Scales: [0.56512415 0.3426168 ].
    Step 261: Loss: 143.523: Scales: [0.56512415 0.3426165 ].
    Step 262: Loss: 143.523: Scales: [0.56512415 0.34261626].
    Step 263: Loss: 143.523: Scales: [0.56512415 0.34261608].
    Step 264: Loss: 143.523: Scales: [0.5651241 0.342616 ].
    Step 265: Loss: 143.523: Scales: [0.56512403 0.34261596].
    Step 266: Loss: 143.523: Scales: [0.565124   0.34261596].
    Step 267: Loss: 143.523: Scales: [0.5651239  0.34261602].
    Step 268: Loss: 143.523: Scales: [0.56512386 0.34261614].
    Step 269: Loss: 143.523: Scales: [0.5651238 0.3426163].
    Step 270: Loss: 143.523: Scales: [0.56512374 0.34261647].
    Step 271: Loss: 143.523: Scales: [0.5651237  0.34261665].
    Step 272: Loss: 143.523: Scales: [0.5651236  0.34261686].
    Step 273: Loss: 143.523: Scales: [0.56512356 0.34261706].
    Step 274: Loss: 143.523: Scales: [0.5651235  0.34261724].
    Step 275: Loss: 143.523: Scales: [0.56512344 0.34261742].
    Step 276: Loss: 143.523: Scales: [0.5651234  0.34261757].
    Step 277: Loss: 143.523: Scales: [0.5651233 0.3426177].
    Step 278: Loss: 143.523: Scales: [0.5651233 0.3426178].
    Step 279: Loss: 143.523: Scales: [0.5651233 0.3426179].
    Step 280: Loss: 143.523: Scales: [0.5651233  0.34261796].
    Step 281: Loss: 143.523: Scales: [0.5651233 0.342618 ].
    Step 282: Loss: 143.523: Scales: [0.5651233 0.342618 ].
    Step 283: Loss: 143.523: Scales: [0.5651233  0.34261796].
    Step 284: Loss: 143.523: Scales: [0.5651233 0.3426179].
    Step 285: Loss: 143.523: Scales: [0.5651233  0.34261784].
    Step 286: Loss: 143.523: Scales: [0.5651233  0.34261775].
    Step 287: Loss: 143.523: Scales: [0.5651233  0.34261766].
    Step 288: Loss: 143.523: Scales: [0.5651233  0.34261757].
    Step 289: Loss: 143.523: Scales: [0.5651233  0.34261748].
    Step 290: Loss: 143.523: Scales: [0.5651234 0.3426174].
    Step 291: Loss: 143.523: Scales: [0.56512344 0.3426173 ].
    Step 292: Loss: 143.523: Scales: [0.5651235  0.34261724].
    Step 293: Loss: 143.523: Scales: [0.5651235  0.34261718].
    Step 294: Loss: 143.523: Scales: [0.5651235  0.34261715].
    Step 295: Loss: 143.523: Scales: [0.5651235  0.34261712].
    Step 296: Loss: 143.523: Scales: [0.5651235  0.34261712].
    Step 297: Loss: 143.523: Scales: [0.5651235  0.34261712].
    Step 298: Loss: 143.523: Scales: [0.5651235  0.34261712].
    Step 299: Loss: 143.523: Scales: [0.5651235  0.34261715].
    Step 300: Loss: 143.523: Scales: [0.5651235  0.34261718].
    Step 301: Loss: 143.523: Scales: [0.5651235 0.3426172].
    Step 302: Loss: 143.523: Scales: [0.5651235  0.34261724].
    Step 303: Loss: 143.523: Scales: [0.5651235  0.34261727].
    Step 304: Loss: 143.523: Scales: [0.5651235 0.3426173].
    Step 305: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 306: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 307: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 308: Loss: 143.523: Scales: [0.5651235  0.34261742].
    Step 309: Loss: 143.523: Scales: [0.5651235  0.34261745].
    Step 310: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 311: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 312: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 313: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 314: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 315: Loss: 143.523: Scales: [0.5651235  0.34261748].
    Step 316: Loss: 143.523: Scales: [0.5651235  0.34261745].
    Step 317: Loss: 143.523: Scales: [0.5651235  0.34261742].
    Step 318: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 319: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 320: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 321: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 322: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 323: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 324: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 325: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 326: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 327: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 328: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 329: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 330: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 331: Loss: 143.523: Scales: [0.5651235  0.34261733].
    Step 332: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 333: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 334: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 335: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 336: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 337: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 338: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 339: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 340: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 341: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 342: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 343: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 344: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 345: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 346: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 347: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 348: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 349: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 350: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 351: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 352: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 353: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 354: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 355: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 356: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 357: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 358: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 359: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 360: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 361: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 362: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 363: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 364: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 365: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 366: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 367: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 368: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 369: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 370: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 371: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 372: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 373: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 374: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 375: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 376: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 377: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 378: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 379: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 380: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 381: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 382: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 383: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 384: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 385: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 386: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 387: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 388: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 389: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 390: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 391: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 392: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 393: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 394: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 395: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 396: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 397: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 398: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 399: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 400: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 401: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 402: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 403: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 404: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 405: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 406: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 407: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 408: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 409: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 410: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 411: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 412: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 413: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 414: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 415: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 416: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 417: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 418: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 419: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 420: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 421: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 422: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 423: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 424: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 425: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 426: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 427: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 428: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 429: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 430: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 431: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 432: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 433: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 434: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 435: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 436: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 437: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 438: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 439: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 440: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 441: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 442: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 443: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 444: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 445: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 446: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 447: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 448: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 449: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 450: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 451: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 452: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 453: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 454: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 455: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 456: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 457: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 458: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 459: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 460: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 461: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 462: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 463: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 464: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 465: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 466: Loss: 143.523: Scales: [0.5651235 0.3426174].
    Step 467: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 468: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 469: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 470: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 471: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 472: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 473: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 474: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 475: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 476: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 477: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 478: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 479: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 480: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 481: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 482: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 483: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 484: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 485: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 486: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 487: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 488: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 489: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 490: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 491: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 492: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 493: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 494: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 495: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 496: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 497: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 498: Loss: 143.523: Scales: [0.5651235  0.34261736].
    Step 499: Loss: 143.523: Scales: [0.5651235  0.34261736].

```python
# View the distribution parameters

print("Class conditional means:")
print(class_conditionals_binary.loc.numpy())
print("\nClass conditional standard deviations:")
print(class_conditionals_binary.stddev().numpy())
```

    Class conditional means:
    [[5.007317  3.4170732]
     [6.2544303 2.8607595]]

    Class conditional standard deviations:
    [[0.5651235  0.34261736]
     [0.5651235  0.34261736]]

```python
# Plot the loss and convergence of the standard deviation parameters

fig, ax = plt.subplots(1, 2, figsize=(14, 5))
ax[0].plot(nlls)
ax[0].set_title("Loss vs epoch")
ax[0].set_xlabel("Epoch")
ax[0].set_ylabel("Average negative log-likelihood")
for k in [0, 1]:
    ax[1].plot(scales_arr[:, k], color=label_colours_binary[k], label=labels_binary[k])
ax[1].set_title("Standard deviation ML estimates vs epoch")
ax[1].set_xlabel("Epoch")
ax[1].set_ylabel("Standard deviation")
plt.legend()
plt.show()
```

We can also plot the contours of the class-conditional Gaussian distributions as before, this time with just binary labelled data. Notice the contours are the same for each class, just with a different centre location.

```python
# Plot the training data with the class-conditional density contours

plt.figure(figsize=(10, 6))
plot_data(x_train, y_train_binary, labels_binary, label_colours_binary)
x0_min, x0_max = x_train[:, 0].min(), x_train[:, 0].max()
x1_min, x1_max = x_train[:, 1].min(), x_train[:, 1].max()
contour_plot((x0_min, x0_max), (x1_min, x1_max), class_conditionals_binary.prob, 2, label_colours_binary)
plt.title("Training set with class-conditional density contours")
plt.show()
```

We can also plot the decision regions for this binary classifier model, notice that the decision boundary is now linear.

```python
# Plot the model's decision regions

plt.figure(figsize=(10, 6))
plot_data(x_train, y_train_binary, labels_binary, label_colours_binary)
x0_min, x0_max = x_train[:, 0].min(), x_train[:, 0].max()
x1_min, x1_max = x_train[:, 1].min(), x_train[:, 1].max()
contour_plot((x0_min, x0_max), (x1_min, x1_max),
             lambda x: predict_class(prior_binary, class_conditionals_binary, x),
             1, label_colours_binary, levels=[-0.5, 0.5, 1.5],
             num_points=500)
plt.title("Training set with decision regions")
plt.show()
```

#### Link to logistic regression

In fact, we can see that our predictive distribution $P(Y=y_0 | X)$ can be written as follows:

$$
\begin{align}
P(Y=y_0 | X) =& ~\frac{P(X | Y=y_0)P(Y=y_0)}{P(X | Y=y_0)P(Y=y_0) + P(X | Y=y_1)P(Y=y_1)}\\
=&  ~\frac{1}{1 + \frac{P(X | Y=y_1)P(Y=y_1)}{P(X | Y=y_0)P(Y=y_0)}}\\
=& ~\sigma(a)
\end{align}
$$

where $\sigma(a) = \frac{1}{1 + e^{-a}}$ is the sigmoid function, and $a = \log\frac{P(X | Y=y_0)P(Y=y_0)}{P(X | Y=y_1)P(Y=y_1)}$ is the _log-odds_.

With our additional modelling assumption of a shared covariance matrix $\Sigma$, it can be shown (using the Gaussian pdf) that $a$ is in fact a linear function of $X$:

$$
a = w^T X + w_0
$$

where

$$
\begin{align}
w =& ~\Sigma^{-1} (\mu_0 - \mu_1)\\
w_0 =& -\frac{1}{2}\mu_0^T \Sigma^{-1}\mu_0 + \frac{1}{2}\mu_1^T\Sigma^{-1}\mu_1 + \log\frac{P(Y=y_0)}{P(Y=y_1)}
\end{align}
$$

The model therefore takes the form $P(Y=y_0 | X) = \sigma(w^T X + w_0)$, with weights $w\in\mathbb{R}^2$ and bias $w_0\in\mathbb{R}$. This is the form used by logistic regression, and explains why the decision boundary above is linear.

In the above we have outlined the derivation of the generative logistic regression model. The parameters are typically estimated with maximum likelihood, as we have done.

Finally, we will use the above equations to directly parameterise the output Bernoulli distribution of the generative logistic regression model.

You should now write the following function, according to the following specification:

* The inputs to the function are:
  * the prior distribution `prior` over the two classes
  * the (batched) class-conditional distribution `class_conditionals`
* The function should use the parameters of the above distributions to compute the weights and bias terms $w$ and $w_0$ as above
* The function should then return a tuple of two numpy arrays for $w$ and $w_0$

```python
#### GRADED CELL ####

# Complete the following function.
# Make sure to not change the function name or arguments.

def get_logistic_regression_params(prior, class_conditionals):
    """
    This function takes the prior distribution and class-conditional distribution as inputs.
    This function should compute the weights and bias terms of the generative logistic
    regression model as above, and return them in a 2-tuple of numpy arrays of shapes
    (2,) and () respectively.
    """
    sigma_inv = np.linalg.inv(class_conditionals_binary.covariance())
    mu_0 = class_conditionals_binary.loc.numpy()[:,0]
    mu_1 = class_conditionals_binary.loc.numpy()[:,1]
    w = np.dot(sigma_inv,(mu_0-mu_1))
    w_0 = (-0.5)*mu_0.T @ sigma_inv @ mu_0 + 0.5*mu_1.T @ sigma_inv @ mu_1 + np.log(prior.probs[0]/prior.probs[1])

    return w, w_0
```

```python
sigma_inv = np.linalg.inv(class_conditionals_binary.covariance())
print(class_conditionals_binary.loc)
mu_0 = class_conditionals_binary.loc.numpy()[:,0]
mu_1 = class_conditionals_binary.loc.numpy()[:,1]
print(mu_0, mu_1)
print(prior.probs[0])
w = np.dot(sigma_inv,(mu_0-mu_1))
w_0 = -0.5*mu_0.T @ sigma_inv @ mu_0 + 0.5*mu_1.T @ sigma_inv @ mu_1 + np.log(prior.probs[0]/prior.probs[1])
w_0
```

    tf.Tensor(
    [[5.007317  3.4170732]
     [6.2544303 2.8607595]], shape=(2, 2), dtype=float32)
    [5.007317  6.2544303] [3.4170732 2.8607595]
    tf.Tensor(0.34166667, shape=(), dtype=float32)

    -152.75925

```python
# Run your function to get the logistic regression parameters

w, w0 = get_logistic_regression_params(prior_binary, class_conditionals_binary)
```

We can now use these parameters to make a contour plot to display the predictive distribution of our logistic regression model.

```python
# Plot the training data with the logistic regression prediction contours

fig, ax = plt.subplots(1, 1, figsize=(10, 6))
plot_data(x_train, y_train_binary, labels_binary, label_colours_binary)
x0_min, x0_max = x_train[:, 0].min(), x_train[:, 0].max()
x1_min, x1_max = x_train[:, 1].min(), x_train[:, 1].max()
X0, X1 = get_meshgrid((x0_min, x0_max), (x1_min, x1_max))

logits = np.dot(np.array([X0.ravel(), X1.ravel()]).T, w) + w0
Z = tf.math.sigmoid(logits)
lr_contour = ax.contour(X0, X1, np.array(Z).T.reshape(*X0.shape), levels=10)
ax.clabel(lr_contour, inline=True, fontsize=10)
contour_plot((x0_min, x0_max), (x1_min, x1_max),
             lambda x: predict_class(prior_binary, class_conditionals_binary, x),
             1, label_colours_binary, levels=[-0.5, 0.5, 1.5],
             num_points=300)
plt.title("Training set with prediction contours")
plt.show()
```

Congratulations on completing this programming assignment! In the next week of the course we will look at Bayesian neural networks and uncertainty quantification.

Probabilistic Deep Learning with TensorFlow 2/.ipynb_checkpoints/Week 1 Programming Assignment COMPLETED-checkpoint.ipynb


# Eigenvalue and eigenvectors calculation

$$
A\mathbf{x} = \lambda \mathbf{x}
$$

### Power method (vector iteration)
- find the largest eigenvalue $\lambda_{max}$
\begin{align}
\mathbf{q}_k & = \frac{\mathbf{z}_{k-1}}{\|\mathbf{z}_{k-1}\|_2}\\
\mathbf{z}_k & = A\mathbf{q}_{k}\\
\lambda_{max}^k & = \mathbf{q}^T_k \mathbf{z}_k
\end{align}

```python

```

```python
%matplotlib inline
from numpy import *
from matplotlib.pyplot import *
import numpy.linalg
import scipy.linalg

n = 9
h = 1./(n-1)

x=linspace(0,1,n)

a = -ones((n-1,))
b = 2*ones((n,))
A = (diag(a, -1) + diag(b, 0) + diag(a, +1))

A /= h**2

print(A)

z0 = ones_like(x)
print(z0)

def PM(A,z0,tol=1e-5,nmax=500):
    q = z0/numpy.linalg.norm(z0,2)
    it = 0
    err = tol + 1.
    while it < nmax and err > tol:
        z = dot(A,q)
        l = dot(q.T,z)
        err = numpy.linalg.norm(z-l*q,2)
        q = z/numpy.linalg.norm(z,2)

        it += 1
    print("error =", err, "iterations =", it)
    print("lambda_max =", l)
    return l,q

l,x = PM(A,z0)

l_np, x_np = numpy.linalg.eig(A)

print("numpy")
print(l_np)
```

    [[128. -64.   0.   0.   0.   0.   0.   0.   0.]
     [-64. 128. -64.   0.   0.   0.   0.   0.   0.]
     [  0. -64. 128. -64.   0.   0.   0.   0.   0.]
     [  0.   0. -64. 128. -64.   0.   0.   0.   0.]
     [  0.   0.   0. -64. 128. -64.   0.   0.   0.]
     [  0.   0.   0.   0. -64. 128. -64.   0.   0.]
     [  0.   0.   0.   0.   0. -64. 128. -64.   0.]
     [  0.   0.   0.   0.   0.   0. -64. 128. -64.]
     [  0.   0.   0.   0.   0.   0.   0. -64. 128.]]
    [1. 1. 1. 1. 1. 1. 1. 1. 1.]
    error = 8.456086477502974e-06 iterations = 82
    lambda_max = 249.7352340857781
    numpy
    [249.73523409 231.55417528 203.23651229 167.55417528 128.
       6.26476591  24.44582472  88.44582472  52.76348771]

### Power method with shift
- find the eigenvalue $\lambda$ **closest** to $\mu$
\begin{align}
M & = A-\mu I\\
M & = LU \\
& \\
M\mathbf{x}_k &= \mathbf q_{k-1}\\
\mathbf{q}_k & = \frac{\mathbf{x}_k}{\|\mathbf{x}_k\|_2}\\
\mathbf{z}_k & = A\mathbf{q}_{k}\\
\lambda^k & = \mathbf{q}^T_k \mathbf{z}_k
\end{align}

```python
def IPM(A,x0,mu,tol=1e-5,nmax=500):
    M = A - mu * eye(A.shape[0])
    Minv = numpy.linalg.inv(M)
    y = x0/numpy.linalg.norm(x0, 2)
    l = 1/dot(y.T,dot(Minv,y))
    it = 0
    err = tol + 1
    while it < nmax and err > tol:
        x = dot(Minv,y)
        y = x/numpy.linalg.norm(x,2)
        l_new = 1/dot(y.T,dot(Minv,y))
        err = abs(l_new - l)
        l = l_new
        it += 1
    print("error =", err, "iterations =", it)
    print("lambda_max =", l + mu)
    return l + mu,y
    pass

l,x = IPM(A,z0,6.)

```

    error = 8.49941671821064e-07 iterations = 2
    lambda_max = 6.264765914246636

```python
M = A - 6 * eye(A.shape[0])
```

```python
A.shape[0]
```

    9

```python

```


## Parametric nonconvex optimization

Consider the following minimization problem ([reference](https://alphaville.github.io/optimization-engine/docs/example_rosenbrock_py))
$$
\begin{align}
    \operatorname*{Minimize}_{\|u\|\leq r}& \sum_{i=1}^{n_u - 1} b (u_{i+1} - u_{i}^2)^2 + (a-u_i)^2
    \\
    \text{subject to: }& 1.5 u_1 - u_2 = 0
    \\
    &u_3 - u_4 + 0.1 \leq 0
    \end{align}
$$
The parameter vector is $p=(a,b)$.

Let us generate a parametric optimiser with $n_u=5$.

```python
import opengen as og
import casadi.casadi as cs

# Build parametric optimizer
# ------------------------------------
u = cs.SX.sym("u", 5)  # decision variables
p = cs.SX.sym("p", 2)  # parameters, p = (a, b)

# cost function:
phi = og.functions.rosenbrock(u, p)

# constraints:
c = cs.vertcat(1.5 * u[0] - u[1],
               cs.fmax(0.0, u[2] - u[3] + 0.1))

# simple bounds on decision variables:
bounds = og.constraints.Ball2(None, 1.5)

# problem formulation
problem = og.builder.Problem(u, p, phi) \
    .with_penalty_constraints(c)        \
    .with_constraints(bounds)
```

## Generate optimizer

The following code will generate your parametric optimizer.

The auto-generated files will be stored in `optimizers/rosenbrock`.

```python
# Configure and build
# (This might take a while)
# ---------------------------------
build_config = og.config.BuildConfiguration()      \
    .with_build_directory("optimizers")            \
    .with_build_mode(og.config.BuildConfiguration.DEBUG_MODE)  \
    .with_tcp_interface_config()
meta = og.config.OptimizerMeta()                   \
    .with_optimizer_name("rosenbrock")
solver_config = og.config.SolverConfiguration()    \
    .with_tolerance(1e-5)                          \
    .with_delta_tolerance(1e-4)                    \
    .with_initial_penalty(1e3)                     \
    .with_penalty_weight_update_factor(5)
builder = og.builder.OpEnOptimizerBuilder(problem, meta,
                                          build_config, solver_config)
builder.build()
```

## Use the auto-generated optimizer

You may now use the auto-generated optimizer in Python.

Tip: First you need to start the server. Then, you can call it as many times as you like. When you're done, you can kill it.

```python
# Start the optimisation server
# ----------------------------------
mng = og.tcp.OptimizerTcpManager('optimizers/rosenbrock')
mng.start()
```

```python
# Call the server
# ----------------------------------
response = mng.call([1.0, 50.0])
if response.is_ok():
    # Solver returned a solution
    solution_data = response.get()
    u_star = solution_data.solution
    exit_status = solution_data.exit_status
    solver_time = solution_data.solve_time_ms
    print("u_star = ", u_star)
    print("solved in = ", solver_time, "ms")

```

```python
# Kill the server
# ----------------------------------
mng.kill()
```

Tip: The auto-generated TCP server you built above listens for requests at port 8333. Since it runs within this docker container, if you want to call it from another application outside this Jupyter notebook you need to forward the port - to do so, run the docker image with -p 8333:8333.


```python
import sympy as sym
from approx1D import least_squares
import scipy
import scipy.integrate

def sin_series(N):
  psi = []
  for k in range(1,N):
    psi_k = sym.sin(sym.pi*k*x)
    psi.append(psi_k)
  return psi

def taylor_series(N):
  psi = []
  for k in range(1,N):
    psi_k = x**k
    psi.append(psi_k)
  return psi

def series(series_type, N):
  if series_type=="Taylor" : return taylor_series(N)
  elif series_type=="sin"  : return sin_series(N)
  else: print("series type unknown ") # sys.exit(0)

X = scipy.arange(0, 1, 0.0001)
import pylab
def plot_solution(series_type, N, func):

  print(("series_type ", series_type, " N ", N, " func ", func))

  psi = series(series_type, N)
  print(psi)
  u, c = least_squares(func, psi, Omega, False)

  u = sym.lambdify([x], u)
  U = [u(xi) for xi in X]
  pylab.plot(X, U)

def plot_u(u):

  u = sym.lambdify([x], u)
  U = [u(xi) for xi in X]
  pylab.plot(X, U)

Omega = [0, 1]
x = sym.Symbol("x")
gauss_bell = sym.exp(-(x-0.5)**2) - sym.exp(-0.5**2)
#step = sym.Piecewise( (1, 0.25 < x and x < 0.75), (0, True)  )
step = sym.Piecewise( (1, 0.25 <= x), (0, True)  )- sym.Piecewise( (1, 0.75 < x), (0, True)  )
hat = sym.Piecewise( (x, x <= 0.5), (0, True)  ) + sym.Piecewise( (1-x, 0.5 < x), (0, True)  )

print(("hat (0.3) ", hat.subs(x, 0.3)))

func=hat

Ns =[5, 10, 15, 20, 25]
#Ns =[2, 3, 4, 5]
for N in Ns:
#  plot_solution("sin", N, func)
  plot_solution("Taylor", N, func)
legend = ["N=%d" % N for N in Ns]
plot_u(func)
legend.append("analytical solution")
pylab.legend(legend)
pylab.show()
pylab.savefig("step.png", title="sin approximation")

```

```python

```

Data Science and Machine Learning/Machine-Learning-In-Python-THOROUGH/EXAMPLES/FINITE_ELEMENTS/INTRO/SRC/34_PLOT_GLOBAL_BELL.ipynb