### Week 7 Homework Notebook
This is to use for python homework practice

### Description
Use for examples or solve homework problems for week 7.

```python
import plotly.plotly
from plotly.graph_objs import Scatter, Layout
from numpy import arange, cos
import numpy as np
import math

import sympy as sp
x, y, z = sp.symbols('x y z')
sp.init_printing()
#sp.init_printing(use_unicode=True)
#sp.init_printing(use_latex='mathjax')
from sympy.utilities.lambdify import lambdify, implemented_function

print('imports completed')
```

    imports completed

```python
class MyFunctions:

    # class init fn
    def __init__(self, low_bound, high_bound, samples):
        self.low_bound = low_bound
        self.high_bound = high_bound
        self.samples = samples

    def get_space(self):
        return np.linspace(self.low_bound, self.high_bound, self.samples)

    # functions for homework
    def h(self, x):
        expr = 2*x ** 3 + 6*x ** 2 - 12
        return expr

    def g(self, x):
        expr = 13*x ** 2 + 76*x
        expr_prime = sp.diff(expr)
        result = expr_prime.evalf(subs={x:100})
        return expr_prime

    def eval_sympy_fn(self, expr_str, x_val):
        #sympy_expr = sp.parsing.parse_expr(expr_str)
        sympy_expr = expr_str
        print(sympy_expr)
        result = sympy_expr.evalf(subs={x:x_val})
        return result

# Test MyFunctions
mf = MyFunctions(-10, 10, 20)

# build data array test function
def build_data(fn_name, space):
    tmparray = []
    fn = getattr(mf, fn_name)
    for i in space:
        tmparray.append(float(fn(i)))
    #print(tmparray)
    return np.array(tmparray)

# build sympy data array using lambda
def test_impl_fn(space):
    f2 = implemented_function(sp.Function('f2'), lambda x: 2*x ** 3 + 6*x ** 2 - 12)
    lam_f2 = lambdify(x, f2(x), 'numpy')
    #print(lam_f2(a))
    return lam_f2(space)

def lambdify_fun(fn, space):
    lam = lambdify(x, fn, 'numpy')
    return lam(space)

def lambdify_prime_fun(fn, space):
    expr_prime = sp.diff(fn)
    lam_p = lambdify(x, expr_prime, 'numpy')
    return lam_p(space)

def lambdify_doubleprime_fun(fn, space):
    expr_primeprime = sp.diff(sp.diff(fn))
    lam_dp = lambdify(x, expr_primeprime, 'numpy')
    return lam_dp(space)

# TESTS
#f1 = implemented_function(sp.Function('f1'), lambda x: x+1)
#lam_f1 = lambdify(x, f1(x))
#r = lam_f1(10)
#print(r)

# THIS ALL WORKS
#a = np.arange(10)
#print(a)
#exstr = lambdastr(x, 2*x ** 3 + 6*x ** 2 - 12)
#f2 = implemented_function(sp.Function('f2'), lambda x: 2*x ** 3 + 6*x ** 2 - 12)
#lam_f2 = lambdify(x, f2(x), 'numpy')
#print('soln is ')
#print(lam_f2(a))

# New Test
#a = np.arange(10)
#print(a)
#print(test_h_fn(a))
#print(test_g_fn(a))
```

```python
# Instantiate
mf = MyFunctions(-50, 50, 100)
#expression = sp.sympify('2*x ** 3 + 6*x ** 2 - 12')
#la = sp.latex('( (x - 1) 2 / 3)')
expression = sp.sympify(sp.Rational(1/60)*x**3 + 5*x +10)
print(expression)

# Required for displaying plotly in jupyter notebook
plotly.offline.init_notebook_mode(connected=True)

# Create traces
#trace1 = Scatter(x=mf.get_space(), y=build_data('h', mf.get_space()), name='y', line=dict(color='#bc42f4'))
#trace2 = Scatter(x=mf.get_space(), y=build_data('g', mf.get_space()), name='y prime', line=dict(color='#f44641'))
#trace1 = Scatter(x=the_space, y=test_h_fn(the_space), name='y', line=dict(color='#bc42f4'))
the_space = mf.get_space()
trace1 = Scatter(x=the_space, y=lambdify_fun(expression, the_space), name='y', line=dict(color='#bc42f4'))
trace2 = Scatter(x=the_space, y=lambdify_prime_fun(expression, the_space), name='y prime', line=dict(color='#f44641'))
trace3 = Scatter(x=the_space, y=lambdify_doubleprime_fun(expression, the_space), name='y double prime', line=dict(color='#0A8A0A'))

# plot it
plotly.offline.iplot({
    "data": [trace1, trace2, trace3],
    "layout": Layout(title="HW Test1")
})
```

    4803839602528529*x**3/288230376151711744 + 5*x + 10

```python

```


# Fundamentals of Data Science
Winter Semester 2021

## Prof. Fabio Galasso, Guido D'Amely, Alessandro Flaborea, Luca Franco, Muhammad Rameez Ur Rahman and Alessio Sampieri
<galasso@di.uniroma1.it>, <damely@di.uniroma1.it>, <flaborea@di.uniroma1.it>, <franco@diag.uniroma1.it>, <rahman@di.uniroma1.it>, <alessiosampieri27@gmail.com>

## Exercise 2: Classification

In Exercise 2, you will re-derive and implement logistic regression and optimize the parameters with Gradient Descent and with the Newton's method. Also, in this exercise you will re-derive and implement Gassian Discriminant Analysis.
We will use datasets generated from the make_classification function from the SkLearn library. Its first output contains the feature values $x^{(i)}_1$ and $x^{(i)}_2$ for the $i$-th data sample $x^{(i)}$. The second contains the ground truth label $y^{(i)}$ for each corresponding data sample.

The completed exercise should be handed in as a single notebook file. Use Markdown to provide equations. Use the code sections to provide your scripts and the corresponding plots.
Submit it by sending an email to galasso@di.uniroma1.it, flaborea@di.uniroma1.it, franco@diag.uniroma1.it and alessiosampieri27@gmail.com  by Wednesday November 17th 2021, 23:59.

## Notation

- $x^i$ is the $i^{th}$ feature vector
- $y^i$ is the expected outcome for the $i^{th}$ training example
- $m$ is the number of training examples
- $n$ is the number of features

Let's start by setting up our Python environment and importing the required libraries:

```python
%matplotlib inline
import numpy as np # imports a fast numerical programming library
import scipy as sp # imports stats functions, amongst other things
import matplotlib as mpl # this actually imports matplotlib
import matplotlib.cm as cm # allows us easy access to colormaps
import matplotlib.pyplot as plt # sets up plotting under plt
import pandas as pd # lets us handle data as dataframes
from sklearn.datasets import make_classification
import seaborn as sns

# sets up pandas table display
pd.set_option('display.width', 500)
pd.set_option('display.max_columns', 100)
pd.set_option('display.notebook_repr_html', True)

import seaborn as sns # sets up styles and gives us more plotting options
```

## Question 1: Logistic Regression with Gradient Ascent **(10 Points)**

### Code and Theory

#### Exercise 1.a **(3 Points)** Equations for the log likelihood, its gradient, and the gradient ascent update rule.

Write and simplify the likelihood $L(\theta)$ and log-likelihood $l(\theta)$ of the parameters $\theta$.

Recall the probabilistic interpretation of the hypothesis $h_\theta(x)= P(y=1|x;\theta)$ and that $h_\theta(x)=\frac{1}{1+\exp(-\theta^T x)}$.

Also derive the gradient $\frac{\delta l(\theta)}{\delta \theta_j}$ of $l(\theta)$ and write the gradient update equation.

Question: Are we looking for a local minimum or a local maximum using the gradient ascent rule?

################# Do not write above this line #################

Your equations and answers here.

################# Do not write below this line #################

#### Exercise 1.b **(7 Points)** Implementation of logistic regression with Gradient Ascent

Code up the equations above to learn the logistic regression parameters. The dataset used here is created using the make_classification function present in the SkLearn library. $x^{(i)}_1$ and $x^{(i)}_2$ represent the two features for the $i$-th data sample $x^{(i)}$ and $y^{(i)}$ is its ground truth label.

```python
X, y = make_classification(n_samples=500, n_features=2, n_informative=2, n_redundant=0, n_classes=2, random_state=5)
X.shape, y.shape
```

    ((500, 2), (500,))

```python
sns.scatterplot(x=X[:,0], y=X[:,1], hue=y);
```

Adding a column of 1's to $X$ to take into account the zero intercept

```python
x = np.hstack([np.ones((X.shape[0], 1)), X])
```

```python
[x[:5,:],x[-5:,:]] # Plot the first and last 5 lines of x, now containing features x0 (constant=1), x1 and x2
```

    [array([[ 1.        ,  2.25698215, -1.34710915],
            [ 1.        ,  1.43699308,  1.28420453],
            [ 1.        ,  0.57927295,  0.23690172],
            [ 1.        ,  0.42538132, -0.24611145],
            [ 1.        ,  1.13485101, -0.61162683]]),
     array([[ 1.        ,  1.56638944,  0.81749944],
            [ 1.        , -1.94913831, -1.90601147],
            [ 1.        ,  1.53440506, -0.11687238],
            [ 1.        , -0.39243599,  1.39209018],
            [ 1.        , -0.11881249,  0.96973739]])]

```python
[y[:5],y[-5:]] # Plot the first and last 5 lines of y
```

    [array([1, 1, 1, 0, 1]), array([1, 0, 0, 0, 1])]

Define the sigmoid function "sigmoid", the function to compute the gradient of the log likelihood  "grad_l" and the gradient ascent algorithm.

################# Do not write above this line #################

```python
def sigmoid(x):
    '''
    Function to compute the sigmoid of a given input x.

    Input:
    x: it's the input data matrix. The shape is (N, H)

    Output:
    g: The sigmoid of the input x
    '''

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################

    return g

def log_likelihood(theta,features,target):
    '''
    Function to compute the log likehood of theta according to data x and label y

    Input:
    theta: it's the model parameter matrix.
    features: it's the input data matrix. The shape is (N, H)
    target: the label array

    Output:
    log_g: the log likehood of theta according to data x and label y
    '''

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################

    return log_l

def predictions(features, theta):
    '''
    Function to compute the predictions for the input features

    Input:
    theta: it's the model parameter matrix.
    features: it's the input data matrix. The shape is (N, H)

    Output:
    preds: the predictions of the input features
    '''

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################

    return preds

def update_theta(theta, target, preds, features, lr):
    '''
    Function to compute the gradient of the log likelihood
    and then return the updated weights

    Input:
    theta: the model parameter matrix.
    target: the label array
    preds: the predictions of the input features
    features: it's the input data matrix. The shape is (N, H)
    lr: the learning rate

    Output:
    theta: the updated model parameter matrix.
    '''

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################
    return theta

def gradient_ascent(theta, features, target, lr, num_steps):
    '''
    Function to execute the gradient ascent algorithm

    Input:
    theta: the model parameter matrix.
    target: the label array
    num_steps: the number of iterations
    features: the input data matrix. The shape is (N, H)
    lr: the learning rate

    Output:
    theta: the final model parameter matrix.
    log_likelihood_history: the values of the log likelihood during the process
    '''

    log_likelihood_history = np.zeros(num_steps)

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################

    return theta, log_likelihood_history

```

################# Do not write below this line #################

Check your grad_l implementation:
grad_l applied to the theta_test (defined below) should provide a value for log_l_test close to the target_value (defined below); in other words the error_test should be 0, up to machine error precision.

```python
target_value = -1.630501731599431

output_test  = log_likelihood(np.array([-7,4,1]),x,y)
error_test=np.abs(output_test-target_value)

print("{:f}".format(error_test))
```

    0.000000

Let's now apply the function gradient_ascent and print the final theta as well as theta_history

```python
# Initialize theta0
theta0 = np.zeros(x.shape[1])

# Run Gradient Ascent method
n_iter=1000
theta_final, log_l_history = gradient_ascent(theta0,x,y,lr=0.5,num_steps=n_iter)
print(theta_final)

```

    [-0.46097042  2.90036399  0.23146846]

Let's plot the log likelihood over iterations

```python
fig,ax = plt.subplots(num=2)

ax.set_ylabel('l(Theta)')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)),log_l_history,'b.')
```

Plot the data and the decision boundary:

```python
# Generate vector to plot decision boundary
x1_vec = np.linspace(X[:,0].min(),X[:,1].max(),2)

# Plot raw data
sns.scatterplot(x=X[:,0], y=X[:,1], hue=y, data=X)

# Plot decision boundary
plt.plot(x1_vec,(-x1_vec*theta_final[1]-theta_final[0])/theta_final[2], color="red")
plt.ylim(X[:,1].min()-1,X[:,1].max()+1)
# Save the theta_final value for later comparisons
theta_GA = theta_final.copy()
```

################# Do not write above this line #################

Discuss these two points:
1. You have implemented the gradient ascent rule. Could we have also used gradient descent instead for the proposed problem? Why/Why not?
2. Let's deeply analyze how the learning rate $\alpha$ and the number of iterations affect the final results. Run the algorithm you have written for different values of $\alpha$ and the number of iterations and look at the outputs you get. Is the decision boundary influenced by these parameters change? Why do you think these parameters are affecting/not affecting the results?

################# Do not write below this line #################

## Question 2: Logistic Regression with non linear boundaries (7 points)

#### Exercise 2.a **(4 Points)** Polynomial features for logistic regression

Define new features, e.g. of 2nd and 3rd degrees, and learn a logistic regression classifier by using the new features, by using the gradient ascent optimization algorithm you defined in Question 1.

In particular, we would consider a polynomial boundary with equation:

$f(x_1, x_2) = c_0 + c_1 x_1 + c_2 x_2 + c_3 x_1^2 + c_4 x_2^2 + c_5 x_1 x_2 + c_6 x_1^3 + c_7 x_2^3 + c_8 x_1^2 x_2 + c_9 x_1 x_2^2$

We would therefore compute 7 new features: 3 new ones for the quadratic terms and 4 new ones for the cubic terms.

Create new arrays by stacking x and the new 7 features (in the order x1x1, x2x2, x1x2, x1x1x1, x2x2x2, x1x1x2, x1x2x2). In particular create x_new_quad by additionally stacking with x the quadratic features, and x_new_cubic by additionally stacking with x the quadratic and the cubic features.

```python
from sklearn.datasets import make_classification

X, y = make_classification(n_samples=500, n_features=2, n_informative=2, n_redundant=0, n_classes=2, random_state=5)
X.shape, y.shape
```

    ((500, 2), (500,))

```python
x = np.hstack([np.ones((X.shape[0], 1)), X])
```

```python
import seaborn as sns
import matplotlib.pyplot as plt
sns.scatterplot(x=X[:,0], y=X[:,1], hue=y);
```

```python
# First extract features x1 and x2 from x and reshape them to x1 vector arrays
x1 = x[:,1]
x2 = x[:,2]
x1 = x1.reshape(x1.shape[0], 1)
x2 = x2.reshape(x2.shape[0], 1)
print(x[:5,:]) # For visualization of the first 5 values
print(x1[:5,:]) # For visualization of the first 5 values
print(x2[:5,:]) # For visualization of the first 5 values
```

    [[ 1.          2.25698215 -1.34710915]
     [ 1.          1.43699308  1.28420453]
     [ 1.          0.57927295  0.23690172]
     [ 1.          0.42538132 -0.24611145]
     [ 1.          1.13485101 -0.61162683]]
    [[2.25698215]
     [1.43699308]
     [0.57927295]
     [0.42538132]
     [1.13485101]]
    [[-1.34710915]
     [ 1.28420453]
     [ 0.23690172]
     [-0.24611145]
     [-0.61162683]]

################# Do not write above this line #################

Your code here

```python

def new_features(x, degree=2):
    '''
    Function to create n-degree features from the input

    Input:
    x: the initial features
    degree: the maximum degree you wantthe features

    Output:
    features: the final features.
              2nd degree features must have the order [x, x1x1, x1x2, x2x2]
              2nd degree features must have the order [x, x1x1, x1x2, x2x2, x1x1x1, x1x1x2, x1x2x2, x2x2x2]
    '''

    features = np.ones(x[:,1].shape[0])

    #####################################################
    ##                 YOUR CODE HERE                  ##
    #####################################################

    return features

```

################# Do not write below this line #################

```python
x_new_quad = new_features(x, degree=2)
x_new_cubic = new_features(x, degree=3)

#reordering output features
temp = np.copy(x_new_quad[:, -1])
x_new_quad[:, -1] = x_new_quad[:, -2]
x_new_quad[:, -2] = temp

temp = np.copy(x_new_cubic[:, -1])
x_new_cubic[:, -1] = x_new_cubic[:, -2]
x_new_cubic[:, -2] = x_new_cubic[:, -3]
x_new_cubic[:, -3] = temp

```

Now use the gradient ascent optimization algorithm to learn theta by maximizing the log-likelihood, both for the case of x_new_quad and x_new_cubic.

```python
# Initialize theta0, in case of quadratic features
theta0_quad = np.zeros(x_new_quad.shape[1])

theta_final_quad, log_l_history_quad = gradient_ascent(theta0_quad,x_new_quad,y,lr=0.5,num_steps=n_iter)

# Initialize theta0, in case of quadratic and cubic features
theta0_cubic = np.zeros(x_new_cubic.shape[1])

# Run Newton's method, in case of quadratic and cubic features
theta_final_cubic, log_l_history_cubic = gradient_ascent(theta0_cubic,x_new_cubic,y,lr=0.5,num_steps=n_iter)

# check and compare with previous results
print(theta_final_quad)
print(theta_final_cubic)
```

    [ 0.07548038  3.32398303  0.27106753 -0.51873505 -0.34088582 -0.04846172]
    [ 0.82395453  2.41957969  1.66049077 -1.13927962 -0.20657699 -0.96074035
      0.34384121 -0.64078458  0.95132383  1.40927121]

```python
# Plot the log likelihood values in the optimization iterations, in one of the two cases.
fig,ax = plt.subplots(num=2)

ax.set_ylabel('l(Theta)')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history_quad)),log_l_history_quad,'b.')
```

#### Exercise 2.b **(3 Points)** Plot the computed non-linear boundary and discuss the questions

First, define a boundary_function to compute the boundary equation for the input feature vectors $x_1$ and $x_2$, according to estimated parameters theta, both in the case of quadratic (theta_final_quad) and of quadratic and cubic features (theta_final_cubic). Refer for the equation to the introductory part of Question 2.

################# Do not write above this line #################

Your code here

```python
def boundary_function(x1_vec, x2_vec, theta_final):

    x1_vec, x2_vec = np.meshgrid(x1_vec,x2_vec)

    if len(theta_final) == 6:
        # boundary function value for features up to quadratic
        c_0, c_1, c_2, c_3, c_4, c_5 = theta_final
        # f =
    elif len(theta_final) == 10:
        # boundary function value for features up to cubic
        c_0, c_1, c_2, c_3, c_4, c_5, c_6, c_7, c_8, c_9 = theta_final
        # f =
    else:
        raise("Number of Parameters is not correct")

    return x1_vec, x2_vec, f
```

################# Do not write below this line #################

Now plot the decision boundaries corresponding to the theta_final_quad and theta_final_cubic solutions.

```python
x1_vec = np.linspace(X[:,0].min()-1,X[:,0].max()+1,200);
x2_vec = np.linspace(X[:,1].min()-1,X[:,1].max()+1,200);

x1_vec, x2_vec, f = boundary_function(x1_vec, x2_vec, theta_final_quad)

sns.scatterplot(x=X[:,0], y=X[:,1], hue=y, data=X);

plt.contour(x1_vec, x2_vec, f, colors="red", levels=[0])
plt.show()
```

```python
x1_vec = np.linspace(X[:,0].min()-1,X[:,0].max()+1,200);
x2_vec = np.linspace(X[:,1].min()-1,X[:,1].max()+1,200);

x1_vec, x2_vec, f = boundary_function(x1_vec, x2_vec, theta_final_cubic)

sns.scatterplot(x=X[:,0], y=X[:,1], hue=y, data=X);

plt.contour(x1_vec, x2_vec, f, colors="red", levels=[0])
plt.show()
```

#### Confusion Matrix

Here you can see the confusion matrices related to the three models you've implemented.

```python
from sklearn.metrics import confusion_matrix
```

```python
## logistic regression with linear buondary

z = np.dot(x,theta_final)
probabilities = sigmoid(z)
y_hat = np.array(list(map(lambda x: 1 if x>0.5 else 0, probabilities)))
confusion_matrix(y, y_hat)
```

    array([[218,  35],
           [ 22, 225]], dtype=int64)

```python
## logistic regression with non linear buondary - quadratic

z = np.dot(x_new_quad,theta_final_quad)
probabilities = sigmoid(z)
y_hat = np.array(list(map(lambda x: 1 if x>0.5 else 0, probabilities)))
confusion_matrix(y, y_hat)
```

    array([[220,  33],
           [ 15, 232]], dtype=int64)

```python
## logistic regression with non linear buondary - cubic

z = np.dot(x_new_cubic,theta_final_cubic)
probabilities = sigmoid(z)
y_hat = np.array(list(map(lambda x: 1 if x>0.5 else 0, probabilities)))
confusion_matrix(y, y_hat)
```

    array([[225,  28],
           [ 11, 236]], dtype=int64)

################# Do not write above this line #################

Write now your considerations. Discuss in particular:
- Look back at the plots you have generated. What can you say about the differences between the linear, quadratic, and cubic decision boundaries? Can you say if the model is improving in performances, increasing the degree of the polynomial? Do you think you can incur in underfitting increasing more and more the degree?
- Let's now delve into some quantitative analysis. The three tables you have generated represent the confusion matrix for the model you have implemented in the first two questions. What can you say about actual performances? Does the increase of the degree have a high effect on the results?

################# Do not write below this line #################

## Question 3: Multinomial Classification (Softmax Regression) **(13 Points)**

### Code and Theory **(10 Points)**
### Report **(3 Points)**

#### Exercise 3.a **(4 Points)**

In the multinomial classification we generally have $K>2$ classes. So the label for the $i$-th sample $X_i$ is $y_i\in\{1,...,K\}$, where $i=1,...,N$. The output class for each sample is estimated by returning a score $s_i$ for each of the K classes. This results in a vector of scores of dimension K.
In this exercise we'll use the *Softmax Regression* model, which is the natural extension of *Logistic Regression* for the case of more than 2 classes. The score array is given by the linear model:

\begin{equation}
s_i =  X_i \theta
\end{equation}

Scores may be interpreted probabilistically, upon application of the function *softmax*. The position in the vector with the highest probability will be predicted as the output class. The probability of the class k for the $i$-th data sample is:

\begin{equation}
p_{ik} = \frac{\exp(X_i \theta_k)}{\sum_{j=1}^K(X_i \theta_j))}
\end{equation}

We will adopt the *Cross Entropy* loss and optimize the model via *Gradient Descent*.
In the first of this exercise we have to:
-    Write the equations of the Cross Entropy loss for the Softmax regression model;
-    Compute the equation for the gradient of the Cross Entropy loss for the model, in order to use it in the gradient descent algorithm.

#### A bit of notation

*  N: is the number of samples
*  K: is the number of classes
*  X: is the input dataset and it has shape (N, H) where H is the number of features
*  y: is the output array with the labels; it has shape (N, 1)
*  $\theta$: is the parameter matrix of the model; it has shape (H, K)

################# Do not write above this line #################

Your equations here.

\begin{equation}
L(\theta) = -\frac{1}{N} \sum_{i=1}^{N}\sum_{j=1}^{K} y_{ik}\log{\frac{\exp(X_i \theta_k)}{\sum_{j=1}^K\exp(X_i \theta_j)}}
\end{equation}

\begin{equation}
\nabla_{\theta_k} L(\theta) = -\frac{1}{N} \sum_{i=1}^{N} [ X_i (y_{ik} - \frac{\exp(X_i \theta_k)}{\sum_{j=1}^K\exp(X_i \theta_j)})]
\end{equation}

################# Do not write below this line #################

#### Exercise 3.b **(4 Points)**

Now we will implement the code for the equations. Let's implement the functions:
-  softmax
-  CELoss
-  CELoss gradient
-  gradient descent

We generate a toy dataset with *sklearn* library. Do not change anything outside the parts provided of your own code (else the provided checkpoint will not work).

```python
from sklearn.datasets import make_classification

X, y = make_classification(n_samples=300, n_features=7, n_informative=7, n_redundant=0, n_classes=3, random_state=1)
X.shape, y.shape
```

    ((300, 7), (300,))

As a hint for the implementations of your functions: consider the labels $y$ as one-hot vector. This will allow matrix operations (element-wise multiplication and summation).

```python
import scipy
import numpy as np

def class2OneHot(vec):
    out_sparse = scipy.sparse.csr_matrix((np.ones(vec.shape[0]), (vec, np.array(range(vec.shape[0])))))
    out_onehot = np.array(out_sparse.todense()).T
    return out_onehot

y_onehot = class2OneHot(y)
```

Let's visualize the generated dataset. We use as visualizzation method the *Principal Component Analysis* (PCA). PCA summarize the high-dimensional feature vectors of each sample into 2  features, which we can illustrate with a 2D plot. Look at the following plot, the 3 generated classes do not seem separable.

```python
from sklearn.decomposition import PCA
import pandas as pd

pca = PCA(n_components=2)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents, columns = ['pc1', 'pc2'])
finalDf = pd.concat([principalDf, pd.DataFrame(y, columns = ['target'])], axis = 1)
```

```python
import seaborn as sns
import matplotlib.pyplot as plt
sns.scatterplot(x='pc1', y='pc2', hue='target', data=finalDf);
```

################# Do not write above this line #################

```python
def softmax(theta, X):
    '''
    Function to compute associated probability for each sample and each class.

    Input:
    theta: it's the model parameter matrix. The shape is (H, K)
    X: it's the input data matrix. The shape is (N, H)

    Output:
    softmax: it's the matrix containing probability for each sample and each class. The shape is (N, K)
    '''
    # Create the matrix for the dot products between theta and X
    dot = np.zeros((len(X), len(theta[0])), dtype = float)

    # Iterate for each column of theta and for each row of X
    for k in range(len(theta[0])):
        for i in range(len(X)):
            # Compute the dot product
            dot[i][k] = np.dot(X[i], theta[:,k])

    # Subtract the maximum to normalize the results
    dot -= np.max(dot)

    # Compute the softmax matrix
    softmax = ( np.exp(dot).T / np.sum(np.exp(dot), axis = 1) ).T

    return softmax

def CELoss(theta, X, y_onehot):
    '''
    Function to compute softmax regression model and Cross Entropy loss.

    Input:
    theta: it's the model parameter matrix. The shape is (H, K)
    X: it's the input data matrix. The shape is (N, H)
    y_onehot: it's the label array in encoded as one hot vector. The shape is (N, K)

    Output:
    loss: The scalar that is the mean error for each sample.
    '''

    # Call the softmax function
    s = softmax(theta, X)

    # Compute loss
    for k in range(len(s[0])):
        for i in range(len(s)):
            loss = (-1/len(X)) * (np.sum((y_onehot[i][k]) * np.log(s[i][k])))

    return loss

def CELoss_jacobian(theta, X, y_onehot):
    '''
    Function to compute gradient of the cross entropy loss with respect the parameters.

    Input:
    theta: it's the model parameter matrix. The shape is (H, K)
    X: it's the input data matrix. The shape is (N, H)
    y_onehot: it's the label array in encoded as one hot vector. The shape is (N, K)

    Output:
    jacobian: A matrix with the partial derivatives of the loss. The shape is (H, K)
    '''
    # Call the softmax function
    s = softmax(theta, X)

    # Compute the jacobian matrix
    jacobian = (-1/len(X)) * (np.dot(X.T, (y_onehot - s)))

    return jacobian

def gradient_descent(theta, X, y_onehot, alpha=0.01, iterations=100):
    '''
    Function to compute gradient of the cross entropy loss with respect the parameters.

    Input:
    theta: it's the model parameter matrix. The shape is (H, K)
    X: it's the input data matrix. The shape is (N, H)
    y_onehot: it's the label array in encoded as one hot vector. The shape is (N, K)
    alpha: it's the learning rate, so it determines the speed of each step of the GD algorithm
    iterations: it's the total number of step the algorithm performs

    Output:
    theta: it's the updated matrix of the parameters after all the iterations of the optimization algorithm. The shape is (H, K)
    loss_history: it's an array with the computed loss after each iteration
    '''

    # We initialize an empty array to be filled with loss value after each iteration
    loss_history = np.zeros(iterations)

    # With a for loop we compute the steps of GD algo
    for it in range(iterations):
        theta = theta - alpha * CELoss_jacobian(theta, X, y_onehot)
        loss_history[it] = CELoss(theta, X, y_onehot)

    return theta, loss_history
```

################# Do not write below this line #################

```python
# Initialize a theta matrix with random parameters
theta0 = np.random.rand(X.shape[1], len(np.unique(y)))

print("Initial Loss with initialized theta is:", CELoss(theta0, X, y_onehot))

# Run Gradient Descent method
n_iter = 1000
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha=0.01, iterations=n_iter)
```

    Initial Loss with initialized theta is: 0.007010001299756874

```python
theta_final
```

    array([[ 0.76359559,  0.82177269,  0.84209622],
           [ 0.74017786,  0.61872511,  0.71074366],
           [ 0.23369714,  0.22120066,  0.85725487],
           [ 0.09755561, -0.20670453,  0.87969952],
           [ 0.68635542,  0.15081377,  0.19565598],
           [ 0.57770955,  0.55748856,  0.35534417],
           [ 0.76163969, -0.25674374,  0.98642803]])

```python
loss = CELoss(theta_final, X, y_onehot)
loss
```

    0.0010401024061862409

```python
fig,ax = plt.subplots(num=2)

ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

#### Exercise 3.c **(2 Points)**

Let's now evaluate the goodness of the learnt based on accuracy:

\begin{equation}
Accuracy = \frac{Number\ of\ correct\ predictions}{Total\ number\ of\ predictions}
\end{equation}

Implement the compute_accuracy function. You may compare the accuracy achieved with learnt model Vs. a random model (random $\Theta$) or one based on $\Theta$'s filled with zeros.

################# Do not write above this line #################

```python
def compute_accuracy(theta, X, y):
    '''
    Function to compute accuracy metrics of the softmax regression model.

    Input:
    theta: it's the final parameter matrix. The one we learned after all the iterations of the GD algorithm. The shape is (H, K)
    X: it's the input data matrix. The shape is (N, H)
    y: it's the label array. The shape is (N, 1)

    Output:
    accuracy: Score of the accuracy.
    '''
    # Call the softmax function
    s = softmax(theta, X)

    # Get the predicted classes
    predict = np.argmax(s, axis = 1)

    # Get the number of correct predictions comparing them with the label array
    # Divide by the total number of predictions (num of rows of softmax matrix)
    accuracy = sum(predict == y)/(float(len(s)))

    return accuracy
```

################# Do not write below this line #################

```python
compute_accuracy(theta_final, X, y)
```

    0.7933333333333333

```python
theta0 = np.random.rand(X.shape[1], len(np.unique(y)))
compute_accuracy(theta0, X, y)
```

    0.52

```python
compute_accuracy(np.zeros((X.shape[1], len(np.unique(y)))), X, y)
```

    0.3333333333333333

### Report **(3 Points)**

Experiment with different values for the learning rate $\alpha$ and the number of iterations. Look how the loss plot changes the convergence rate and the resulting accuracy metric. Report also execution time of each run. For this last step you could you %%time at the beginning of the cell to display time needed for the algorithm.

```python
%%time

# Initialize a theta matrix with random parameters
theta0 = np.random.rand(X.shape[1], len(np.unique(y)))

print("Initial Loss with initialized theta is:", CELoss(theta0, X, y_onehot))

# Run Gradient Descent method
n_iter = 100
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha=0.001, iterations=n_iter)
```

    Initial Loss with initialized theta is: 0.0008109861612985368
    CPU times: user 1.41 s, sys: 9.57 ms, total: 1.42 s
    Wall time: 1.43 s

**Write your Report here**

```python
# Initialize a theta matrix with random parameters
theta0 = np.random.rand(X.shape[1], len(np.unique(y)))
```

Learning rate $\alpha = 0.01$

Iterations = $100$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.01, iterations = 100)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.01, 'Iter:', 100)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.01$

Iterations = $500$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.01, iterations = 500)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.01, 'Iter:', 500)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.01$

Iterations = $1,000$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.01, iterations = 1000)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.01, 'Iter:', 1000)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.005$

Iterations = $100$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.005, iterations = 100)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.005, 'Iter:', 100)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.005$

Iterations = $500$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.005, iterations = 500)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.005, 'Iter:', 500)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.005$

Iterations = $1,000$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.005, iterations = 1000)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.005, 'Iter:', 1000)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.001$

Iterations = $100$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.001, iterations = 100)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.001, 'Iter:', 100)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.001$

Iterations = $500$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.001, iterations = 500)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.001, 'Iter:', 500)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.001$

Iterations = $1,000$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.001, iterations = 1000)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.001, 'Iter:', 1000)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

Learning rate $\alpha = 0.001$

Iterations = $10,000$

```python
%%time

# Run Gradient descent method
theta_final, log_l_history = gradient_descent(theta0, X, y_onehot, alpha = 0.001, iterations = 10000)

# Compute the accuracy
acc = compute_accuracy(theta_final, X, y)

print('LR:', 0.001, 'Iter:', 10000)
print('Accuracy:', acc)

# Loss plot
fig,ax = plt.subplots(num=2)
ax.set_ylabel('loss')
ax.set_xlabel('Iterations')
_=ax.plot(range(len(log_l_history)), log_l_history,'b.')
```

| LR | Iter | Accuracy | Time |
|---|---|---|---|
| 0.01 | 100 |0.666  | 1.36 s |
| 0.01 | 500 | 0.773 | 6.67 s |
| 0.01 |1,000 | 0.786 | 13.2 s |
| 0.005 | 100 | 0.526 | 1.36 s |
| 0.005 | 500 | 0.753 | 6.64 s |
| 0.005 | 1,000 | 0.773 | 13.2 s |
| 0.001 | 100 | 0.383 | 1.37 s |
| 0.001 | 500 | 0.526 | 6.62 s |
| 0.001 | 1,000 | 0.666 | 13.2 s |
| 0.001 | 10,000 | 0.786 | 2 min 11 s |

From data we can see that we get the best accuracy score ($0.786$) with $\alpha = 0.01$ e $1,000$ iterations.

We observe that with the same number of iterations, in terms of accuracy, we have better results with higher learning rates.
Infact, with lower learning rates we need more iterations to reach the minimum loss, as shown by the plots.
To conferm this hypothesis, we tried with $\alpha = 0.001$ and $10,000$ iterations and we get the same accuracy of $\alpha = 0.01$ e $1,000$ iterations. So, we can say that if we divide by $n$ our learning rate, to get the same accuracy level, we need to multiply our number of iterations by the same $n$.

In addition, it seems that with the same number of iterations, changing the learning rates doesn't affect so much the time needed for the execution of the algorithm. Obviously, the time depends mainly on the number of iterations and increase with it.

```python
# Set the learning rates
lr = ['0.01', '0.005', '0.001']

# Get the plot for each different number of iterations
iterations = [100, 500, 1000]
accuracy = np.asarray([[0.666, 0.526, 0.383],
                       [0.773, 0.753, 0.526],
                       [0.786, 0.773, 0.666]])

for i in range(len(iterations)):
    fig, ax = plt.subplots()
    b = ax.bar(lr, accuracy[i], width = 0.5)
    ax.set_ylabel('Accuracy')
    ax.set_xlabel('Learning Rates')
    ax.set_title(f'Iterations: {iterations[i]}')
    ax.grid(True, linewidth = 0.5)
    plt.show()
```

## Question 4: Multinomial Naive Bayes **(6 Points)**

### Code and Theory

The Naive Bayes classifier is a probabilistic machine learning model often used for classification tasks, e.g. document classification problems.

In the multinomial Naive Bayes classification you generally have $K>2$ classes, and the features are assumed to be generated from a multinomial distribution.

##### __*Example Data*__
General models consider input data as values. In the case of MultinomialNB, being used mainly in the field of document classification, these data consider how many features $X_i$ are present in the sample. Basically, it is a count of features within each document.

Taking into account $D=3$ documents and a vocabulary consisting of $N=4$ words, the data are considered as follows.

|  | $w_1$ | $w_2$ | $w_3$ | $w_4$ |
|---|---|---|---|---|
| $d_1$  | 3 | 0 | 1 | 1 |
| $d_2$ | 2 | 1 | 3 |0|
| $d_3$ | 2 | 2 | 0 |2|

By randomly generating the class to which each document belongs we have $y=[1,0,1]$

##### __*A bit of notation*__
- $Y =\{y_1, y_2, ... , y_{|Y|}\}$: set of classes

- $V =\{w_1, w_2, ... , w_{|V|}\}$: set of vocabulary

-  $D =\{d_1, d_2, ... , d_{|D|}\}$: set of documents

-  $N_{yi}$:  count of a specific word $w_i$ in each unique class, e.g. for $y=1$ you select $D_1$ and $D_3$, then for third column  you have $N_{y,3}=1$
-  $N_y$: total count of features for a specific class, e.g. for $y=1$ you sum all rows values which the correspondent label is 1, so $N_y=11$

-  $n$: total number of features (words in vocabulary)
-  $\alpha$: smoothing parameters

##### __*Task*__
Find the class $y$ to which the document is most likely to belong given the words $w$.
Use the Bayes formula and the posterior probability for it.

Bayes Formula:
\begin{equation}
P(A|B) = \frac{P(A)*P(B|A)}{P(B)}
\end{equation}

Where:
- P(A): Prior probability of A
- P(B): Prior probability of B
- P(B|A): Likelihood, multiplying posterior probability, that is multinomial Naive Bayes is:
\begin{equation}
P(B|A) = \left(\frac{N_{yi}+\alpha}{N_{y}+\alpha*n\_features}\right)^{X_{doc,i}}
\end{equation}

**Reminder: do not change any part of this notebook outside the assigned work spaces**

#### Generate random dataset

```python
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

X, y = make_classification(n_samples=300, n_features=7, n_informative=7, n_redundant=0, n_classes=3, random_state=1)
X = np.floor(X)-np.min(np.floor(X))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train.shape, y_train.shape
```

    ((240, 7), (240,))

#### Step0: $N_y$ and $N_{yi}$

```python
def feature_count(X, y, classes, n_classes, n_features):
        '''
        Function to compute the count of a specific word in each unique class and the total count.

        Input:
        X: it's the input data matrix.
        y: label array
        classes: unique values of y
        n_classes: number of classes
        n_features: it's the number of word in Vocabulary.

        Output:
        N_yi:   count of a specific word $w_i$ in each unique class
        N_y: total count of features for a specific class
        '''
        N_yi = np.zeros((n_classes, n_features)) # feature count
        N_y = np.zeros((n_classes)) # total count

        for i in range(len(y)):
            # Compute N_y counting the features for each specific class
            N_y[y[i]] += np.sum(X[i])

            # Compute N_yi adding counting the specific words in each class
            N_yi[y[i]] += (X[i])

        return N_yi, N_y
```

```python
n_samples_train, n_features = X_train.shape
classes = np.unique(y_train)
n_classes = 3
alpha = 0.1
N_yi, N_y = feature_count(X_train, y_train, classes, n_classes, n_features)
print(N_yi)
print(N_y)
```

    [[615. 599. 546. 541. 676. 701. 677.]
     [730. 729. 617. 642. 664. 776. 599.]
     [730. 735. 803. 806. 659. 813. 817.]]
    [4355. 4757. 5363.]

#### Step1: Prior Probability
The probability of a document being in a specific category from the given set of documents.

######################################

Your equations here:
\begin{equation}
P(y_j) = \frac{training-examples-of-the-j-class}{all-train-set}
\end{equation}

######################################

```python
def prior_(X, y, n_classes, n_samples):
        """
        Calculates prior for each unique class in y.

        Input:
        X: it's the input data matrix.
        y: label array
        n_classes: number of classes
        n_samples: number of documents

        Output:
        P: prior probability for each class. Shape: (, n_classes)
        """
        classes = np.unique(y)
        P = np.zeros(n_classes)

        # Implement Prior Probability P(A)
        for i in classes:
            P[i] = np.count_nonzero(y == i)/len(X)

        return P
```

```python
prior_prob = prior_(X_train, y_train, n_classes, n_samples_train)
print(prior_prob)
```

    [0.3        0.34583333 0.35416667]

#### Step2
Posterior Probability: The conditional probability of a word occurring in a document given that the document belongs to a particular category.

\begin{equation}
P(w_i|y_j) = \left(\frac{N_{yi}+\alpha}{N_{y}+\alpha*n\_features}\right)^{X_{doc,i}}
\end{equation}

Likelihood for a single document:
######################################

Your equations here:
\begin{equation}
P(w|y_j) = \prod_{i=1}^{n}\frac{N_{yi}+\alpha}{N_{y}+\alpha*n\_features}
\end{equation}

######################################

```python
def posterior_(x_i, i, h, N_y, N_yi, n_features, alpha):
        """
        Calculates posterior probability. aka P(w_i|y_j) using equation in the notebook.

        Input:
        x_i: feature x_i
        i: feature index.
        h: a class in y
        N_yi:   count of a specific word in each unique class
        N_y: total count of features for a specific class
        n_features: it's the number of word in Vocabulary.
        alpha: smoothing parameter

        Output:
        posterior: P(xi | y). Float.
        """

        # Implement Posterior Probability
        posterior = float(((N_yi[h][i] + alpha)/(N_y[h] + alpha * n_features))**x_i)

        return posterior

def likelihood_(x, h, N_y, N_yi, n_features, alpha):
        """
        Calculates Likelihood P(w|j_i).

        Input:
        x: a row of test data. Shape(n_features,)
        h: a class in y
        N_yi:   count of a specific word in each unique class
        N_y: total count of features for a specific class
        n_features: it's the number of word in Vocabulary.
        alpha: smoothing parameter

        Output:
        likelihood: Float.
        """

        tmp = []
        for i in range(x.shape[0]):
            tmp.append(posterior_(x[i], i, h, N_y, N_yi, n_features, alpha))

        # Implement Likelihood
        likelihood = float(np.prod(tmp))

        return likelihood
```

```python
print(posterior_(X_test[0][0], 1, 0, N_y, N_yi, n_features, alpha))
print(posterior_(X_test[0][1], 1, 0, N_y, N_yi, n_features, alpha))
print(posterior_(X_test[0][2], 1, 0, N_y, N_yi, n_features, alpha))
```

    1.761863411226497e-08
    1.761863411226497e-08
    2.423335789117236e-09

```python
# Example of likelihood for first document
likelihood_(X_test[0], 0, N_y, N_yi, n_features, alpha)
```

    2.7754694679413126e-53

#### Step3
Joint Likelihood that, given the words, the documents belongs to specific class
######################################

Your equations here:
\begin{equation}
P(y_i|w) = P(y_i)*P(w|y_i)
\end{equation}

######################################

Finally, from the probability that the document is in that class given the words, take the argument correspond to max value.

\begin{equation}
y(D) = argmax_{y \in Y} \frac{P(y|w)}{\sum_{j}P(y_j|w)}
\end{equation}

```python
def joint_likelihood(X, prior_prob, classes, n_classes, N_y, N_yi, n_features, alpha):
        """
        Calculates the joint probability P(y_i|w) for each class and makes it probability.
        Then take the argmax.

        Input:
        X: test data
        prior_prob:
        classes:
        n_classes:
        N_yi:   count of a specific word in each unique class
        N_y: total count of features for a specific class
        n_features: it's the number of word in Vocabulary.
        alpha: smoothing parameter

        Output:
        predicted_class: Predicted class of the documents. Int. Shape: (,#documents)
        """
        samples, features = X.shape
        predict_proba = np.zeros((samples,n_classes))

        # Calculate Joint Likelihood of each row for each class, then normalize in order to make them probabilities
        # Finally take the argmax to have the predicted class for each document

        for i in range(samples):
            for j in range(n_classes):
                l = likelihood_(X[i], j, N_y, N_yi, n_features, alpha)
                print(l)
                predict_proba[i][j] = (prior_prob[j]*l)
        print(predict_proba)

        predicted_class = []
        for i in range(samples):
            predicted_class.append(np.argmax(predict_proba[i]/np.sum(predict_proba[i])))

        return predicted_class
```

```python
yhat = joint_likelihood(X_test, prior_prob, classes, n_classes, N_y, N_yi, n_features, alpha)
```

#### Step4: Calculate the Accuracy Score

```python
print('Accuracy: ', np.round(accuracy_score(yhat, y_test),3))
```

    Accuracy:  0.717

**Sanity Check**

Here we use a function from the sklearn library, one of the most widely used in machine learning. MultinomialNB() implements the required algorithm, so the result of your implementation should be equal to the output of the following function.

```python
from sklearn import naive_bayes
clf = naive_bayes.MultinomialNB(alpha=0.1)
clf.fit(X_train,y_train)
sk_y = clf.predict(X_test)
print('Accuracy: ', np.round(accuracy_score(sk_y, y_test),3))
```

    Accuracy:  0.717


This notebook is used to generate data regarding the Goldbach conjecture: rows are of the form $N, a, b$ where $N$ is an even number, $a< b$ are prime and $N=a + b$.

First we write a Python function `goldbach` that takes `N` and returns all pairs of numbers $a, b$.

```python
import sympy as sym
import pandas as pd

def goldbach(N):
    """Returns all pairs of primes that sum to give N"""
    primes = list(sym.primerange(1, N))
    sums = []
    for i, p1 in enumerate(primes):
        for p2 in primes[i:]:
            if p1 + p2 == N:
                sums.append((p1, p2))
    return sums
```

Let us use the above function to create our data:

```python
maxN = 500
data = [[N, *pair] for N in range(4, maxN + 1)
        for pair in goldbach(N) if N % 2 == 0 ]
```

Let us write our data to an excel file:

```python
df = pd.DataFrame(data, columns=["N","a", "b"])  # Create a data frame
df.to_excel("data/goldbach.xlsx")  # Write it to excel
```


# Separation of Variables in 3D, waveguide

An infinitely long rectangluar metal pipe (sides $a$ and $b$) is grounded, but one end, at $x=0$ is maintained at a specified potential $V_0(y,z)$. Find the potential inside the pipe

This is a 3D problem and thus:

$$\frac{\partial^2V}{\partial x^2}+\frac{\partial^2V}{\partial y^2}+\frac{\partial^2V}{\partial z^2}=0 \label{eq:1}\tag{1}$$

Boundary conditions (Dirichlet):

1. $V=0$ when $y=0$
2. $V=0$ when $y=a$
3. $V=0$ when $z=0$
4. $V=0$ when $z=b$
5. $V\rightarrow0$ as $x\rightarrow \infty$
3. $V=V_0(y,z)$ when $x=0$

### Separation of variables: as products

$$V(x,y,z)=X(x)Y(y)Z(z)\tag{2}$$

$$\frac 1 X \frac{d^2X}{dx^2}+\frac 1 Y \frac{d^2Y}{dy^2}+\frac 1 Z \frac{d^2Z}{dz^2}=0\tag{3}$$

Three constants: $C_1$:positive, $C_2$:negative, $C_3$:negative

$$ C_2 = -k^2,
C_3 = -l^2,
C_1 = k^2 + l^2 $$

This turns a PDE in three ODEs which we can solve:

```python
from sympy.interactive import printing # use latex for printing
printing.init_printing(use_latex=True)
from sympy import Function, dsolve, Eq, Derivative, sin, cos, symbols # import necessary methods
from sympy.abc import x,y,z # import variables
```

```python
X = Function('X',real=True)
k, l = symbols('k l',positive=True)
f1_ode = Eq(Derivative(X(x), x, 2) - (k**2+l**2)*X(x))
f1_ode
```

```python
sol1 = dsolve(f1_ode, X(x))
sol1
```

```python
Y = Function('Y',real=True)
f2_ode = Eq(Derivative(Y(y), y, 2) + k**2*Y(y))
f2_ode
```

```python
sol2 = dsolve(f2_ode, Y(y))
sol2
```

```python
Z = Function('Z',real=True)
f3_ode = Eq(Derivative(Z(z), z, 2) + l**2*Z(z))
f3_ode
```

```python
sol3 = dsolve(f3_ode, Z(z))
sol3
```

From boundary conditions (except (6)) we get:

$$V(x,y,z)=C\textrm{e}^{-\pi \sqrt{(n/a)^2+(m/b)^2}x} \sin{(n\pi y/a)} \sin{(m\pi z/b)}$$

with

$$ k = \frac{n\pi}{a},\ l=\frac{m\pi}{b}$$

which we can use to define a double sum for all possible $V_0(y,z)$

$$ V(x,y,z)= \sum_{n=1}^{\infty}\sum_{m=1}^{\infty} C_{n,m}\textrm{e}^{-\pi \sqrt{(n/a)^2+(m/b)^2}x} \sin{(n\pi y/a)} \sin{(m\pi z/b)} $$

and get the coefficients $C_{n,m}$ from BC (6)

$$ V(0,y,z)= \sum_{n=1}^{\infty}\sum_{m=1}^{\infty} C_{n,m} \underbrace{\textrm{e}^{-\pi \sqrt{(n/a)^2+(m/b)^2}x}}_{=1 \textrm{ for }x=0} \sin{(n\pi y/a)} \sin{(m\pi z/b)} = V_0(y,z)$$

using the same approach again and multiplying with:

$$ \sin{(n'\pi y/a)} \sin{(m'\pi z/b)}$$

and integrating from 0 to $a$ and 0 to $b$:

$$ \sum_{n=1}^{\infty}\sum_{m=1}^{\infty} C_{n,m} \int_0^a \sin{(n\pi y/a)} \sin{(n'\pi y/b)}\,dy \int_0^b \sin{(m\pi z/a)} \sin{(m'\pi z/b)}\,dz  = \int_0^a \int_0^b V_0 \sin{(n'\pi y/a)} \sin{(m'\pi z/b)}\,dydz$$

solving the integrals on the left side:

```python
from sympy import integrate, pi
n, m, k, l = symbols('n m k l', positive = True, integer=True)
a, b = symbols('a b', constant = True)
integrate(integrate(sin(n*pi*y/a)*sin(m*pi*y/a)*sin(k*pi*z/b)*sin(l*pi*z/b), (y, 0, a)), (z, 0, b)) # var y, from 0 to a
```

we get for $C_{n,m}$:

$$ C_{n,m} = \frac{4}{ab} \int_0^a \int_0^b V_0(y,z) \sin{(n\pi y/a)} \sin{(m\pi z/b)}\,dydz$$

Example: $V_0(y,z)$ is constant:

$$ C_{n,m} = \frac{4V_0}{ab} \int_0^a \int_0^b \sin{(n\pi y/a)} \sin{(m\pi z/b)}\,dydz = \begin{cases}
    0, & \text{if $n$ or $m$ is even}.\\
    \frac{16V_0}{nm\pi^2}, & \text{if $n$ is odd}.
\end{cases}$$

and thus:

$$ V(x,y,z)= \frac{16 V_0}{\pi^2} \sum_{n,m=1,3,5,\ldots}^{\infty} \frac{1}{nm}\textrm{e}^{-\pi \sqrt{(n/a)^2+(m/b)^2}x} \sin{(n\pi y/a)} \sin{(m\pi z/b)} $$

```python
%matplotlib notebook
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(figsize=(5.,5.))
ax = fig.gca(projection='3d')

n=1
m=1
a=1
b=1
V0 = 1
x = np.arange(0,1.01,0.01)  # range for x
y = np.arange(0,a+0.01,0.01)  # range for y
#z = np.arange(0,b+0.01,0.01)
X,Y = np.meshgrid(x,y)  # generate 2D mesh from x and y
Z=.1
V=(16*V0/np.pi**2)*(1/(n*m)*np.exp(-np.pi*np.sqrt((n/a)**2+(m/b)**2)*x)*np.sin(n*np.pi*Y/a)*np.sin(m*np.pi*Z/b))  # define V(x,y) on the mesh X,Y

# Plot a crossection
cset = ax.contour3D(X, Y, V, zdir='z', offset=.1, cmap='Blues',levels=64) # lineplot on the xz plane
ann = ax.text(.7,.8,1, "n={}".format(n), color='k',fontsize=24)
```

    <IPython.core.display.Javascript object>

```python

```

Separation of Variables in 3D, waveguide.ipynb


```python
import sympy as sp
import numpy as np

sp.init_printing()
```

```python
xi, L, A, E = sp.symbols("xi L A E")
Nue = sp.Matrix([(1-xi)/2, (1+xi)/2])
Nue
```

```python
Bue=sp.diff(Nue,xi)*2/L
Bue
```

```python
KeTRUSS1D = sp.lambdify((A,E,L),A*E*sp.integrate(Bue*(Bue.T),(xi,-1,1))*L/2)
print(KeTRUSS1D(A,E,L))
```

    [[A*E/L -A*E/L]
     [-A*E/L A*E/L]]

Adatok megadása

```python
A1=50
A2=20

E1=100e3
E2=200e3

L=2e3

FT=60e3
```

Elem-csomópont összerendelések tárolása az `en` mátrixban

```python
en=np.array([
    [1,3],
    [2,3],
    [3,4],
    [3,5]]) - 1
```

Elemi merevségi mátrixok

```python
Ke1 = KeTRUSS1D(A1,E1,L)
Ke2 = KeTRUSS1D(A1,E1,L)
Ke3 = KeTRUSS1D(A1,E1,L)
Ke4 = KeTRUSS1D(A2,E2,2*L)
```

Globális merevségi mátrix megadása során első lépésben egy zérus elemekkel
kitöltött mátrixot hozunk létre, majd a megfelelő helyekre betesszül az
egyes elemem merevségi mátrixainak elemeit.

```python
KG=np.zeros((5,5))
KG
```

    array([[ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.]])

Egy lehetséges leprogramozása ennek az alábbiakban látható,
ahol felhasználjuk az elem-csomópont összerendlés mátrixot:

[np.ix_](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ix_.html)

Az 1-es elem merevségi mátrixának elhelyezése
a globális merevségi mátrixban:

```python
elemSzam=1
KG[np.ix_(en[elemSzam-1],en[elemSzam-1])] += Ke1
KG
```

    array([[ 2500.,     0., -2500.,     0.,     0.],
           [    0.,     0.,     0.,     0.,     0.],
           [-2500.,     0.,  2500.,     0.,     0.],
           [    0.,     0.,     0.,     0.,     0.],
           [    0.,     0.,     0.,     0.,     0.]])

Az 2-es elem merevségi mátrixának elhelyezése
a globális merevségi mátrixban:

```python
elemSzam=2
KG[np.ix_(en[elemSzam-1],en[elemSzam-1])] += Ke2
KG
```

    array([[ 2500.,     0., -2500.,     0.,     0.],
           [    0.,  2500., -2500.,     0.,     0.],
           [-2500., -2500.,  5000.,     0.,     0.],
           [    0.,     0.,     0.,     0.,     0.],
           [    0.,     0.,     0.,     0.,     0.]])

Az 3-es elem merevségi mátrixának elhelyezése
a globális merevségi mátrixban:

```python
elemSzam=3
KG[np.ix_(en[elemSzam-1],en[elemSzam-1])] += Ke3
KG
```

    array([[ 2500.,     0., -2500.,     0.,     0.],
           [    0.,  2500., -2500.,     0.,     0.],
           [-2500., -2500.,  7500., -2500.,     0.],
           [    0.,     0., -2500.,  2500.,     0.],
           [    0.,     0.,     0.,     0.,     0.]])

Az 4-es elem merevségi mátrixának elhelyezése
a globális merevségi mátrixban:

```python
elemSzam=4
KG[np.ix_(en[elemSzam-1],en[elemSzam-1])] += Ke4
KG
```

    array([[ 2500.,     0., -2500.,     0.,     0.],
           [    0.,  2500., -2500.,     0.,     0.],
           [-2500., -2500.,  8500., -2500., -1000.],
           [    0.,     0., -2500.,  2500.,     0.],
           [    0.,     0., -1000.,     0.,  1000.]])

Tehát a globális merevségi mátrix:

```python
KG
```

    array([[ 2500.,     0., -2500.,     0.,     0.],
           [    0.,  2500., -2500.,     0.,     0.],
           [-2500., -2500.,  8500., -2500., -1000.],
           [    0.,     0., -2500.,  2500.,     0.],
           [    0.,     0., -1000.,     0.,  1000.]])

A globális tehervektor:

```python
FG = np.zeros(5)
FG[2 - 1] += FT
```

A kondenzált merevségi mátrixot és a kondenzált tehervektort megkapjuk a kényszerekkel ellátott szabadságfokokhoz tartozó sorok és oszlopok törlésével:

(megtartjuk a maradék részt)

```python
szabad = [1,2]
```

A kondenzált merevségi mátrixot megkapjuk az **első** és **második** sorok/oszlopok megtartásával:

```python
print(np.ix_(szabad,szabad))
```

    (array([[1],
           [2]]), array([[1, 2]]))

```python
KGkond = KG[np.ix_(szabad,szabad)]
KGkond
```

    array([[ 2500., -2500.],
           [-2500.,  8500.]])

A kondezált tehervektort megkapjuk az **első** és **második** sor megtartásával:

```python
FGkond = FG[np.ix_(szabad)]
FGkond
```

    array([ 60000.,      0.])

```python
# a teljes lineáris egyenletrendszer alulhatározott (~0-val osztás)
np.linalg.solve(KG,FG)
```

    array([ -3.10899383e+17,  -3.10899383e+17,  -3.10899383e+17,
            -3.10899383e+17,  -3.10899383e+17])

Megoldás az ismeretlen elmozdulásokra

```python
Umego = np.linalg.solve(KGkond,FGkond)
Umego
```

    array([ 34.,  10.])

Tehát a teljes globális csomóponti elmozdulásvektor:

```python
UG = np.zeros(5)
UG[np.ix_(szabad)] += Umego
UG
```

    array([  0.,  34.,  10.,   0.,   0.])

Eredmények megjelenítése

[matplotlib.patches](https://matplotlib.org/api/patches_api.html)

```python
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
```

```python
nagy = 50
R = 300

plt.figure()
ax = plt.gca()
ax.set_xticklabels([])
ax.set_yticklabels([])
## eredeti konfiguráció
# rudak
ax.add_patch(mpatches.Rectangle((-L-R/2,-L),R,L,
                                fc = (1,1,1,1),ec = (0.5,0.5,0.5,1), ls = "--", lw = 2))
ax.add_patch(mpatches.Rectangle((-L-R/2,-2*L),R,L,
                                fc = (1,1,1,1),ec = (0.5,0.5,0.5,1), ls = "--", lw = 2))
ax.add_patch(mpatches.Rectangle((L-R/2,-L),R,L,
                                fc = (1,1,1,1),ec = (0.5,0.5,0.5,1), ls = "--", lw = 2))
ax.add_patch(mpatches.Rectangle((L-R/2,-3*L),R,2*L,
                                fc = (1,1,1,1),ec = (0.5,0.5,0.5,1), ls = "--", lw = 2))
# # csomópontok
ax.add_patch(mpatches.Circle((L,0),R,
                             fc = (1,1,1,1),ec = (0.5,0.5,0.5,1),ls = "--", lw = 2))
ax.add_patch(mpatches.Rectangle((-1.5*L,-L - R/2), 3*L, R,
                                fc = (1,1,1,1),ec = (0.5,0.5,0.5,1),ls = "--", lw = 2))

## elmozdult állapot
# rudak
ax.add_patch(mpatches.Rectangle((-L-R/2,-L + nagy*UG[2]),R,L + nagy*(UG[0] - UG[2]),
                                fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Rectangle((-L-R/2,-2*L + nagy*UG[3]),R,L + nagy*(UG[2] - UG[3]),
                                fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Rectangle((L-R/2,-L + nagy*UG[2]),R,L + nagy*(UG[1] - UG[2]),
                                fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Rectangle((L-R/2,-L -2*L + nagy*UG[4]),R,2*L + nagy*(UG[2] - UG[4]),
                                fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))

# csomópontok
ax.add_patch(mpatches.Circle((-L,0+nagy*UG[0]),R,
                             fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Circle((L,nagy*UG[1]),R,
                             fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Rectangle((-1.5*L,-L - R/2 + nagy*UG[2]),3*L,R,
                                fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Circle((-L,-L-L),R,
                             fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
ax.add_patch(mpatches.Circle((L,-L - 2*L),R,
                             fc = (1,1,1,1),ec = (0,0,0,1), lw = 2))
# terhelés
ax.add_patch(mpatches.Arrow(L,nagy*UG[1],0,3*R,
                            fc = (1,0,0,1),ec = (1,0,0,1), lw = 1, width = 2*R))
plt.axis("equal")
plt.show()
```

A teljes csomóponti terhelésvektor
$\mathbf{F}_\mathrm{TOT} = \mathbf{F}_\mathrm{REAK} + \mathbf{F}_\mathrm{KÜL}$:

```python
FTOT = np.dot(KG,UG)
FTOT
```

    array([-25000.,  60000.,      0., -25000., -10000.])

```python
FREAK = FTOT - FG
FREAK
```

    array([-25000.,      0.,      0., -25000., -10000.])

Az egyes elemekhez tartozó lokális elmozdulásvektorok:

```python
Ue1 = UG[en[0]]
Ue1
```

    array([  0.,  10.])

```python
Ue2 = UG[en[1]]
Ue2
```

    array([ 34.,  10.])

```python
Ue3 = UG[en[2]]
Ue3
```

    array([ 10.,   0.])

```python
Ue4 = UG[en[3]]
Ue4
```

    array([ 10.,   0.])

Az egyes elemekhez tartozó lokális tehervektor:

```python
Fe1 = np.dot(Ke1,Ue1)
Fe1
```

    array([-25000.,  25000.])

```python
Fe2 = np.dot(Ke2,Ue2)
Fe2
```

    array([ 60000., -60000.])

```python
Fe3 = np.dot(Ke3,Ue3)
Fe3
```

    array([ 25000., -25000.])

```python
Fe4 = np.dot(Ke4,Ue4)
Fe4
```

    array([ 10000., -10000.])

Tehát a normál igénybevételek:

```python
N1 = Fe1[0]
N1
```

```python
N2 = Fe2[0]
N2
```

```python
N3 = Fe3[0]
N3
```

```python
N4 = Fe4[0]
N4
```

Vagyis mindegyik rúd nyomó igénybevétel alatt van.

A rudakban ébredő feszültség:

```python
sig1 = N1/A1
sig1
```

```python
sig2 = N2/A1
sig2
```

```python
sig3 = N3/A1
sig3
```

```python
sig4 = N3/A2
sig4
```

A rudak alapváltozásai:

```python
eps1 = sig1/E1
eps1
```

```python
eps2 = sig2/E1
eps2
```

```python
eps3 = sig3/E1
eps3
```

```python
eps4 = sig4/E2
eps4
```


```python
%matplotlib notebook
```

```python
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
from matplotlib.ticker import ScalarFormatter
import math
```

This notebook assumes you have completed the notebook [Introduction of sine waves](TDS_Introduction-sine_waves.ipynb). This notebook follows the same pattern of time domain waveform generation: instantaneous frequency -> phase step -> total phase -> time domain waveform.

Our goal is to track features of different acoustic impedance in material using a low power time domain waveform. Time delay spectrometry (TDS) is one implementation of this goal. To understand TDS we need to understand the waveform which is used by TDS called a chirp. A chirp is a sinusoid that is constantly varying in frequency. The chirp is generated by integrating a varying phase step which is derived from an instantaneous frequency profile. We will generate a chirp in this notebook.
The phase of the chirp can be found by integrating the instantaneous frequency:

\begin{equation}
	f(t)=\frac{f_{end}-f_{start}}{T_c}t + f_{start}
\end{equation}

\begin{equation}
\Delta\phi(t) = 2\pi f(t)\Delta t
\end{equation}

\begin{equation}
\phi (t)=\int_{}^{} \Delta\phi(t) = \int_{}^{} 2\pi f(t) dt = \int_{}^{}\frac{f_{end}-f_{start}}{T_c}tdt + \int_{}^{}f_{start}dt
\end{equation}

\begin{equation}
 \phi (t)= \frac{f_{end}-f_{start}}{T_c}\int_{}^{}tdt + f_{start}\int_{}^{}dt
\end{equation}

\begin{equation}
 \phi (t)= \frac{f_{end}-f_{start}}{T_c}\frac{t^2}{2} + f_{start}t
\end{equation}

This gives the time series value of

\begin{equation}
x(t) = e^{j\phi (t)} = e^{j(\frac{f_{end}-f_{start}}{T_c}\frac{t^2}{2} + f_{start}t)}
\end{equation}

But the formula for phase requires squaring time which will cause numeric errors as the time increases. Another approach is to implement the formula for phase as a cummulative summation.

\begin{equation}
\phi_{sum} (N)=\sum_{k=1}^{N} \Delta\phi(k) = \sum_{k=1}^{N} 2\pi f(k) t_s = \sum_{k=1}^{N}(\frac{f_{end}-f_{start}}{T_c}k + f_{start})t_s
\end{equation}

This allow for the phase always stay between 0 and two pi by subtracting two phi whenever the phase exceeds the value. We will work with the cummlative sum of phase, but then compare it to the integral to determine how accurate the cummulative sum is.

```python
speed_of_ultrasound_m_per_sec={ 'tissue':1540, #In most tissue
                                'water':1481,
                                'air': 343
                              }
```

```python
#Try out differment chirp paramters to determine which would produce the best seperation of distances for the gel pads
c_m_per_sec = speed_of_ultrasound_m_per_sec['water']

f_start_Hz=200_000
#talk about difference and similarity of sine wave example, answer why not 32 samples
f_stop_Hz=1_800_000

for Tc_sec in [0.01, 0.1, 1.0, 10.0]:
    S_Hz_per_sec = (f_stop_Hz - f_start_Hz)/Tc_sec
    d_m = 0.015 #1 cm
    tau_sec = 2 *d_m/c_m_per_sec
    f_location_Hz = S_Hz_per_sec * tau_sec
    print(f"Tc = {Tc_sec}. For a target at {d_m*100} cm the expected freq is {f_location_Hz} Hz with a time delay of {tau_sec} sec")

```

    Tc = 0.01. For a target at 1.5 cm the expected freq is 3241.053342336259 Hz with a time delay of 2.025658338960162e-05 sec
    Tc = 0.1. For a target at 1.5 cm the expected freq is 324.1053342336259 Hz with a time delay of 2.025658338960162e-05 sec
    Tc = 1.0. For a target at 1.5 cm the expected freq is 32.41053342336259 Hz with a time delay of 2.025658338960162e-05 sec
    Tc = 10.0. For a target at 1.5 cm the expected freq is 3.241053342336259 Hz with a time delay of 2.025658338960162e-05 sec

```python
#Tc is the max depth we are interested in
Tc_sec=1.0

f_start_Hz=200_000
#talk about difference and similarity of sine wave example, answer why not 32 samples
f_stop_Hz=1_800_000

#sample rate
fs=2_000_000
ts=1/fs

total_samples= math.ceil(fs*Tc_sec)
n = np.arange(0,total_samples, step=1, dtype=np.float64)
t_sec=n*ts
t_usec = t_sec *1e6

#This is the frequency of the chirp over time. We assume linear change in frequency
chirp_freq_slope_HzPerSec=(f_stop_Hz-f_start_Hz)/Tc_sec

#Compute the instantaneous frequency which is a linear function
chirp_instantaneous_freq_Hz=chirp_freq_slope_HzPerSec*t_sec+f_start_Hz
chirp_instantaneous_angular_freq_radPerSec=2*np.pi*chirp_instantaneous_freq_Hz

#Since frequency is a change in phase the we can plot it as a phase step
chirp_phase_step_rad=chirp_instantaneous_angular_freq_radPerSec*ts

#The phase step can be summed (or integrated) to produce the total phase which is the phase value
#for each point in time for the chirp function
chirp_phase_rad=np.cumsum(chirp_phase_step_rad)
#The time domain chirp function
chirp = np.exp(1j*chirp_phase_rad)
```

```python
#We can see, unlike the complex exponential, the chirp's instantaneous frequency is linearly increasing.
#This corresponds with the linearly increasing phase step.
fig, ax = plt.subplots(2, 1, sharex=True,figsize = [8, 8])
lns1=ax[0].plot(t_usec,chirp_instantaneous_freq_Hz,linewidth=4, label='instantanous frequency');
ax[0].set_title('Comparing the instantaneous frequency and phase step')
ax[0].set_ylabel('instantaneous frequency (Hz)')

axt = ax[0].twinx()
lns2=axt.plot(t_usec,chirp_phase_step_rad,linewidth=2,color='black', linestyle=':', label='phase step');
axt.set_ylabel('phase step (rad)')

#ref: https://stackoverflow.com/questions/5484922/secondary-axis-with-twinx-how-to-add-to-legend
lns = lns1+lns2
labs = [l.get_label() for l in lns]
ax[0].legend(lns, labs, loc=0)

#We see that summing or integrating the linearly increasing phase step gives a quadratic function of total phase.
ax[1].plot(t_usec,chirp_phase_rad,linewidth=4,label='chirp');
ax[1].plot([t_usec[0], t_usec[-1]],[chirp_phase_rad[0], chirp_phase_rad[-1]],linewidth=1, linestyle=':',label='linear (x=y)');
ax[1].set_title('Cumulative quandratic phase function of chirp')
ax[1].set_xlabel('time ($\mu$sec)')
ax[1].set_ylabel('total phase (rad)')
ax[1].legend();

```

    <IPython.core.display.Javascript object>

```python
print(f'chirp_phase_step_rad[0] = {chirp_phase_step_rad[0]} rad and the end is chirp_phase_step_rad[-1] = {chirp_phase_step_rad[-1]}')
print(f'amplitude = {chirp_phase_step_rad[-1] - chirp_phase_step_rad[0]} rad')
print(f'offset = {(chirp_phase_step_rad[-1] - chirp_phase_step_rad[0])/2+chirp_phase_step_rad[0]} rad')
```

    chirp_phase_step_rad[0] = 0.6283185307179585 rad and the end is chirp_phase_step_rad[-1] = 5.654864263187504
    amplitude = 5.026545732469545 rad
    offset = 3.141591396952731 rad

```python
#We now what to load this into GNURadio. So we will only write the raw file and then load that into GNURadio. This will cause the array size to be lost
chirp.astype('complex64').tofile(f'../data/make_chirp_for_GNURadio_conf_fs_{int(fs)}_Tc_sec_{Tc_sec}_complex64.bin')
print(f'The sample rate is {int(fs)}')
```

    The sample rate is 2000000


# Hello world for linear regression in PyStan

This is meant to be a first notebook for exploring how to use PyStan to do:

* MAP estimation
* MCMC sampling using the No-U-Turn (NUTS) extension of Hamlitonian Monte Carlo

Meant to allow side-by-side comparison with alternative probabilistic programming packages like PyMC3.

Inspired by:

* Bob Carpenter's blog post:

https://statmodeling.stat.columbia.edu/2017/05/31/compare-stan-pymc3-edward-hello-world/

* The "hello world" example in the PyMC3 journal article:

Salvatier, Wiecki, and Fonnesbeck. "Probabilistic programming in Python using PyMC3".
https://peerj.com/articles/cs-55.pdf

```python
import numpy as np
import pandas as pd
```

```python
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style('whitegrid');
sns.set_context("notebook", font_scale=1.5, rc={"lines.linewidth": 2.0})

%matplotlib inline
```

# Import PyStan

If you need to install it first, using the `spr_2020s_env` conda environment just do:

```console
$ conda activate spr_2020s_env
$ conda install -c conda-forge pystan
```

```python
import pystan
```

# Model as MATH

Likelihood:
\begin{align}
 p(y_{1:N} | x_{1:N}, \alpha, \beta, \sigma) &= \text{NormalPDF}(\alpha + \beta_1 x_{n1} + \beta_2 x_{n2}, \sigma^2)
\end{align}

Priors on the variance of the likelihood:
\begin{align}
 p(\sigma) &= \text{HalfNormalPDF}(\sigma | 0, 1) =
 \begin{cases}
     2 \cdot \text{NormalPDF}(\sigma | 0, 1) & \sigma > 0
\\
     0.0  & \sigma \leq 0
 \end{cases}
\end{align}

Priors on regression coefficients:
\begin{align}
 p(\alpha) &= \text{NormalPDF}(0, 10)
\\
 p(\beta_1) &= \text{NormalPDF}(0, 10)
\\
 p(\beta_2) &= \text{NormalPDF}(0, 10)
\\
\end{align}

# Simulate data from the model

```python
# Initialize random number generator
np.random.seed(123)

# True regression coefficients
alpha = 1.0
beta = np.asarray([1.0, 1.337])

# True regression std dev
sigma = 0.3

# Size of dataset
N = 100

# Data Features (input variable)
x1_N = np.random.randn(N)
x2_N = np.random.randn(N)
x_N2 = np.hstack([x1_N[:,np.newaxis], x2_N[:,np.newaxis]])

# Simulate outcome variable
y_N = alpha + beta[0]*x1_N + beta[1]*x2_N + np.random.randn(N) * sigma

true_params = dict(
    alpha=alpha,
    beta=beta,
    sigma=sigma)
```

# Plot of observed data

```python
_, axgrid = plt.subplots(nrows=1, ncols=2, figsize=(8,3), sharex=True, sharey=True);

axgrid[0].plot(x1_N, y_N, 'k.'); axgrid[0].set_xlabel('x1'); axgrid[0].set_ylabel('y');
axgrid[1].plot(x2_N, y_N, 'b.'); axgrid[1].set_xlabel('x2'); axgrid[1].set_ylabel('y');
```

# Model specified in Stan

```python
model_code = '''
data {
  int N;
  vector[N] y;
  matrix[N, 2] x;
}
parameters {
  real alpha;
  vector[2] beta;
  real<lower=0> sigma; // constrain sigma > 0
}
model {
  alpha ~ normal(0, 10);
  beta ~ normal(0, 10);  // implicitly applied to each dim of vector beta
  sigma ~ normal(0, 1);  // implicitly takes into account the constraint: sigma > 0

  y ~ normal(alpha + x * beta, sigma);
}
'''
```

```python
# Create dictionary to specify concrete values
# for each parameter named in 'data' section of model

data_N10 = dict(N=10, y=y_N[:10], x=x_N2[:10])
```

```python
data_N100 = dict(N=100, y=y_N[:100], x=x_N2[:100])
```

# Setup: Compile the model

```python
# Compile the model
stan_model = pystan.StanModel(model_code=model_code)
```

    INFO:pystan:COMPILING THE C++ CODE FOR MODEL anon_model_9f0db3f7b821fb462af2f1e2557f82cf NOW.

# Setup: Prepare for plotting results

```python
def plot_traces(fit):
    fig, ax_grid = plt.subplots(nrows=2, ncols=2, sharex=True, sharey=True);
    for ax, var_name, dim in zip(
            ax_grid.flatten(), ['alpha', 'sigma', 'beta', 'beta'], [None, None, 0, 1]):
        var_samples_S = fit[var_name]
        true_val = true_params[var_name]

        if dim is not None:
            var_samples_S = var_samples_S[:, dim]
            true_val = true_val[dim]
            var_name = var_name + "[%d]" % dim

        S = var_samples_S.size
        ax.plot(var_samples_S, 'b.', markersize=4, alpha=0.3, label='samples')
        ax.plot(true_val * np.ones(S), 'r-', label='true_value')
        ax.set_ylabel(var_name)

    ax.legend(bbox_to_anchor=(1.1, 1.0))
    ax.set_ylim([0, 2.0]);
```

# Run MAP estimator for N=10 dataset

```python
map_N10 = stan_model.optimizing(
    data=data_N10,
    algorithm='LBFGS',
    init='0',
    verbose=True,
    iter=1000, seed=101)
```

```python
for key, arr in map_N10.items():
    print(key, arr)
```

    alpha 1.0693495791082472
    beta [1.00373535 1.44519637]
    sigma 0.2004478445079128

```python
for key, arr in true_params.items():
    print(key, arr)
```

    alpha 1.0
    beta [1.    1.337]
    sigma 0.3

# Run MAP estimator for N=100 dataset

```python
map_N100 = stan_model.optimizing(
    data=data_N100,
    algorithm='LBFGS',
    init='0',
    verbose=True,
    iter=1000, seed=101)
```

```python
for key, arr in map_N10.items():
    print(key, arr)
```

    alpha 1.0693495791082472
    beta [1.00373535 1.44519637]
    sigma 0.2004478445079128

# Run MCMC sampler for N=10 dataset

```python
fit_N10 = stan_model.sampling(
    data=data_N10,
    algorithm='NUTS',
    init='0',
    verbose=True,
    iter=1000, chains=1, warmup=500, thin=1, seed=101)
```

```python
plot_traces(fit_N10)
```

# Run MCMC sampler for N=100 dataset

```python
fit_N100 = stan_model.sampling(
    data=data_N100,
    algorithm='NUTS',
    init='0',
    verbose=True,
    iter=1000, chains=1, warmup=500, thin=1, seed=101)
```

```python
plot_traces(fit_N100)
```

```python

```


## GeostatsPy: Variogram Calculation for Subsurface Data Analytics in Python

### Michael Pyrcz, Associate Professor, University of Texas at Austin

##### [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)

### PGE 383 Exercise: Basic Univariate Summary Statistics and Data Distribution Representativity Plotting in Python with GeostatsPy

Here's a simple workflow with some basic variogram calcuation with irregularly sampled data. This should help you get started with variogram caculation in subsurface modeling.

#### Spatial Continuity

**Spatial Continuity** is the correlation between values over distance.

* No spatial continuity – no correlation between values over distance, random values at each location in space regardless of separation distance.

* Homogenous phenomenon have perfect spatial continuity, since all values as the same (or very similar) they are correlated.

We need a statistic to quantify spatial continuity! A convenient method is the Semivariogram.

#### The Semivariogram

Function of difference over distance.

* The expected (average) squared difference between values separated by a lag distance vector (distance and direction), $h$:

\begin{equation}
\gamma(\bf{h}) = \frac{1}{2 N(\bf{h})} \sum^{N(\bf{h})}_{\alpha=1} (z(\bf{u}_\alpha) - z(\bf{u}_\alpha + \bf{h}))^2
\end{equation}

where $z(\bf{u}_\alpha)$ and $z(\bf{u}_\alpha + \bf{h})$ are the spatial sample values at tail and head locations of the lag vector respectively.

* Calculated over a suite of lag distances to obtain a continuous function.

* the $\frac{1}{2}$ term converts a variogram into a semivariogram, but in practice the term variogram is used instead of semivariogram.
* We prefer the semivariogram because it relates directly to the covariance function, $C_x(\bf{h})$ and univariate variance, $\sigma^2_x$:

\begin{equation}
C_x(\bf{h}) = \sigma^2_x - \gamma(\bf{h})
\end{equation}

Note the correlogram is related to the covariance function as:

\begin{equation}
\rho_x(\bf{h}) = \frac{C_x(\bf{h})}{\sigma^2_x}
\end{equation}

The correlogram provides of function of the $\bf{h}-\bf{h}$ scatter plot correlation vs. lag offset $\bf{h}$.

\begin{equation}
-1.0 \le \rho_x(\bf{h}) \le 1.0
\end{equation}

#### Variogram Observations

The following are common observations for variograms that should assist with their practical use.

##### Observation \#1 - As distance increases, variability increase (in general).

This is common since in general, over greater distance offsets, there is often more difference between the head and tail samples.

In some cases, such as with spatial cyclicity of the hole effect variogram model the variogram may have negative slope over somelag distance intervals

Negative slopes at lag distances greater than half the data extent are often caused by too few pairs for a reliable variogram calculation

##### Observation \#2 - Calculated with over all possible pairs separated by lag vector, $\bf{𝐡}$.

We scan through the entire data set, searching for all possible pair combinations with all other data.  We then calculate the variogram as one half the expectation of squared difference between all pairs.

More pairs results in a more reliable measure.

##### Observation \#3 - Need to plot the sill to know the degree of correlation.

**Sill** is the variance, $\sigma^2_x$

Given stationarity of the variance, $\sigma^2_x$, and variogram $\gamma(\bf{h})$:

we can define the covariance function:

\begin{equation}
C_x(\bf{h}) = \sigma^2_x - \gamma(\bf{h})
\end{equation}

The covariance measure is a measure of similarity over distance (the mirror image of the variogram as shown by the equation above).

Given a standardized distribution $\sigma^2_x = 1.0$, the covariance, $C_x(\bf{h})$, is equal to the correlogram, $\rho_x(\bf{h})$:

\begin{equation}
\rho_x(\bf{h}) = \sigma^2_x - \gamma(\bf{h})
\end{equation}

##### Observation \#4 - The lag distance at which the variogram reaches the sill is know as the range.

At the range, knowing the data value at the tail location provides no information about a value at the head location of the lag distance vector.

##### Observation \#5 - The nugget effect, a discontinuity at the origin

Sometimes there is a discontinuity in the variogram at distances less than the minimum data spacing.  This is known as **nugget effect**.

The ratio of nugget / sill, is known as relative nugget effect (%). Modeled as a discontinuity with no correlation structure that at lags, $h \gt \epsilon$, an infinitesimal lag distance, and perfect correlation at $\bf{h} = 0$.
Caution when including nuggect effect in the variogram model as measurement error, mixing populations cause apparent nugget effect

This exercise demonstrates the semivariogram calculation with GeostatsPy. The steps include:

1. generate a 2D model with sequential Gaussian simulation
2. sample from the simulation
3. calculate and visualize experimental semivariograms

#### Objective

In the PGE 383: Stochastic Subsurface Modeling class I want to provide hands-on experience with building subsurface modeling workflows. Python provides an excellent vehicle to accomplish this. I have coded a package called GeostatsPy with GSLIB: Geostatistical Library (Deutsch and Journel, 1998) functionality that provides basic building blocks for building subsurface modeling workflows.

The objective is to remove the hurdles of subsurface modeling workflow construction by providing building blocks and sufficient examples. This is not a coding class per se, but we need the ability to 'script' workflows working with numerical methods.

#### Getting Started

Here's the steps to get setup in Python with the GeostatsPy package:

1. Install Anaconda 3 on your machine (https://www.anaconda.com/download/).
2. From Anaconda Navigator (within Anaconda3 group), go to the environment tab, click on base (root) green arrow and open a terminal.
3. In the terminal type: pip install geostatspy.
4. Open Jupyter and in the top block get started by copy and pasting the code block below from this Jupyter Notebook to start using the geostatspy functionality.

You will need to copy the data file to your working directory.  They are available here:

* Tabular data - sample_data.csv at https://git.io/fh4gm.

There are exampled below with these functions. You can go here to see a list of the available functions, https://git.io/fh4eX, other example workflows and source code.

#### Load the required libraries

The following code loads the required libraries.

```python
import geostatspy.GSLIB as GSLIB                       # GSLIB utilies, visualization and wrapper
import geostatspy.geostats as geostats                 # GSLIB methods convert to Python
```

We will also need some standard packages. These should have been installed with Anaconda 3.

```python
import os                                               # to set current working directory
import numpy as np                                      # arrays and matrix math
import pandas as pd                                     # DataFrames
import matplotlib.pyplot as plt                         # plotting
```

If you get a package import error, you may have to first install some of these packages. This can usually be accomplished by opening up a command window on Windows and then typing 'python -m pip install [package-name]'. More assistance is available with the respective package docs.

#### Set the working directory

I always like to do this so I don't lose files and to simplify subsequent read and writes (avoid including the full address each time).  Also, in this case make sure to place the required (see above) GSLIB executables in this directory or a location identified in the environmental variable *Path*.

```python
os.chdir("c:/PGE383")                                   # set the working directory
```

#### Loading Tabular Data

Here's the command to load our comma delimited data file in to a Pandas' DataFrame object.

```python
df = pd.read_csv("sample_data.csv")                     # read a .csv file in as a DataFrame
#print(df.iloc[0:5,:])                                  # display first 4 samples in the table as a preview
df.head()                                               # we could also use this command for a table preview
```

      X
      Y
      Facies
      Porosity
      Perm
      AI

      0
      100.0
      900.0
      1.0
      0.100187
      1.363890
      5110.699751

      1
      100.0
      800.0
      0.0
      0.107947
      12.576845
      4671.458560

      2
      100.0
      700.0
      0.0
      0.085357
      5.984520
      6127.548006

      3
      100.0
      600.0
      0.0
      0.108460
      2.446678
      5201.637996

      4
      100.0
      500.0
      0.0
      0.102468
      1.952264
      3835.270322

We will work by-facies, that is separating sand and shale facies and working with them separately.  This command extracts the sand and shale 'Facies" into new DataFrames for our analysis.

```python
# Separate Sample Data by-facies - Deepcopy
df_sand = pd.DataFrame.copy(df[df['Facies'] == 1]).reset_index() # copy only 'Facies' = sand records
df_shale = pd.DataFrame.copy(df[df['Facies'] == 0]).reset_index() # copy only 'Facies' = shale records
df_sand.head()                                          # preview the sand only DataFrame
```

      index
      X
      Y
      Facies
      Porosity
      Perm
      AI

      0
      0
      100.0
      900.0
      1.0
      0.100187
      1.363890
      5110.699751

      1
      8
      100.0
      100.0
      1.0
      0.137453
      5.727603
      5823.241783

      2
      9
      200.0
      900.0
      1.0
      0.137062
      14.771314
      5621.146994

      3
      10
      200.0
      800.0
      1.0
      0.125984
      10.675436
      4292.700500

      4
      18
      300.0
      900.0
      1.0
      0.111516
      27.999817
      4183.466773

#### Summary Statistics for Tabular Data

The table includes X and Y coordinates (meters), Facies 1 and 0 (1 is sandstone and 0 interbedded sand and mudstone), Porosity (fraction), and permeability as Perm (mDarcy).

There are a lot of efficient methods to calculate summary statistics from tabular data in DataFrames. The 'describe' command provides count, mean, minimum, maximum, and quartiles all in a nice data table. We use transpose just to flip the table so that features are on the rows and the statistics are on the columns.

Let's look at and compare the sumamry statistics for sand and shale.

```python
df_sand.describe().transpose()                          # summary table of sand only DataFrame statistics
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      index
      162.0
      117.981481
      59.058816
      0.000000
      75.250000
      117.500000
      157.750000
      260.000000

      X
      162.0
      831.111111
      238.857269
      50.000000
      800.000000
      945.000000
      975.000000
      1005.000000

      Y
      162.0
      526.987654
      142.707797
      100.000000
      469.000000
      509.000000
      549.000000
      939.000000

      Facies
      162.0
      1.000000
      0.000000
      1.000000
      1.000000
      1.000000
      1.000000
      1.000000

      Porosity
      162.0
      0.181001
      0.037196
      0.083842
      0.155735
      0.194823
      0.207754
      0.242298

      Perm
      162.0
      293.798999
      399.989890
      0.381032
      31.996547
      155.888123
      385.177730
      2642.999829

      AI
      162.0
      3441.701689
      969.924695
      1844.166880
      2748.296631
      3179.596509
      3998.567914
      6197.834381

```python
df_shale.describe().transpose()                         # summary table of shale only DataFrame statistics
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      index
      99.0
      149.666667
      93.588330
      1.000000
      41.500000
      201.000000
      225.500000
      259.000000

      X
      99.0
      300.444444
      196.365820
      40.000000
      201.000000
      231.000000
      300.000000
      970.000000

      Y
      99.0
      425.111111
      183.665784
      29.000000
      376.000000
      416.000000
      456.000000
      989.000000

      Facies
      99.0
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000

      Porosity
      99.0
      0.100212
      0.014483
      0.058871
      0.091902
      0.101987
      0.109846
      0.141657

      Perm
      99.0
      3.568463
      6.782476
      0.033611
      0.723788
      1.530878
      3.743835
      52.500870

      AI
      99.0
      5450.493543
      728.871517
      3595.586977
      4897.828718
      5570.604405
      5951.816775
      7881.898531

The facies have strong differences in their summary statistics.

Let's transform the data grouped overall both facies (sand and shale) and separated by facies to normal score values (Gaussian distributed with a mean of 0.0 and variance of 1.0). This is required for sequential Gaussian simulation (common target for our variogram models) and the Gaussian transform assists with outliers and provides more interpretable variograms.

Let's look at the inputs for the GeostatsPy nscore program.  Note the output include an ndarray with the transformed values (in the same order as the input data in Dataframe 'df' and column 'vcol'), and the transformation table in original values and also in normal score values.

```python
geostats.nscore                                         # see the input parameters required by the nscore function
```

    <function geostatspy.geostats.nscore(df, vcol, wcol=0, ismooth=0, dfsmooth=0, smcol=0, smwcol=0)>

The following command will transform the Porosity and Permeabilty to standard normal. Note: we perform the transform for porsity and permeability and for each for the cases of:

1. all facies together
2. sand facies only
3. shale facies only

```python
#Transform to Gaussian by Facies
df['NPor'], tvPor, tnsPor = geostats.nscore(df, 'Porosity') # nscore transform for all facies porosity
df_sand['NPor'], tvPorSand, tnsPorSand = geostats.nscore(df_sand, 'Porosity')  # nscore transform for sand facies porosity
df_shale['NPor'], tvPorShale, tnsPorShale = geostats.nscore(df_shale, 'Porosity')  # nscore transform for shale facies porosity
df['NPerm'], tvPermSand, tnsPermSand = geostats.nscore(df, 'Perm')  # nscore transform for all facies permeability
df_sand['NPerm'], tvPermSand, tnsPermSand = geostats.nscore(df_sand, 'Perm')  # nscore transform for sand facies permeability
df_shale['NPerm'], tvPermShale, tnsPermShale = geostats.nscore(df_shale, 'Perm')  # nscore transform for shale facies permeability
```

Let's look at a previous of one of the DataFrames to make sure that we now have the normal score porosity and permeability.

```python
df_sand.head()                                          # preview sand DataFrame with nscore transforms
```

      index
      X
      Y
      Facies
      Porosity
      Perm
      AI
      NPor
      NPerm

      0
      0
      100.0
      900.0
      1.0
      0.100187
      1.363890
      5110.699751
      -2.158819
      -2.158819

      1
      8
      100.0
      100.0
      1.0
      0.137453
      5.727603
      5823.241783
      -0.817609
      -1.468475

      2
      9
      200.0
      900.0
      1.0
      0.137062
      14.771314
      5621.146994
      -0.839418
      -0.955141

      3
      10
      200.0
      800.0
      1.0
      0.125984
      10.675436
      4292.700500
      -1.031153
      -1.237102

      4
      18
      300.0
      900.0
      1.0
      0.111516
      27.999817
      4183.466773
      -1.621370
      -0.694044

That looks good! One way to check is to see if the relative magnitudes of the normal score transformed values match the original values.  e.g. that the normal score transform of 0.10 porosity normal score is less than the normal score transform of 0.14 porsity.  Also, the normal score transform of values close to the mean value should be close to 0.0

Let's also check the original and transformed sand and shale porosity distributions.

```python
plt.subplot(121)                                        # plot original sand and shale porosity histograms
plt.hist(df_sand['Porosity'], facecolor='red',bins=np.linspace(0.0,0.4,50),alpha=0.2,density=True,edgecolor='black',label='Sand')
plt.hist(df_shale['Porosity'], facecolor='blue',bins=np.linspace(0.0,0.4,50),alpha=0.2,density=True,edgecolor='black',label = 'Shale')
plt.xlim([0.05,0.25]); plt.ylim([0,35.0])
plt.xlabel('Porosity (fraction)'); plt.ylabel('Frequency'); plt.title('Porosity Sand and Shale')
plt.legend(loc='upper left')
plt.grid(True)

plt.subplot(122)                                        # plot nscore transformed sand and shale histograms
plt.hist(df_sand['NPor'], facecolor='red',bins=np.linspace(-3.0,3.0,40),histtype="stepfilled",alpha=0.2,density=True,cumulative=False,edgecolor='black',label='Sand')
plt.hist(df_shale['NPor'], facecolor='blue',bins=np.linspace(-3.0,3.0,40),histtype="stepfilled",alpha=0.2,density=True,cumulative=False,edgecolor='black',label='Shale')
plt.xlim([-3.0,3.0]); plt.ylim([0,0.50])
plt.xlabel('Nscore Porosity'); plt.ylabel('Density'); plt.title('Nscore Porosity Sand and Shale')
plt.legend(loc='upper left')
plt.grid(True)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.2, wspace=0.2, hspace=0.3)
plt.show()
```

The normal score transform has correctly transformed the porosity over sand and shale facies to standard normal.  Let's plot the location maps of normal score transforms of porosity and permeability for all facies, sand facies and shale facies.

```python
cmap = plt.cm.plasma                    # color map
plt.subplot(231)
GSLIB.locmap_st(df,'X','Y','NPor',0,1000,0,1000,-3,3,'Nscore Porosity - All Facies','X (m)','Y (m)','Nscore Porosity',cmap)

plt.subplot(232)
GSLIB.locmap_st(df_sand,'X','Y','NPor',0,1000,0,1000,-3,3,'Nscore Porosity - Sand Facies','X (m)','Y (m)','Nscore Porosity',cmap)

plt.subplot(233)
GSLIB.locmap_st(df_shale,'X','Y','NPor',0,1000,0,1000,-3,3,'Nscore Porosity - Shale Facies','X (m)','Y (m)','Nscore Porosity',cmap)

plt.subplot(234)
GSLIB.locmap_st(df,'X','Y','NPerm',0,1000,0,1000,-3,3,'Nscore Permeability - All Facies','X (m)','Y (m)','Nscore Permeability',cmap)

plt.subplot(235)
GSLIB.locmap_st(df_sand,'X','Y','NPerm',0,1000,0,1000,-3,3,'Nscore Permeability - Sand Facies','X (m)','Y (m)','Nscore Permeability',cmap)

plt.subplot(236)
GSLIB.locmap_st(df_shale,'X','Y','NPerm',0,1000,0,1000,-3,3,'Nscore Permeability - Shale Facies','X (m)','Y (m)','Nscore Permeability',cmap)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.8, wspace=0.4, hspace=0.3)
plt.show()
```

Let's see what the parameters are for the gamv, irregular data, experimental variogram calculation program.

```python
geostats.gamv                                           # see the input parameters required by the gamv function
```

    <function geostatspy.geostats.gamv(df, xcol, ycol, vcol, tmin, tmax, xlag, xltol, nlag, azm, atol, bandwh, isill)>

We can use the location maps to help determine good variogram calculation parameters.

We are ready to calculate variogram! Let's calculate isotropic variograms for the transformed normal score porosity and permeability for sand, shale and mixed (without separating sand and shale).  Some information on the parameters that I chose:

```p
tmin = -9999.; tmax = 9999.;
lag_dist = 100.0; lag_tol = 50.0; nlag = 7; bandh = 9999.9; azi = 0; atol = 90.0; isill = 1
```
* tmin, tmax are trimming limits - set to have no impact, no need to filter the data
* lag_dist, lag_tol are the lag distance, lag tolerance - set based on the common data spacing (100m) and tolerance as 100% of lag distance for additonal smoothing
* nlag is number of lags - set to extend just past 50 of the data extent
* bandh is the horizontal band width - set to have no effect
* azi is the azimuth -  it has not effect since we set atol, the azimuth tolerance, to 90.0
* isill is a boolean to standardize the distribution to a variance of 1 - it has no effect since the nscore transform sets the variance to 1.0

Let's try running these variograms and visualizing them.

```python
# Calculate Sample Data Isotropic Variograms
tmin = -9999.; tmax = 9999.;
lag_dist = 100.0; lag_tol = 100.0; nlag = 7; bandh = 9999.9; azi = 0; atol = 90.0; isill = 1

lag, por_sand_gamma, por_sand_npair = geostats.gamv(df_sand,"X","Y","NPor",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)
lag, por_shale_gamma, por_shale_npair = geostats.gamv(df_shale,"X","Y","NPor",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)
lag, por_gamma, por_npair = geostats.gamv(df,"X","Y","NPor",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)

lag, perm_sand_gamma, perm_sand_npair = geostats.gamv(df_sand,"X","Y","NPerm",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)
lag, perm_shale_gamma, perm_shale_npair = geostats.gamv(df_shale,"X","Y","NPerm",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)
lag, perm_gamma, perm_npair = geostats.gamv(df,"X","Y","NPerm",tmin,tmax,lag_dist,lag_tol,nlag,azi,atol,bandh,isill)

plt.subplot(121)
plt.plot(lag,por_gamma,'x',color = 'black',label = 'All')
plt.plot(lag,por_sand_gamma,'x',color = 'orange',label = 'Sand')
plt.plot(lag,por_shale_gamma,'x',color = 'brown',label = 'Shale')
plt.plot([0,2000],[1.0,1.0],color = 'black')
plt.xlabel(r'Lag Distance $\bf(h)$, (m)')
plt.ylabel(r'$\gamma \bf(h)$')
plt.title('Isotropic NSCORE Porosity Variogram')
plt.xlim([0,700])
plt.ylim([0,1.8])
plt.legend(loc='upper left')
plt.grid(True)

plt.subplot(122)
plt.plot(lag,perm_gamma,'x',color = 'black',label = 'All')
plt.plot(lag,perm_sand_gamma,'x',color = 'orange',label = 'Sand')
plt.plot(lag,perm_shale_gamma,'x',color = 'brown',label = 'Shale')
plt.plot([0,2000],[1.0,1.0],color = 'black')
plt.xlabel(r'Lag Distance $\bf(h)$, (m)')
plt.ylabel(r'$\gamma \bf(h)$')
plt.title('Isotropic NSCORE Permeaiblity Variogram')
plt.xlim([0,700])
plt.ylim([0,1.8])
plt.legend(loc='upper left')
plt.grid(True)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.2, wspace=0.2, hspace=0.3)
plt.show()
```

The experiental variograms have some interesting features:

* the range of the sand porosity and permeability is greater than the shale porosity and permeability range
* although the shale short range experimental points may be nosy due to sparce shale data

There is much more that you could try out, e.g. direction variograms.

I hope you find this code and demonstration useful. I'm always happy to discuss geostatistics, statistical modeling, uncertainty modeling and machine learning,

*Michael*

**Michael Pyrcz**, Ph.D., P.Eng. Associate Professor The Hildebrand Department of Petroleum and Geosystems Engineering, Bureau of Economic Geology, The Jackson School of Geosciences, The University of Texas at Austin
On Twitter I'm the **GeostatsGuy** and on YouTube my lectures are on the channel, **GeostatsGuy Lectures**.