# Examples of data for which K-means or EM does not work

```
from sklearn.datasets import make_circles, make_moons, make_blobs
import matplotlib.pyplot as plt
import numpy as np
import torch  #pytorch
np.set_printoptions(precision=2)

# -- Circles data
CX, Cy = circles = make_circles (100, noise = 0.07, factor = 0.3)
# CX has the data points in rows and Cy has the corresponding class label.
# We sort the data so that data from each class is contiguous. This is not
# needed, it is just for convinience for later illustrations.
indices = np.argsort(Cy)
CX = CX[indices]
Cy = Cy[indices]

# plot data
colors = ['r*', 'go', 'bx', 'yo', 'ro'] # colors to use for the two classes.
plt.figure(figsize=(20, 5))
plt.subplot(1,3,1)
for i in range (CX.shape[0]) :
  plt.plot(CX[i,0], CX[i,1], colors[Cy[i]])

#  -- Half moon data
MX, My = circles = make_moons (100, noise = 0.07)
indices = np.argsort(My)
MX = MX[indices]
My = My[indices]
plt.subplot(1,3,2)

for i in range (MX.shape[0]) :
  plt.plot(MX[i,0], MX[i,1], colors[My[i]])

#  -- Blobs data
BX, By = circles = make_blobs(n_samples=100, n_features=2, centers=2, cluster_std=1.0, center_box=(-10.0, 10.0), )
indices = np.argsort(By)
BX = BX[indices]
By = By[indices]
plt.subplot(1,3,3)

for i in range (BX.shape[0]) :
  plt.plot(BX[i,0], BX[i,1], colors[By[i]])

```

```
# k-means on this data - on GPUs (using cuda)
def nD_k_Means (data, n_classes) :
    # nD K Means  --
    # data is a 2D tensor n_data by n_dim in size. Each row is a data vector.
    # returns the 2D tensor n_data by 1 vector assigning a data point to one of the n_classes
    # along with the mean value of each class.
    n_dim = data.shape[1];
    n_data = data.shape[0];
    # -- initialization -- randomly select points in the range
    data_max, ind = torch.max(data, 0)
    data_min, ind = torch.min(data, 0)
    mu = torch.zeros(n_classes, n_dim).cuda()
    # randomly pick n_classes points from the data a initial means
    mu = data [torch.randint(n_data, (n_classes,)), :]
    for i in range (20):
        # -- Assignment-step-------------------------
        distances = (data[:, None, :] - mu[None, :,  :])**2
        distances = distances.sum(2) #n_data by n_classes array of distances from each of the K means
        val, ind = torch.min(distances, 1)
        Z = torch.zeros((n_data, n_classes)).cuda ()
        Z[torch.arange(n_data,), ind] = 1
        #--- Mean computation step
        N = torch.sum(Z, 0)
        mu = (data[:,None,:] * Z[:,:,None]).sum(0)/N[:,None]
    # return results
    val, ind = torch.max(Z, 1)
    return(ind, mu)

# k-means on Circles data
labels, mu = nD_k_Means (torch.tensor(CX.copy()).cuda(), 2)
plt.figure(figsize=(20, 5))
plt.subplot(1,3,1)
for i in range (CX.shape[0]) :
  plt.plot(CX[i,0], CX[i,1], colors[labels[i]+2])

# k-means on Moon data
labels, mu = nD_k_Means (torch.tensor(MX.copy()).cuda(), 2)
plt.subplot(1,3,2)
for i in range (MX.shape[0]) :
  plt.plot(MX[i,0], MX[i,1], colors[labels[i]])

# k-means on Blobs data
labels, mu = nD_k_Means (torch.tensor(BX.copy()).cuda(), 2)
plt.subplot(1,3,3)
for i in range (BX.shape[0]) :
  plt.plot(BX[i,0], BX[i,1], colors[labels[i]])

```

```
# try EM on this -- using cuda
def nD_EM_GMM (data, n_classes) :
    # nD GMM  -- we will estimate the gaussian paramters of "data"
    # data is a 2D tensor n_data by n_dim in size. Each row is a data vector.
    # returns the 2D tensor n_data by n_dim probability of each data point being each class.

    n_dim = data.shape[1];
    n_data = data.shape[0];
    # -- initialization -- randomly select points in the range
    data_max, ind = torch.max(data, 0)
    data_min, ind = torch.min(data, 0)
    # mu = torch.zeros(n_classes, n_dim).cuda()
    # randomly pick n_classes number of points from the data as initial mean estimates
    mu = data [torch.randint(n_data, (n_classes,)), :]
    # n_classes by n_dim tensor -- one mean (rows) for each of the n_classes mixture components (classes).
    data_var = torch.var(data, 0)/10 # can use variance of data to initialize
    Sigma = torch.zeros(n_classes, n_dim, n_dim).double().cuda()
    for d in range(n_dim) :
        Sigma[: , d, d] = data_var [d]
    #n_classes by n_dim by n_dim covariance tensor
    pi = torch.ones((n_classes,1)).cuda()/ n_classes # n_classes by 1 tensor -- equal mixture probabilities
    Z = torch.zeros([n_data, n_classes]).cuda()  # creates an zero tensor
    for i in range (20):
        # -- E-step-------------------------
        for k in range(n_classes) :
          Z[:,k] = pi[k]*torch.distributions.multivariate_normal.MultivariateNormal (mu[k,:], covariance_matrix= Sigma[k, :, :]).log_prob(data).exp ()
        Z = Z/(Z.sum(1)[:,None])  # normalize the Z rows.
        # -- M-step-------------------------
        N = Z.sum(0)
        mu = (data[:,None,:] * Z[:,:,None]).sum(0)/N[:,None]
        d = data[:, None, :] - mu[None, :,  :]
        d = torch.matmul (d [:,:,:,None], d [:,:,None,:])
        Sigma = Z[:,:, None, None] * d  # performing the product with z_{ik} Sigma is a N by n_classes by n_dim by n_dim tensor
        Sigma = (Sigma.sum(0))/N[:,None,None]
        pi = N/N.sum()
        #print("\n mu=", mu.shape, mu, "\n Sigma =", Sigma.shape, Sigma, "\n pi=", pi, "\n data=", data.shape)
    return(Z, mu, Sigma)

# -- EM on Circles
Z, mu, Sigma = nD_EM_GMM (torch.from_numpy(CX.copy()).cuda(), 2)
val, labels = torch.max(Z, 1)
plt.figure(figsize=(20, 5))
plt.subplot(1,3,1)
for i in range (CX.shape[0]) :
  plt.plot(CX[i,0], CX[i,1], colors[labels[i]])

# --- EM on Moon Shape
Z, mu, Sigma = nD_EM_GMM (torch.from_numpy(MX.copy()).cuda(), 2)
val, labels = torch.max(Z, 1)
plt.subplot(1,3,2)
for i in range (MX.shape[0]) :
  plt.plot(MX[i,0], MX[i,1], colors[labels[i]])

# --- EM on Blobs Shape
Z, mu, Sigma = nD_EM_GMM (torch.from_numpy(BX.copy()).cuda(), 2)
val, labels = torch.max(Z, 1)
plt.subplot(1,3,3)
for i in range (BX.shape[0]) :
  plt.plot(BX[i,0], BX[i,1], colors[labels[i]])
```

# Transformation of data to graph and then to a matrix

If you observe the datasets, we want to group the first dataset points into concentric circles and half-circles in the other. The core property we are trying to use is **local** distance among points. We want to associate points together if they are locally together. Local distances matter more than distances that are far away. Distances between points that are far away should not factor into the clustering. If we have a grouping strategy where only local distances play the determining role, we would have a solution. Graph spectral grouping is such an approach.

We will capture the local properties in a weighted graph and then use the matrix representation of this graph to transform the points into space when local distances are captured accurately at the expense of far-away distances. In this transformed space, simple k-means or EM or a linear classifier will work.

The first step in the process is to represent the data collection, $D=\{ \mathbf{x_i} | i=1,\cdots,N\}$ as a graph, $G=(V, E)$. Each data element, $\mathbf{x_i}$, is represented by a vertex (or node) in that graph; there is a one-to-one correspondence between the vertices and the data elements. Note that the graph will be as large as the dataset. There are edges (sometimes referred to as links) between the vertices associated with **weights**. The weight, $w_{ij}$ for each edge (or link) between two vertices, $v_i$ and $v_j$, is a non-negative number ($w_{ij} \ge 0$) capturing the **similarity** of the data element. One possibility is a Gaussian similarity function given by:

\begin{equation}
w_{ij} = s({\bf x_i}, {\bf x_j}) = \exp \left ( - ||{\bf x_i} - {\bf x_j}||^2/2 \sigma^2 \right )
\end{equation}

It is worth pointing out that one can have other definitions of similarity. Also, note that the concept of *distance* between data elements is *inversely* related to *similarity*.

If two vertices are not connected, i.e., two data elements are not at all similar, then it means that $w_{ij} = 0$.

The weight, $w_{ii}$, is typically assigned zero, i.e., no self-loops. However, some have used the diagonal values to capture the relative importance among the data element. For example, the diagonal entries could be confidence values associated with the corresponding data element. More confident data elements will factor prominently in the grouping. An example is in ([Sarkar and Boyer, 1996](https://marathon.csee.usf.edu/~sarkar/PDFs/change-cvpr.pdf); [Sarkar and Boyer, 1998](https://figment.csee.usf.edu/~sarkar/PDFs/change-cviu.pdf)), where the data elements are boundary contours pieces in images, the diagonal values are inversely proportional to the fit of the contour pieces to straight lines or arcs.

The similarity relationship is symmetric, so the edges are also **undirected**, i.e., $w_{ji} = w_{ij}$.

We can organize these weights as a $N \times N$ matrix $W$, called the **adjacency matrix**. The matrix will be **symmetric**.

```
# Example of the graph construction

import torch
import numpy as np
import matplotlib.pyplot as plt

# Vary these parameters and conduct an ablation study of the quality of the separation
batch_size = 100  # batchsize for distance computation. Pairwise distance computation is the most expensive step
kernel_width = 0.7 # bandwidth of the Gaussian similarity
k_neighbors = 200  # number of nearest neighbors considered to compute the graph
dataset = "CX"
data = CX.copy()  # select the dataset to operate on
data_labels = My.copy() # true labels for the data

#----------------------Functions used--------------------
#`squared_distance_batch` contains some broadcast magic that allows us to compute the distance from each point in a batch to all points in the data.
# See - https://numpy.org/doc/stable/user/basics.broadcasting.html
#  and https://scipy-lectures.org/intro/numpy/operations.html
def squared_distance_batch(b, a):
    #print(a.shape, a[None,:].shape, b.shape, b[:,None,:].shape)
    return (((a[None,:,:] - b[:,None,:]) ** 2).sum(2))

def gaussian(squared_dist, bandwidth):
    return torch.exp(-0.5 * (squared_dist / (bandwidth**2))) / (bandwidth * torch.sqrt(2 * torch.tensor(np.pi)))

#---------------------- Main code starts here --------------------
X = torch.from_numpy(data).cuda()
N = X.shape[0]
W = torch.zeros((X.shape[0], X.shape[0]))
for i in range(0, N, batch_size):
    s = slice(i, min(N, i + batch_size))
    distance = squared_distance_batch(X, X[s])
    W[:,s] = distance
#--------------------Section 1: k-Nearest Neighbors distances only----------------
# keep k-nearest neighbors. zero out rest
sorted, indices = torch.sort(W, dim=1, descending=False)
for i in range(N) :
    for j in range (k_neighbors , N) :
        W[i, indices[i, j]] = 9999.0 #large distance
# Need to keep the weight matrix symmetric, otherwise we essentially have a direct graph and spectral clustering will not work
W = 0.5*(W+W.transpose(0,1))

#--------------------Section 2: What is this step doing?----------------
#for i in range(N) :
#    for j in range(N) :
#        for k in range(N) :
#            w = W[i,k] + W[k,j]
#            if (w < W[i,j]) :
#                W[i,j] = w
#-----------------------------------------------------------

W = gaussian(W, kernel_width)

# Zero out the diagonal. In an application, these could be replaced by confidence
# values associated with data elements or to express relative importance among
# data elements.
torch.diagonal(W).fill_(0)

#---------------------- Main code ends here --------------------
# Visualize the matrix

print("Dataset:", dataset, ", k_neighbors =",k_neighbors)

plt.figure(figsize=(20,10))
A = W.cpu()
plt.subplot(2,2,1)
plt.imshow(A, 'gray')
plt.title('Adjacency weight matrix (W). Larger values are brighter pixels')

plt.subplot(2,2,2)
plt.imshow(My[:, None] ^ My[:,None].transpose(), 'gray')
plt.title ('within class edges are in show in black')

plt.subplot(2,2,3)
W_hist = torch.histc(W, 100).cpu()
x_hist = torch.linspace(A.min(), A.max(), 100).cpu()
plt.plot(x_hist, np.log(W_hist + 1)/np.log(10))
plt.xlabel('weights')
plt.ylabel('Log_10 # of edges')
plt.title('Histogram of weights w_{ij}');

```

# Inclass Work (15 min)

Breakout into groups of two and experiment with the above code. Report on the following and submit directly on canvas as pdf.

1. Comment of the nature of the weight matrix for the circles, moon, and blobs data.

Changing line 11 above we can switch between the Moon (MX matrix), the Circle (CX matrix), and the Blobs (BX matrix).

In all 3 cases, the weight matrix appears to segment the data into NW and SE quadrants. Each type shows slightly different results with the context of the segment however.

2. Change the value of k in computation of the nearest neighbors and comment on the impact on the weight matrix.

As k is increased the sparsity of the NxN adjacency matrix is increased predominantly in both NW and SE quadtrants, but if very large k is used, SW and NE quadrants begin to populate as well with non-zero w coefficients.

```

```

```
# torch code snippet to demonstrate how sorting can be used to select the k-nearest neighbors in a weighted graph.
# Is it possible to replace the for loops with just statements? Let me know if you find a solution.
# for-loops slows processing down significantly.

x = torch.randn(3, 4)
sorted, index = torch.sort(x, dim=1, descending=True)
print(x, "\n", sorted, "\n", index, "\n")
for i in range(3) :
  for j in range (2,4) :
    print(i, j, index[i,j])
    x[i, index[i,j]] = 0
print(x)

```

    tensor([[-2.1737, -0.7031,  0.3261, -0.6332],
            [ 0.1683, -1.0596, -1.0154,  1.3052],
            [-0.1275, -1.9488, -1.0870,  0.4655]])
     tensor([[ 0.3261, -0.6332, -0.7031, -2.1737],
            [ 1.3052,  0.1683, -1.0154, -1.0596],
            [ 0.4655, -0.1275, -1.0870, -1.9488]])
     tensor([[2, 3, 1, 0],
            [3, 0, 2, 1],
            [3, 0, 2, 1]])

    0 2 tensor(1)
    0 3 tensor(0)
    1 2 tensor(2)
    1 3 tensor(1)
    2 2 tensor(2)
    2 3 tensor(1)
    tensor([[ 0.0000,  0.0000,  0.3261, -0.6332],
            [ 0.1683,  0.0000,  0.0000,  1.3052],
            [-0.1275,  0.0000,  0.0000,  0.4655]])

# Eigenvalues and Eigenvector of Graphs

We will use the eigenvectors of the matrix representation of the graph to embed the nodes into another space, where the distributions reflect the data similarity measure we are trying to capture. First, here is a short primer on eigenvalues and eigenvectors. (As a linear algebra reference, you can use the supplementary materials in Szeliski's textbook to brush up your memory or learn from the [excellent lectures by Prof. Gilbert Strang at MIT](https://ocw.mit.edu/resources/res-18-010-a-2020-vision-of-linear-algebra-spring-2020/). Linear algebra is used extensively in computer vision, and more expertise you can gain in it, the better.)

If $\mathbf{W}$ is a $N \times N$ matrix and $\mathbf{v}$ is a $N \times 1$ vector, then their product $ \mathbf{W v} $ is another $N \times 1$ vector $\mathbf{w} = \mathbf{W v}$. In other words a matrix transforms a vector into another vector.  Eigenvectors are special vectors that get transformed into scaled versions of itself.

\begin{equation}
\mathbf{W v} = \lambda \mathbf{v}
\end{equation}

For a $N \times N$ matrix, there can be at most $N$ eigenvectors. Each eigenvector, $\mathbf{v_i}$ will be associated with an eigenvalue, $\lambda_i$. In general, these eigenvalues and the entries of the eigenvectors can be complex-valued. **However, if a matrix is symmetric, the eigenvalues and eigenvectors will be real-valued.** We can arrange these eigenvalues and eigenvectors in one compact matrix representation.

\begin{eqnarray}
\mathbf{W V} = \Lambda \mathbf{V} = \mathbf{V} \Lambda
\end{eqnarray}

where $\mathbf{V} = \left [\mathbf{v_1}, \cdots, \mathbf{v_N}\right]$, i.e., the eigenvectors are organized as the columns of this matrix. The corresponding eigenvalues, $\lambda_1, \cdots, \lambda_N$, are organized as the *diagonal* matrix $\Lambda$.

Eigenvectors are **orthonormal**. Each eigenvector has a unit magnitude, and two eigenvectors are mutually orthogonal.

\begin{equation}
\mathbf{v_i^T v_j} = \left \{ \begin{array} {c c} 1 & i=j \\ 0 & i \neq j \end{array}\right .
\end{equation}
or equivalently,
\begin{equation}
\mathbf{V^T V} = \mathbf{I}
\end{equation}

Typically, these eigenvalues and eigenvectors are arranged in ascending or descending order to reflect the order of importance. For adjacency matrix representations, larger eigenvalues are more critical as from the following mathematical manipulation.

\begin{eqnarray}
\mathbf{W v} & = & \lambda \mathbf{v} \\
\mathbf{v^T W v} & = & \lambda \mathbf{v^T v} \\
\lambda & = & \frac{\mathbf{v^T W v}}{\mathbf{v^T v}}
\end{eqnarray}

The fraction of the two scalars on the right, $\frac{\mathbf{v^T W v}}{\mathbf{v^T v}}$ is an interesting quantity. If we let entries of the vectors $\mathbf{v}$ denote the importance of the corresponding vertex (node) in the graph, then the *scalar* $\mathbf{v^T W v}$ is a measure of **cohesion** of nodes, how tightly the nodes are connected. The normalization by the denominator keeps the values in the vector $\mathbf{v}$ from being very large. Of course, for eigenvectors $\mathbf{v^T v} = 1$. Eigenvectors are unit magnitude. In fact, the [Rayleigh Ritz theorem](https://en.wikipedia.org/wiki/Rayleigh_quotient) shows that the ratio $\frac{\mathbf{v^T W v}}{\mathbf{v^T v}}$ is maximized (or minimized) by an eigenvector of the matrix $\mathbf{W}$. In our particular context, we want to preserve and maximize the cohesion of the nodes, so we are concerned with larger eigenvalues.

In summary, the eigenvalues of the adjacency matrix give an idea of the amount of **cohesion** in the graph, and the corresponding eigenvector gives us weights to associate with each node to achieve that cohesion.

```
# Computing Eigenvalues and Eigenvectors of the weight matrix and visualize them.

k_largest = 5
# Compute k-largest eigenvectors.
[eigenvalues, U]= torch.lobpcg(W, k=k_largest, method='ortho', largest=True)
eigenvalues = eigenvalues.cpu()
U = U.cpu()
# Note: computing eigenvalues and eigenvectors using eig function (as shown below) is wasteful
# as we do not need all the eigenvectors.
#[eigenvalues, U]=torch.eig(W, eigenvectors=True) #- wastefull as we do not need all the eigenvectors.

N = W.shape[0]
plt.figure(figsize=(15, 30))
plt.subplot(4,2,1)
plt.plot(eigenvalues, '-o')
plt.title('eigenvalues')

for i in range(0, 2) :
    plt.subplot(4,2,2*i+3)
    plt.plot(U[:,i])
    plt.axis([0, N, torch.min(U[:,i]), torch.max(U[:,i])])
    plt.title('Eigenvector # {}'.format(i+1))

    plt.subplot(4,2,2*i+4)
    plt.scatter(data[:,0], data[:,1], c= (U[:,i]-torch.min(U[:,i])), s=100, edgecolors="black")
    plt.gray()
    plt.title('Data point: Color proportional to eigenvector entry value')

```

# Inclass work (10 min)

1. Test that the eigenvectors of the weight matrix are orthonormal, i.e. orthonogal to each other, each vector has unit norm.

2. Show that ${\bf v}$ is an eigenvector, then so is $-{\bf v}$, i.e., there is sign ambiguity in the specification of the eigenvectors.

# Using eigenvectors to embed data in a new space

So far, we have seen how the **entries** of each eigenvector can be associated with the graph's respective vertices (nodes) to denote its participation in the possible cluster. Instead of considering these eigenvectors individually, we can think them as a group and associate a **"coordinate"** to each vertex (or node), which then can be used to **group** the vertices into clusters.

For each eigenvector, $\mathbf{v_i}$, the j-th entry, $v_{ij}$, is associated with the j-th data point or j-th graph node. If we take all the j-th entries of the eigenvectors, $\{v_{1,j}, v_{2, j}, \cdots, v_{N, j} \}$, it forms a cooridinate vector associated with the j-th node. We truncate and consider only the top k largest eigenvectors, then the new coordinate for each data point is only k-dimensional, $\{v_{1,j}, \cdots, v_{k, j} \}$.

We usually multiply these entries with the sqrt of the eigenvalues to make them proportional to the dimension's overall strength. (The choice of the sqrt is not arbitrary but has a mathematical reason. But that discussion is a long sidebar.) Remember, the eigenvalues capture the average coherence with the nodes' weights according to the corresponding eigenvector. Thus, the new coordinates are:

\begin{equation}
\mathbf{x_i}^{new} = \left [ \sqrt{\lambda_1} v_{1,i}, \cdots, \sqrt{\lambda_k} v_{k,i} \right ]^T
\end{equation}

# Other possible matrix representations

In addition to the weighted adjacency matrix representation, other possible matrix representations of graphs include the Laplacian, signed Laplacian, and normalized versions of these representations. Here we define some of these. The suitability of each representation is dependent on the application.

A graph **Laplacian** is defined as $L = D - W$, where $D$ is a diagonal matrix composed of each vertex *degree*. The degree of a vertex $v_i$ is the total of the weights of the edges connected to the vertex.

\begin{equation}
d_i = \sum_{j=1}^N w_{ij}
\end{equation}

The degree of a vertex is a measure of how well the node is connected to other nodes. The degree matrix $D$ is defined as the diagonal matrix with the degrees, $\{d_i, \cdots, d_N\}$, on the diagonal. Note that the Laplacian matrix is also symmetric, and the off-diagonal terms are negative. **The sum of the rows and the columns of the Laplacian is zero.** The graph Laplacian was used by ([Sarkar and Soundararajan, 2000](https://ieeexplore.ieee.org/abstract/document/857006)) to group edge segments in images.

The signless **Laplacian** is defined as $Q = D + W$. This matrix is also symmetric, however, the rows and the columns do not sum to zero. This has been proposed by ([Cvetković and Simić, 2010](https://www.sciencedirect.com/science/article/pii/S0024379509002808)) but never used in computer vision application as of today (Jan 2021).

There are also **normalized** versions of these representations. We will start with the normalized version of the original weighted adjacency matrix. The weight of each edge, $w_{ij}$, between two vertices $v_i$ and $v_j$ is normalized by the vertices' degrees. This was used by ([Shi and Malik, 2000](https://ieeexplore.ieee.org/document/868688)) for image segmentation. This is the famous normalized cut approach that appeared in the same year as the plain Laplacian proposal of Sarkar and Soundararajan.

\begin{equation}
w^{sym}_{ij} = \frac{w_{ij}}{\sqrt{d_i} \sqrt{d_j}}
\end{equation}

There is a second way to normalize the weight matrix, but that does not result in a symmetric matrix.

\begin{equation}
w^{rw}_{ij} = \frac{w_{ij}}{d_i}
\end{equation}
This matrix is related to random walks on the graph hence the superscript rw. Note $w^{rw}_{ij} \neq w^{rw}_{ji}$.

The two kinds of normalized adjacency matrices can be expressed in matrix form as
\begin{equation}
W_{sym} = D^{-\frac{1}{2}} W D^{-\frac{1}{2}}
\end{equation}
and
\begin{equation}
W_{rw} = D^{-1} W
\end{equation}

Using these normalized versionf of the weight adjacency matrices, we can define normalized version of the Laplacian and the sunsigned Laplacian matrices as follows
\begin{eqnarray}
L_{sym} & = & I - W_{sym} = I - D^{-\frac{1}{2}} W D^{-\frac{1}{2}} = D^{-\frac{1}{2}} (D - W) D^{-\frac{1}{2}} = D^{-\frac{1}{2}} L D^{-\frac{1}{2}} \\
L_{rm} & = & I - W_{rm} = I - D^{-1} W  = D^{-1} (D - W) = D^{-1} L  \\
Q_{sym} & = & I + W_{sym} = I + D^{-\frac{1}{2}} W D^{-\frac{1}{2}} = D^{-\frac{1}{2}} (D + W) D^{-\frac{1}{2}} = D^{-\frac{1}{2}} Q D^{-\frac{1}{2}} \\
Q_{rm} & = & I + W_{rm} = I + D^{-1} W  = D^{-1} (D + W) = D^{-1} L  \\
\end{eqnarray}

For more information about spectral clustering read the following papers

Von Luxburg, Ulrike. "[A tutorial on spectral clustering](https://link.springer.com/article/10.1007/s11222-007-9033-z)." Statistics and computing 17.4 (2007): 395-416.

Cvetković, Dragoš, and Slobodan Simić. "[Graph spectra in computer science](https://www.sciencedirect.com/science/article/pii/S0024379510006117)." Linear Algebra and its Applications 434.6 (2011): 1545-1562.

```
D = torch.diag_embed(W.sum(0))
D_inv = torch.diag_embed(1/W.sum(0))
D_invsqrt = torch.diag_embed(1/torch.sqrt(W.sum(0)))

W_sym = D_invsqrt @ (W @ D_invsqrt)
W_rw = D_inv @ W

L = D - W
Q = D + W
L_sym = torch.eye(W.shape[0]) - W_sym
L_rw = torch.eye(W.shape[0]) - W_rw
Q_sym = torch.eye(W.shape[0]) + W_sym
Q_rw = torch.eye(W.shape[0]) + W_rw

# Visualize the matrix
plt.figure(figsize=(20,20))
plt.subplot(2,3,1)
plt.imshow(W.cpu(), 'gray')
plt.subplot(2,3,2)
plt.imshow(Q.cpu(), 'gray')
plt.subplot(2,3,3)
plt.imshow(L.cpu(), 'gray')
```

# Spectral Clustering Algorithm

1. Input:  Data $X$ and $k$ - the number of clusters to construct.

2. Construct a similarity graph, represented by its weighted adjacency matrix, ${\bf W}$.
3. (optional) Normalize the weight matrix ${\bf W}$ by the vertex degrees.
4. (optional) Compute other variations such as the Laplacian ${\bf L}$, or signless or normalized versions of it.
5.  Compute the first $k$ largest eigenvectors of the adjacency form
>>>(or $k+1$ smallest eigenvectors of the Laplacian form).
6. Let ${\bf X}^{new} = [ (\lambda_1)^{\frac{1}{2}}{\bf u_1}, \cdots, (\lambda_k)^{\frac{1}{2}}{\bf u_k} ]$ be the matrix containing the eigenvector as columns. Let the corresponding eigenvalues be $\lambda_1, \cdots, \lambda_k$. The number of rows of its matrix will be equal to the number of data points.
>>>(If working with a Laplacian form, ignore the smallest eigenvector, which is associated with a eigenvalue of zero, and arrange the rest as a matrix, multiplied by the inverse of the corresponding eigenvalue. ${\bf X}^{new} = [ (\lambda_1)^{-\frac{1}{2}}{\bf u_1}, \cdots, (\lambda_k)^{-\frac{1}{2}}{\bf u_k} ]$)

7. Assign the $i$-th row of this matrix, weighted by the corresponding eigenvalue is the new coordinate of the $i$-th data point.

>> $\begin{equation}
 {\bf x_i}^{new} = [(\lambda_1)^{\frac{1}{2}} u_{1i}, (\lambda_2)^{\frac{1}{2}}u_{2i}, \cdots, (\lambda_k)^{\frac{1}{2}} u_{ki}]^T
 \end{equation}$

>>>>>(If working with the Laplacian form use: $\begin{equation}
{\bf x_i}^{new} = [(\lambda_1)^{-\frac{1}{2}} u_{1i}, (\lambda_1)^{-\frac{1}{2}} u_{2i}, \cdots, (\lambda_1)^{-\frac{1}{2}} u_{ki}]^T
\end{equation}$)

8. Perform k-means or EM on this new transform data matrix, whose rows are the data units.

9. Output: the clusters returned by k-means or EM

```
# Cluster in the new transformed dimension

k_selected = 2
# Compute k-selected eigenvectors.

#---------------Compute Weight Matrix   -----------------------
#-----     For W, Q, and their normalized version W_sym, W_rm, Q, Q_sym, Q_rw use the following
[eigenvalues, U]=torch.lobpcg(W, k=k_selected, method='ortho', largest=True)
#-----     Compute Graph Spectral Space-----------------------
data_new = U * torch.sqrt(eigenvalues)

#------------------------------------------------------------------
#-----     For Laplacian, L, and its normalized versions, L_sym, L_rw use the following three lines
#[eigenvalues, U]=torch.lobpcg(L_sym, k=k_selected+1, method='ortho', largest=False)
#U = U[:, 1:k_selected+1]; # For Laplacian we have to ignore the smallest eigenvector, which will be all ones.
#eigenvalues = eigenvalues[1:k_selected+1]
#-----     Compute Graph Spectral Space
#data_new = U * torch.sqrt(1/eigenvalues)
#------------------------------------------------------------------

print(U.shape, eigenvalues.shape, data_new.shape, data_new.dtype)

#---------------------Using k-means-------------------------------------
#labels, mu = nD_k_Means (data_new.double().cuda(), 2)

#---------------------Using EM -------------------------------------
Z, mu, Sigma = nD_EM_GMM (data_new.double().cuda(), 2)
val, labels = torch.max(Z, 1)

#---------------------Display-------------------------------------
plt.figure(figsize=(10, 10))
plt.subplot(2,2,1)
for i in range (data.shape[0]) :
    plt.plot(data[i,0], data[i,1], colors[data_labels[i]])
plt.title ('Data in orginal space with true labels')

data_new = data_new.cpu()
plt.subplot(2,2,2)
for i in range (data.shape[0]) :
    plt.plot(data_new[i,0], data_new[i,1], colors[data_labels[i]])
plt.title ('Data in eigenspace (v_1, v_2) (top 2 eigenvectors)');

plt.subplot(2,2,3)
for i in range (data.shape[0]) :
  plt.plot(data[i,0], data[i,1], colors[labels[i]+2])

plt.subplot(2,2,4)
for i in range (data.shape[0]) :
  plt.plot(data_new[i,0], data_new[i,1], colors[labels[i]+2])

```

# An example application in image segmentation

The most important part is the construction of the weight matrix. The more rich a feature set that goes into it, the better the performance.

```
from google.colab import drive
drive.mount('/content/drive')
!ls /content/drive/MyDrive/Colab\ Notebooks/data/

```

    Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
    hawaii.png   house_2.png       lizard.jpg	  mountain_peak_1.png
    house_1.png  house_facade.png  MOT16-08-raw.webm  mountain_peak_2.png

```
def reduce_size (img, scale_percent = 20) :
  #calculate the scale_percent of original dimensions
  width = int(img.shape[1] * scale_percent / 100)
  height = int(img.shape[0] * scale_percent / 100)
  # resize image
  return (cv2.resize(img, (width, height)))
```

```
#----------------------Functions used--------------------
#`squared_distance_batch` contains some broadcast magic that allows us to compute the distance from each point in a batch to all points in the data.
# See - https://numpy.org/doc/stable/user/basics.broadcasting.html
#  and https://scipy-lectures.org/intro/numpy/operations.html
def squared_distance_batch(b, a):
    #print(a.shape, a[None,:].shape, b.shape, b[:,None,:].shape)
    return (((a[None,:,:] - b[:,None,:]) ** 2).sum(2))

def gaussian(squared_dist, squared_bandwidth):
    return torch.exp(-0.5 * (squared_dist / (squared_bandwidth))) / (torch.sqrt(2 * squared_bandwidth*torch.tensor(np.pi)))

```

```
def compute_weight_matrix (data, batch_size) :
    X = torch.from_numpy(data).cuda()
    N = X.shape[0]
    print('Computing distances between pixels:')
    W = torch.zeros((X.shape[0], X.shape[0]))
    for i in range(0, N, batch_size):
        s = slice(i, min(N, i + batch_size))
        distance = squared_distance_batch(X, X[s])
        W[:,s] = distance

    #-----------------------------------------------------------
    print('Computing gaussian kernel', W.shape)
    kernel_width = torch.var(W)
    print('Gaussian kernel (variance of distances):', kernel_width)
    W = gaussian(W, kernel_width)

    return(W)

```

```
def spectral_clustering_torch (data, batch_size=500, type=0, k_selected = 2):

    W = compute_weight_matrix (data, batch_size)
    plt.subplot(5,2,2)
    plt.imshow(W)
    if type == 0 :
        #---------------Compute Weight Matrix   -----------------------
        #-----     For W, Q, and their normalized version W_sym, W_rm, Q, Q_sym, Q_rw use the following
        [eigenvalues, U]=torch.lobpcg(W, k=k_selected, method='ortho', largest=True)
        #-----     Compute Graph Spectral Space-----------------------
        data_new = U * torch.sqrt(eigenvalues)
    else :
        #------------------------------------------------------------------
        #-----     For Laplacian, L, and its normalized versions, L_sym, L_rw use the following three lines
        [eigenvalues, U]=torch.lobpcg(L_sym, k=k_selected+1, method='ortho', largest=False)
        U = U[:, 1:k_selected+1]; # For Laplacian we have to ignore the smallest eigenvector, which will be all ones.
        eigenvalues = eigenvalues[1:k_selected+1]
        #-----     Compute Graph Spectral Space
        data_new = U * torch.sqrt(1/eigenvalues)
        #------------------------------------------------------------------

    print(U.shape, eigenvalues.shape, data_new.shape, data_new.dtype)

    #---------------------Using k-means-------------------------------------
    labels, mu = nD_k_Means (data_new.double().cuda(), k_selected)

    #---------------------Using EM -------------------------------------
    # %time Z, mu, Sigma = nD_EM_GMM (data_new.double().cuda(), k_selected)
    #val, labels = torch.max(Z, 1)

    return (labels)
```

```
import cv2
import matplotlib.pyplot as plt
import numpy as np
import torch
plt.figure(figsize=(20,30))

# Load an color image in grayscale
img = cv2.imread('/content/drive/MyDrive/Colab Notebooks/data/hawaii.png')

# Change color spaces. Can experiment with other spaces.
img = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)
print("Input image size:", img.shape)

img = reduce_size(img, scale_percent=35)
plt.subplot(5,2,1)
plt.imshow(cv2.cvtColor(img, cv2.COLOR_LAB2RGB))
print("Reduced image size:", img.shape)

pixels = np.float32(img.reshape((-1,3))) # change the shape of the 2D image into a 1D array

##------------------
%time X = spectral_clustering_torch(pixels, batch_size=50000, type=0, k_selected = 3).cpu().numpy()
##------------------

seg_img = np.uint8(X.reshape([img.shape[0], img.shape[1]]))
plt.subplot(5,2,3)
plt.imshow(seg_img)
```

# How do you evaluate which segmentation is better?

https://www2.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/resources.html

**Berkeley Segmentation Data Set and Benchmarks 500 (BSDS500)**

This new dataset is an extension of the BSDS300, where the original 300 images are used for training / validation and 200 fresh images, together with human annotations, are added for testing. Each image was segmented by five different subjects on average. Performance is evaluated by measuring Precision / Recall on detected boundaries and three additional region-based metrics.


\title{Cascaded Integrator Comb (CIC) Filter in myHDL}
\author{Steven K Armour}
\maketitle

# Python Libraries Utilized

```python
import numpy as np
import scipy.signal as sig
import pandas as pd

from sympy import *
init_printing()
from IPython.display import display, Math, Latex

from myhdl import *
from myhdlpeek import Peeker

import matplotlib.pyplot as plt
%matplotlib inline
```

# Acknowledgments
The orgianl CIC filter written in myHDL was done by [Christopher Felton](https://github.com/cfelton)
(myHDL.old version [here](https://github.com/jandecaluwe/site-myhdl-retired/blob/master/_ori/pages/projects/gciccomplete.txt))

Author of myHDL [Jan Decaluwe](http://www.myhdl.org/users/jandecaluwe.html) and the author of the myHDL Peeker [XESS Corp.](https://github.com/xesscorp/myhdlpeek)

# Refrances

# CIC Filter Backround and Pro/Cons of use

# CIC Comb (dervitive) Unit
## 1st Order Comb Unit

### Sympy Anyslsis

One of the two fundumental stages of the CIC filter is the Derivative stage also called a Comb. The lowest order Comb filter is governed by the following equation

```python
n, M=symbols('n, M')
y=Function('y')(n)
x=Function('x')

Comb1Eq=Eq(y, x(n)-x(n-1)); Comb1Eq
```

where by applying the "Z" Transform to 1st order Comb equation we get

```python
z=symbols('z')
Y=Function('Y')(z); X=Function('X')(z)
Comb1ZEq=Eq(Y, (1-z**-1)*X); Comb1ZEq
```

Which results in the following Transfer Function and discrate inpulse responce

```python
H=Function('H')(z)
Comb1TFEq=Eq(H, solve(Comb1ZEq, Y)[0]/X); Comb1TFEq
```

The Transfer function has the following zeros and poles

```python
TFnum, TFden=fraction(Comb1TFEq.rhs); TFnum, TFden
TFZeros=solve(TFnum, z); TFPoles=solve(TFden)
TFPZStatment=f"""{H} has zeros from {TFnum} at {TFZeros}
and poles from {TFden} at {TFPoles}"""
print(TFPZStatment)
```

    H(z) has zeros from z - 1 at [1]
    and poles from z at [0]

### scipy anyslsis

using the exspersion for the numurator and denomnator there coefecnts can be build off thusly and read into scipy's signal module to numerical inspect the resulting transfer function thusly where we will use a sample time

```python
TFCoefN=lambda NumDen: [float(i) for i in Poly(NumDen, z).coeffs()]
TFnumC=TFCoefN(TFnum); TFdenC=TFCoefN(TFden)
TFnumC, TFdenC
```

```python
Comb1Sci=sig.TransferFunction(TFnumC, [1, 0], dt=1/4.8e3)
Comb1Sci
```

    TransferFunctionDiscrete(
    array([ 1., -1.]),
    array([ 1.,  0.]),
    dt: 0.00020833333333333335
    )

```python
w, mag, phase = sig.dbode(Comb1Sci )
fig, ax=plt.subplots(nrows=2, ncols=1, sharex=True)
plt.title('Bode Plot of 1st order Comb Section')
ax[0].semilogx(w, mag, label='Mag')
ax[0].set_ylabel('Mag [dB]')
ax[1].semilogx(w, phase, label='Phase')
ax[1].set_ylabel('Phase[deg]')
ax[1].set_xlabel(r'$\omega[rad/dt]$')
None

```

```python
plt.title('Impulse Respince of 1st Order Comb Filter')
tout, yout=sig.dimpulse(Comb1Sci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

```python
plt.title('Step Respince of 1st Order Comb Filter')
tout, yout=sig.dstep(Comb1Sci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

## M-th Order Comb Unit

```python
M=symbols('M')
CombMEq=Eq(y, x(n)-x(n-M)); CombMEq
```

```python
CombMZEq=Eq(Y, (1-z**-M)*X); CombMZEq
```

```python
H=Function('H')(z)
CombMTFEq=Eq(H, simplify(solve(CombMZEq, Y)[0]/X)); CombMTFEq
CombMTFEq=Eq(H, expand(CombMTFEq.rhs*z**M)/z**M); CombMTFEq

sups={M:3}
CombMTFEq=CombMTFEq.subs(sups); CombMTFEq

```

```python
TFnum, TFden=fraction(CombMTFEq.rhs); TFnum, TFden
TFZeros=solve(TFnum, z); TFPoles=solve(TFden)
TFPZStatment=f"""{H} has zeros from {TFnum} at {TFZeros}
and poles from {TFden} at {TFPoles}"""
print(TFPZStatment)
```

    H(z) has zeros from z**3 - 1 at [1, -1/2 - sqrt(3)*I/2, -1/2 + sqrt(3)*I/2]
    and poles from z**3 at [0]

```python
TFCoefN=lambda NumDen: [float(i) for i in Poly(NumDen, z).coeffs()]
TFnumC=TFCoefN(TFnum); TFdenC=TFCoefN(TFden)
for i in range(sups[M]-1):
    TFnumC.insert(1, 0)

for i in range(sups[M]):
    TFdenC.insert(1, 0)
TFnumC, TFdenC
```

```python
CombMSci=sig.TransferFunction(TFnumC, TFdenC, dt=1/4.8e3)
CombMSci
```

    TransferFunctionDiscrete(
    array([ 1.,  0.,  0., -1.]),
    array([ 1.,  0.,  0.,  0.]),
    dt: 0.00020833333333333335
    )

```python
w, mag, phase = sig.dbode(CombMSci )
fig, ax=plt.subplots(nrows=2, ncols=1, sharex=True)
plt.title(f'Bode Plot of {sups[M]} order Comb Section')
ax[0].semilogx(w, mag, label='Mag')
ax[0].set_ylabel('Mag [dB]')
ax[1].semilogx(w, phase, label='Phase')
ax[1].set_ylabel('Phase[deg]')
ax[1].set_xlabel(r'$\omega[rad/dt]$')
None
```

```python
plt.title(f'Impulse Respince of {sups[M]} order Comb Section')
tout, yout=sig.dimpulse(CombMSci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

```python
plt.title(f'Step Respince of {sups[M]} order Comb Section')
tout, yout=sig.dstep(CombMSci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

## myHDL Implimintation of M-th Order Comb Section

### $z^{-1}$ as a delay

```python
TestValues=pd.read_pickle('SinGen16.pkl')
plt.stem(TestValues['Time[s]'], TestValues['GenValue'])
plt.plot(TestValues['Time[s]'], TestValues['GenValue'])
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

```python
def CombDelay(x_in, y_out, DataValIn, clk ):
    """
    Modfied D flip-flop repersenting a delay of z^-1
    Ports:
    ------
    x_in:    2's compliment input
    y_out:    2's compliment output
    DataValIn:  data valid input, flow control signal for the different rates
    clk:  System clock, must be equal to or greater than the max
          sample rate after interpolation or before decimation
    """
    @always(clk.posedge)
    def logic():
        if DataValIn:
            y_out.next=x_in
    return logic
```

Using the output sin genratr values with 16 bit persisoin we optain the following test data

```python
BitWidth=16
L=2**(BitWidth)
maxminV =2*L
Peeker.clear()

x_in=Signal(intbv(0, min=-L, max=L)); Peeker(x_in, 'x_in')

y_out=Signal(intbv(0, min=-maxminV, max=maxminV)); Peeker(y_out, 'y_out')
CombDelayTracker=[]

DataValIn=Signal(bool(0)); Peeker(DataValIn, 'DataValIn')
clk=Signal(bool(0)); Peeker(clk, 'clk')

DUT=CombDelay(x_in=x_in, y_out=y_out, DataValIn=DataValIn, clk=clk)

def CombDelay_TB():
    TestValueGen=TestValues['GenValue'].iteritems()

    @always(delay(1))
    def clkgen():
        clk.next = not clk

    @instance
    def stimules():
        #test the DataValIn responce
        for step, val in TestValueGen:
            x_in.next=int(val)
            if step<12:
                DataValIn.next=False
            else:
                DataValIn.next=1
            CombDelayTracker.append(int(y_out))

            yield clk.negedge

        raise StopSimulation

    return instances()

```

```python
N=TestValues.shape[0]
sim = Simulation(DUT, CombDelay_TB(), *Peeker.instances()).run()
Peeker.to_wavedrom(start_time=40, stop_time=60, tock=True)
```

```python
plt.stem(TestValues['Time[s]'], TestValues['GenValue']-np.array(CombDelayTracker))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

### Comb Section

```python
def CombSection(x_in, y_out, DataValIn, DataValOut, clk, rst, MthOrderParm=1):
    """
     Comb (difference) section of CIC with parameterizable to Mth Order

    Parameters:
    -----------
    MthOrderParm: Delay Order of the Comb section

    Ports:
    ------
    x_in:    2's compliment input
    y_out:    2's compliment output
    DataValIn:  data valid input, flow control signal for the different rates
    clk:  System clock, must be equal to or greater than the max
          sample rate after interpolation or before decimation
    rst:  System reset
    """
    #quntiztion
    Q = len(x_in)-1
    #word lengths
    L = 2**Q

    #create the dealy structer
    #Comb Delay Instansis
    Delay_i = [None for i in range(MthOrderParm)]
    #Comb Delay Instansis Interconects
    Delay_iX = [Signal(intbv(0, min=-L, max=L)) for i in range(MthOrderParm)]

    #output of delay block = to x[n-M]
    subx=Delay_iX[MthOrderParm-1]

    for i in range(MthOrderParm):
        if i ==0:
            Delay_i[i]=CombDelay(x_in=x_in, y_out=Delay_iX[i], DataValIn=DataValIn, clk=clk)
        else:
            Delay_i[i]=CombDelay(x_in=Delay_iX[i-1], y_out=Delay_iX[i], DataValIn=DataValIn, clk=clk)

    @always(clk.posedge)
    def logic():
        if rst:
            y_out.next=0
        else:
            if DataValIn:
                y_out.next=x_in-subx
                DataValOut.next=True
            else:
                DataValOut.next=False
    return instances()

```

### Comb Test

```python
BitWidth=16
L=2**(BitWidth)
maxminV =2*L
Peeker.clear()

x_in=Signal(intbv(0, min=-L, max=L)); Peeker(x_in, 'x_in')

y_out=Signal(intbv(0, min=-maxminV, max=maxminV)); Peeker(y_out, 'y_out')
CombTracker=[]

DataValIn=Signal(bool(0)); Peeker(DataValIn, 'DataValIn')
DataValOut=Signal(bool(0)); Peeker(DataValOut, 'DataValOut')

clk=Signal(bool(0)); Peeker(clk, 'clk')
rst=Signal(bool(0)); Peeker(rst, 'rst')

DUT=CombSection(x_in=x_in, y_out=y_out,
                DataValIn=DataValIn, DataValOut=DataValOut,
                clk=clk, rst=rst,
                MthOrderParm=1)

def CombSection_TB():
    TestValueGen=TestValues['GenValue'].iteritems()

    @always(delay(1))
    def clkgen():
        clk.next = not clk

    @instance
    def stimules():
        #test the DataValIn responce
        for step, val in TestValueGen:
            x_in.next=int(val)
            if step<12:
                DataValIn.next=False

            elif step==12:
                DataValIn.next=True

            elif step==100:
                rst.next=True
                DataValIn.next=False

            elif step==101:
                rst.next=False
                DataValIn.next=False

            elif step==112:
                DataValIn.next=False

            else:
                DataValIn.next=True
            CombTracker.append(int(y_out))

            yield clk.negedge

        raise StopSimulation

    return instances()

```

```python
N=TestValues.shape[0]
sim = Simulation(DUT, CombSection_TB(), *Peeker.instances()).run()
Peeker.to_wavedrom(start_time=40, stop_time=60, tock=True)
```

```python
plt.stem(TestValues['Time[s]'], TestValues['GenValue']-np.array(CombTracker))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

```python
BitWidth=16
L=2**(BitWidth)
maxminV =2*L
Peeker.clear()

x_in=Signal(intbv(0, min=-L, max=L)); Peeker(x_in, 'x_in')

y_out=Signal(intbv(0, min=-maxminV, max=maxminV)); Peeker(y_out, 'y_out')
CombTracker=[]

DataValIn=Signal(bool(0)); Peeker(DataValIn, 'DataValIn')
DataValOut=Signal(bool(0)); Peeker(DataValOut, 'DataValOut')

clk=Signal(bool(0)); Peeker(clk, 'clk')
rst=Signal(bool(0)); Peeker(rst, 'rst')

DUT=CombSection(x_in=x_in, y_out=y_out,
                DataValIn=DataValIn, DataValOut=DataValOut,
                clk=clk, rst=rst,
                MthOrderParm=3)
```

```python
N=TestValues.shape[0]
sim = Simulation(DUT, CombSection_TB(), *Peeker.instances()).run()
Peeker.to_wavedrom(start_time=40, stop_time=60, tock=True)
```

```python
plt.stem(TestValues['Time[s]'], TestValues['GenValue']-np.array(CombTracker))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

# CIC Accumultor (Integrator) Unit

## Sympy anylsis

```python
y=Function('y')
Accum1Eq=Eq(y(n), x(n)+y(n-1)); Accum1Eq
```

```python
Accum1ZEq=Eq(Y, X+z**-1 *Y); Accum1ZEq
```

```python
Accum1TFEq=Eq(H, solve(Accum1ZEq, Y)[0]/X); Accum1TFEq
```

```python
TFnum, TFden=fraction(Accum1TFEq.rhs); TFnum, TFden
TFZeros=solve(TFnum, z); TFPoles=solve(TFden)
TFPZStatment=f"""{H} has zeros from {TFnum} at {TFZeros}
and poles from {TFden} at {TFPoles}"""
print(TFPZStatment)
```

    H(z) has zeros from z at [0]
    and poles from z - 1 at [1]

```python
sups[M]=0
TFCoefN=lambda NumDen: [float(i) for i in Poly(NumDen, z).coeffs()]
TFnumC=TFCoefN(TFnum); TFdenC=TFCoefN(TFden)
for i in range(sups[M]-1):
    TFnumC.insert(1, 0)

for i in range(sups[M]):
    TFdenC.insert(1, 0)
TFnumC, TFdenC
```

```python
AccumSci=sig.TransferFunction(TFnumC, TFdenC, dt=1/4.8e3)
AccumSci
```

    TransferFunctionDiscrete(
    array([ 1.]),
    array([ 1., -1.]),
    dt: 0.00020833333333333335
    )

```python
w, mag, phase = sig.dbode(AccumSci )
fig, ax=plt.subplots(nrows=2, ncols=1, sharex=True)
plt.title(f'Bode Plot of Accum Section')
ax[0].semilogx(w, mag, label='Mag')
ax[0].set_ylabel('Mag [dB]')
ax[1].semilogx(w, phase, label='Phase')
ax[1].set_ylabel('Phase[deg]')
ax[1].set_xlabel(r'$\omega[rad/dt]$')
```

```python
plt.title(f'Impulse Respince of Accum Section')
tout, yout=sig.dimpulse(AccumSci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

```python
plt.title(f'Step Respince of Accum Section')
tout, yout=sig.dstep(AccumSci, n=100)
plt.plot(tout, yout[0])
plt.xlabel('time[sec]'); plt.ylabel('Amp')
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
```

```python
def AccumSection(x_in, y_out, DataValIn, DataValOut, clk, rst):
    """
    Simple Integrator

    Ports:
    ------
    x_in:    2's compliment input
    y_out:    2's compliment output
    DataValIn:  data valid input, flow control signal for the different rates
    clk:  System clock, must be equal to or greater than the max
          sample rate after interpolation or before decimation
    rst:  System reset
    """

    @always(clk.posedge)
    def logic():
        if rst:
            y_out.next=0
        else:
            test=0

            if DataValIn and (test<y_out.max and test>y_out.min):
                y_out.next=y_out+x_in
                DataValOut.next=True
            else:
                DataValOut.next=False

    return instances()

```

```python
BitWidth=16
L=2**(BitWidth)
maxminV =4*L
Peeker.clear()

x_in=Signal(intbv(0, min=-L, max=L)); Peeker(x_in, 'x_in')

y_out=Signal(intbv(0, min=-maxminV, max=maxminV)); Peeker(y_out, 'y_out')
AccumTracker=[]

DataValIn=Signal(bool(0)); Peeker(DataValIn, 'DataValIn')
DataValOut=Signal(bool(0)); Peeker(DataValOut, 'DataValOut')

clk=Signal(bool(0)); Peeker(clk, 'clk')
rst=Signal(bool(0)); Peeker(rst, 'rst')

DUT=AccumSection(x_in=x_in, y_out=y_out,
                DataValIn=DataValIn, DataValOut=DataValOut,
                clk=clk, rst=rst)

def AccumSection_TB():
    TestValueGen=TestValues['GenValue'].iteritems()

    @always(delay(1))
    def clkgen():
        clk.next = not clk

    @instance
    def stimules():
        #test the DataValIn responce
        for step, val in TestValueGen:
            x_in.next=int(val)
            if step<12:
                DataValIn.next=False

            elif step==12:
                DataValIn.next=True

            elif step==100:
                rst.next=True
                DataValIn.next=False

            elif step==101:
                rst.next=False
                DataValIn.next=False

            elif step==112:
                DataValIn.next=False

            else:
                DataValIn.next=True
            AccumTracker.append(int(y_out))

            yield clk.negedge

        raise StopSimulation

    return instances()

```

```python
N=TestValues.shape[0]
sim = Simulation(DUT, AccumSection_TB(), *Peeker.instances()).run()
Peeker.to_wavedrom(start_time=40, stop_time=60, tock=True)
```

```python
plt.stem(TestValues['Time[s]'], TestValues['GenValue']-np.array(AccumTracker))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

# CIC Filter Additial Segments

```python
def PassThroughSection(x_in, y_out, DataValIn, DataValOut):
    """
    Place holder insted of interpoplation or decimation

    Ports:
    ------
    x_in:    2's compliment input
    y_out:    2's compliment output
    DataValIn:  data valid input, flow control signal for the different rates
    """

    @always_comb
    def logic():
        y_out.next=x_in
        DataValOut.next=DataValIn
    return logic
```

```python
BitWidth=16
L=2**(BitWidth)
maxminV =4*L
Peeker.clear()

x_in=Signal(intbv(0, min=-L, max=L)); Peeker(x_in, 'x_in')

y_out=Signal(intbv(0, min=-maxminV, max=maxminV)); Peeker(y_out, 'y_out')
PassThroughTracker=[]

DataValIn=Signal(bool(0)); Peeker(DataValIn, 'DataValIn')
DataValOut=Signal(bool(0)); Peeker(DataValOut, 'DataValOut')

clk=Signal(bool(0)); Peeker(clk, 'clk')
rst=Signal(bool(0)); Peeker(rst, 'rst')

DUT=PassThroughSection(x_in=x_in, y_out=y_out,
                       DataValIn=DataValIn, DataValOut=DataValOut)

def PassThroughSection_TB():
    TestValueGen=TestValues['GenValue'].iteritems()

    @always(delay(1))
    def clkgen():
        clk.next = not clk

    @instance
    def stimules():
        #test the DataValIn responce
        for step, val in TestValueGen:
            x_in.next=int(val)
            if step<12:
                DataValIn.next=False

            elif step==12:
                DataValIn.next=True

            PassThroughTracker.append(int(y_out))

            yield clk.negedge

        raise StopSimulation

    return instances()

```

```python
N=TestValues.shape[0]
sim = Simulation(DUT, PassThroughSection_TB(), *Peeker.instances()).run()
Peeker.to_wavedrom(start_time=40, stop_time=60, tock=True)
```

```python
plt.stem(TestValues['Time[s]'], TestValues['GenValue']-np.array(PassThroughTracker))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

```

```python

```


```python
%config InlineBackend.figure_format = 'retina'

from matplotlib import rcParams
rcParams["savefig.dpi"] = 96
rcParams["figure.dpi"] = 96
```

# The Shin (2015) model

## Introduction

The model proposed by [Shin (2015)](http://dx.doi.org/10.1007/s12665-015-4588-z)
is an equivalent circuit that aims to reproduce SIP data This model
predicts that the complex resistivity spectra $\rho^*$ of a
polarizable rock sample can be described by

\begin{equation}
\rho^* = \sum_{i=1}^2 \frac{\rho_i}{(i\omega)^{n_i} \rho_iQ_i + 1}
\end{equation}

where $\omega$ is the measurement angular frequencies
($\omega=2\pi f$) and $i$ is the imaginary unit.

Here, $\rho^*$ depends on 3 pairs of parameters:

- $\rho_i \in [0, \infty)$, the resistivity of the resistance element in Shin's circuit.
- $Q_i \in [0, \infty)$, the capacitance of the CPE.
- $n_i \in [0, 1]$, the exponent of the CPE impedance (0 = resistor, 0.5 = warburg, 1.0 = capacitance).

In this tutorial we will perform batch inversion of all SIP data files provided with BISIP with the Shin (2015) and double Cole-Cole models, and we will compare their respective relaxation time ($\tau$) parameters.

## Exploring the parameter space

First import the required packages.

```python
import numpy as np
from bisip import PeltonColeCole
from bisip import Shin2015
from bisip import DataFiles

np.random.seed(42)
```

```python
# Load the data file paths
data_files = DataFiles()

results = {'Shin': {},
           'Pelton': {},
          }

nsteps = 1000
for fname, fpath in data_files.items():

        if fname == 'SIP-K389175':
            model = Shin2015(fpath, nsteps=nsteps)
            model.fit()
            results['Shin'][fname] = model

#             model = PeltonColeCole(fpath, nsteps=nsteps, n_modes=2)
#             model.fit()
#             results['Pelton'][fname] = model

```

    100%|██████████| 1000/1000 [00:01<00:00, 724.53it/s]

```python
fig = results['Shin']['SIP-K389175'].plot_traces()
```

```python
fig = results['Shin']['SIP-K389175'].plot_fit(discard=500)
```

```python
import matplotlib.pyplot as plt
freq = np.logspace(-2, 100, 10000)
w = 2*np.pi*freq
theta = np.array([8.16E02, 3.12E03, np.log(1.80E-14), np.log(2.25E-06), 0.3098, 0.4584])

Z = model.forward(theta, w)
plt.plot(*Z)
```

### Conclusions

```python

```

```python

```

```python

```


## Network_Analysis

```python
import networkx as nx
import numpy as np
import pandas as pd
import scipy.io as scio
import matplotlib.pyplot as plt
import seaborn as sns

import sympy as sp   #符号运算包
from sympy import exp
from scipy.integrate import odeint  #odeint是求解微分方程的函数

%matplotlib inline
plt.style.use('ggplot')
```

```python
dataFile = '../Matlab/data/ER_2000_02.mat'
data = scio.loadmat(dataFile)
data['A'].shape
```

    (2000, 2000)

```python
net = nx.Graph(data['A'])
```

```python
# 可视化
# pos=nx.spring_layout(BA)
# nx.draw(BA,pos,with_labels=True,node_size=150)
# plt.show()
```

```python
# average_shortest_path_length = nx.average_shortest_path_length(net)
# print('average shortest path length: ', average_shortest_path_length)
```

```python
#计算图的平均度
def average_degree(G):
    dict_vertex_degree = G.degree()    #提取出所有的度
    df_vertex_degree = pd.DataFrame(list(dict(dict_vertex_degree).items()),
                                    columns=['vertex','degree'])    #转换为dataframe
    aver_degree = df_vertex_degree['degree'].mean()    #求平均
    k_2 = df_vertex_degree['degree']**2
    aver_degree_square = k_2.mean()
    return aver_degree, aver_degree_square
```

```python
degrees = average_degree(net)
print("<k>：",degrees[0])
```

    <k>： 400.126

```python
print("BA: lambd_c = ",degrees[0]/degrees[1])
print("WS/ER: lambd_c = ",1/degrees[0])
```

    BA: lambd_c =  0.0024942260020121565
    WS/ER: lambd_c =  0.002499212747984385


# Rotation Matrices in Sympy

```python
from sympy import cos, sin, pi, simplify, symbols, sqrt, atan2
from sympy.matrices import Matrix
import numpy as np
```

```python
#Degrees to Radians
dtr = np.pi/180
#Radians to Degrees
rtd = 180 / np.pi

#  Joint angles
q1, q2, q3, q4 = symbols('q1:5')

# Define Rotation Matrices around X, Y, and Z
R_x = Matrix([[1,    0,         0 ],
              [0,   cos(q1), -sin(q1)],
              [0,   sin(q1),  cos(q1)]
             ])

R_y = Matrix([[cos(q2),    0,   sin(q2)],
              [   0    ,   1 ,    0    ],
              [-sin(q2) ,  0  , cos(q2)]
             ])

R_z = Matrix([[cos(q3),  -sin(q3),   0],
              [sin(q3) , cos(q3)  ,  0],
              [  0      ,   0      , 0],
             ])

```

```python
x_Rot, y_Rot, z_Rot = R_x.evalf(subs = {q1: 45*dtr}), R_y.evalf(subs = {q2: 60*dtr}), R_z.evalf(subs = {q3: 75*dtr})
print(x_Rot, y_Rot, z_Rot)
```

    Matrix([[1.00000000000000, 0, 0], [0, 0.707106781186548, -0.707106781186547], [0, 0.707106781186547, 0.707106781186548]]) Matrix([[0.500000000000000, 0, 0.866025403784439], [0, 1.00000000000000, 0], [-0.866025403784439, 0, 0.500000000000000]]) Matrix([[0.258819045102521, -0.965925826289068, 0], [0.965925826289068, 0.258819045102521, 0], [0, 0, 0]])

# Composition of Rotations

```python
#Intrinsic Rotations
#YZ_intrinsic = symbols('YZ_intrinsic')

YZ_intrinsic = R_y.evalf(subs = {q2: 45*dtr}) * R_z.evalf(subs = {q3: 60*dtr})

print(YZ_intrinsic)

YZ_extrinsic = R_z.evalf(subs = {q3: 60*dtr}) * R_y.evalf(subs = {q2: 45*dtr})
print(YZ_extrinsic)

```

    Matrix([[0.353553390593274, -0.612372435695795, 0], [0.866025403784439, 0.500000000000000, 0], [-0.353553390593274, 0.612372435695794, 0]])
    Matrix([[0.353553390593274, -0.866025403784439, 0.353553390593274], [0.612372435695795, 0.500000000000000, 0.612372435695794], [0, 0, 0]])

# Euler Angles from a Rotation Matrix

```python
# Extrinsic (Fixed Axis) X-Y-Z Rotation Matrix
R_XYZ = Matrix([[ 0.353553390593274, -0.306186217847897, 0.883883476483184],
            [ 0.353553390593274,  0.918558653543692, 0.176776695296637],
            [-0.866025403784439,               0.25, 0.433012701892219]])

# beta = atan2(-r31, sqrt(r11*r11  +  r21*r21))
# alpha = atan2(r21, r11)
# gamma = atan2(r32, r33)

'''r11 = 0.353553390593274
r21 = 0.353553390593274
r31 = -0.866025403784439
r32 = 0.25
r33 = 0.433012701892219'''

r11 = R_XYZ[0, 0]
r21 = R_XYZ[1, 0]
r31 = R_XYZ[2, 0]
r32 = R_XYZ[2, 1]
r33 = R_XYZ[2, 2]

beta = atan2(-r31, sqrt(r11 * r11  +  r21 * r21))
alpha = atan2(r21, r11)
gamma = atan2(r32, r33)

print(alpha * rtd, beta * rtd, gamma * rtd)
```

    45.0000000000000 60.0000000000000 30.0000000000000

# Homogeneous Transformations and their Inverse

```python
# Problem Statement:
  # Let P be a vector expressed in frame {B} with (x,y,z)
  # coordinates = (15.0, 0.0, 42.0)
  # Rotate P about the Y-axis by angle = 110 degrees.
  # Then translate the vector 1 unit
  # in the X-axis and 30 units in the Z-axis.
  # Print the new (x, y, z) coordinates of P after the transformation.
```

```python
pointVector = Matrix([[15.0], [0.0], [42.0], [1]])
print(pointVector)
q_y, t_x, t_y, t_z  = symbols('q_y, t_x, t_y, t_z')

transformationMatrix = Matrix([[cos(q_y),   0,   sin(q_y), t_x],
                              [   0    ,    1,      0    , t_y],
                              [-sin(q_y) ,  0 ,  cos(q_y), t_z],
                              [   0 ,       0 ,      0,     1 ]
                             ])
transformedVector = simplify(transformationMatrix * pointVector)

th = transformedVector.evalf(subs = {q_y: 110 * dtr, t_x: 1.0, t_y: 0.0, t_z: 30.0})

print(th)
```

    Matrix([[15.0000000000000], [0.0], [42.0000000000000], [1]])
    Matrix([[35.3367879231231], [0], [1.53976466853329], [1.00000000000000]])

```python
'''
The following steps are taken to obtain coordinate frame E from frame A.

From Frame A to B to E:

Frame A: Located at [0, 0, 0]
Frame B: Rotate Frame A about a_y by -90 degrees. Translate A by [-2, 2, 4]
Frame E: Rotate Frame B about b_x by 90 degrees. Translate B by [0, 2, 0]
From Frame A to C to D to E:

Frame C: Translate A by [4, 4, 0]
Frame D: Rotate Frame C about c_x by 90 degrees. Translate C by [-3, 3, 2]
Frame E: Rotate Frame D about d_Z by 90 degrees. Translate D by [-3, 2, 3]
'''
```

    '\nThe following steps are taken to obtain coordinate frame E from frame A.\n\nFrom Frame A to B to E:\n\nFrame A: Located at [0, 0, 0]\nFrame B: Rotate Frame A about a_y by -90 degrees. Translate A by [-2, 2, 4]\nFrame E: Rotate Frame B about b_x by 90 degrees. Translate B by [0, 2, 0]\nFrom Frame A to C to D to E:\n\nFrame C: Translate A by [4, 4, 0]\nFrame D: Rotate Frame C about c_x by 90 degrees. Translate C by [-3, 3, 2]\nFrame E: Rotate Frame D about d_Z by 90 degrees. Translate D by [-3, 2, 3]\n'

```python
def rot_x(q1):
    R_x = Matrix([[1,    0,         0 ],
              [0,   cos(q1), -sin(q1)],
              [0,   sin(q1),  cos(q1)],
              [0,0,0]
             ])
    return R_x

def rot_y(q2):
    R_y = Matrix([[cos(q2),    0,   sin(q2)],
              [   0    ,   1 ,    0    ],
              [-sin(q2) ,  0  , cos(q2)],
              [0,0,0]
             ])
    return R_y

def rot_z(q3):
    R_z = Matrix([[cos(q3),  -sin(q3),   0],
              [sin(q3) , cos(q3)  ,  0],
              [  0      ,   0      , 0],
              [0,0,0]
             ])
    return R_z

def zeroRotation():
    R_0 = Matrix([[1, 0, 0],
                  [0, 1, 0],
                  [0, 0, 1],
                  [0, 0, 0]
             ])
    return R_0
```

```python
frame_b_frm_a =  rot_y(-90 * dtr).col_insert(3, Matrix([-2, 2, 4, 1]))
frame_e_frm_b =  rot_x(90 * dtr).col_insert(3, Matrix([0, 2, 0, 1]))

frame_e_frm_a = frame_b_frm_a * frame_e_frm_b
print(frame_e_frm_a)
```

    Matrix([[6.12323399573677e-17, -1.00000000000000, -6.12323399573677e-17, -2], [0, 6.12323399573677e-17, -1.00000000000000, 4], [1.00000000000000, 6.12323399573677e-17, 3.74939945665464e-33, 4], [0, 0, 0, 1]])

```python
frame_c_frm_a = zeroRotation().col_insert(3, Matrix([4, 4, 0, 1]))
frame_d_frm_c = rot_x(90 * dtr).col_insert(3, Matrix([-3, 3, 2, 1]))
frame_e_frm_d = rot_z(90 * dtr).col_insert(3, Matrix([-3, 2, 3, 1]))

frame_e_frm_a_new = frame_c_frm_a * frame_d_frm_c * frame_e_frm_d

print(frame_e_frm_a - frame_e_frm_a_new)

```

    Matrix([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]])

```python

```


# Consumption Equivalent Variation (CEV)

1. Use the model in the **ConsumptionSaving.pdf** slides and solve it using **egm**
2. This notebooks estimates the *cost of income risk* through the Consumption Equivalent Variation (CEV)

We will here focus on the cost of income risk but the CEV can be used to estimate the value of many different aspects of an economy. For eaxample, [Oswald (2019)](http://qeconomics.org/ojs/index.php/qe/article/view/701 "The option value of homeownership") estimated the option value of homeownership using a similar strategy as described below.

**Goal:** To estimate the CEV by comparing the *value of life* under the baseline economy and an alternative economy with higher permanent income shock variance along with a consumption compensation.

**Value of Life:**
1. Let the *utility function* be a generalized version of the CRRA utility function with $\delta$ included as a potential consumption compensation.
\begin{equation}
{u}(c,\delta) = \frac{(c\cdot(1+\delta))^{1-\rho}}{1-\rho}
\end{equation}
2. Let the *value of life* of a synthetic consumer $s$ for a given level of permanent income shock varaince, $\sigma_{\psi}$, and $\delta$, be
\begin{equation}
{V}_{s}({\sigma}_{\psi},\delta)=\sum_{t=1}^T \beta ^{t-1}{u}({c}^{\star}_{s,t}({\sigma}_{\psi},\delta),\delta)
\end{equation}
where ${c}^{\star}_{s,t}({\sigma}_{\psi},\delta)$ is optimal consumption found using the **egm**. The value of life is calcualted in the function `value_of_life(.)` defined below.

**Consumption Equivalent Variation:**
1. Let $V=\frac{1}{S}\sum_{s=1}^SV(\sigma_{\psi},0)$ be the average value of life under the *baseline* economy with the baseline value of $\sigma_{\psi}$ and $\delta=0$.
2. Let $\tilde{V}(\delta)=\frac{1}{S}\sum_{s=1}^SV(\tilde{\sigma}_{\psi},\delta)$ be the average value of life under the *alternative* economy with $\tilde{\sigma}_{\psi} > \sigma_{\psi}$.

The CEV is the value of $\delta$ that sets $V=\tilde{V}(\delta)$ and can be estimated as
\begin{equation}
\hat{\delta} = \arg\min_\delta (V-\tilde{V}(\delta))^2
\end{equation}
where the objective function is calculated in `obj_func_cev(.)` defined below.

# Setup

```python
%matplotlib inline
%load_ext autoreload
%autoreload 2

import time
import numpy as np
import scipy.optimize as optimize

import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']

import sys
sys.path.append('../')

import ConsumptionSavingModel as csm
from ConsumptionSavingModel import ConsumptionSavingModelClass
```

# Setup the baseline model and the alternative model

```python
par = {'simT':40}
model = ConsumptionSavingModelClass(name='baseline',solmethod='egm',**par)

# increase the permanent income with 100 percent and allow for consumption compensation
par_cev = {'sigma_psi':0.2,'do_cev':1,'simT':40}
model_cev = ConsumptionSavingModelClass(name='cev',solmethod='egm',**par_cev)
```

```python
model.solve()
model.simulate()
```

    model solved in 6.1 secs
    model simulated in 5.0 secs

# Average value of life

**Define Functions:** value of life and objective function used to estimate "cev"

```python
def value_of_life(model):
    # utility associated with consumption for all N and T
    util = csm.utility(model.sim.c,model.par)

    # discounted sum of utility
    disc = np.ones(model.par.simT)
    disc[1:] = np.cumprod(np.ones(model.par.simT-1)*model.par.beta)

    disc_util = np.sum(disc*util,axis=1)

    # return average of discounted sum of utility
    return np.mean(disc_util)

def obj_func_cev(theta,model_cev,value_of_life_baseline):
    # update cev-parameter
    setattr(model_cev.par,'cev',theta)

    # re-solve and simulate alternative model
    model_cev.solve(do_print=False)
    model_cev.simulate(do_print=False)

    # calculate value of life
    value_of_life_cev = value_of_life(model_cev)

    # return squared difference to baseline
    return (value_of_life_cev - value_of_life_baseline)*(value_of_life_cev - value_of_life_baseline)
```

**Baseline value of life and objective function at cev=0**

```python
value_of_life_baseline = value_of_life(model)

obj_func_cev(0.0,model_cev,value_of_life_baseline)
```

    3.4684611543061683

```python
# plot the objective function
grid_cev = np.linspace(0.0,0.2,20)
grid_obj = np.empty(grid_cev.size)
for j,cev in enumerate(grid_cev):
    grid_obj[j] = obj_func_cev(cev,model_cev,value_of_life_baseline)

plt.plot(grid_cev,grid_obj);
```

# Estimate the Consumption Equivalent Variation (CEV)

```python
res = optimize.minimize_scalar(obj_func_cev, bounds=[-0.01,0.5],
                               args=(model_cev,value_of_life_baseline),method='golden')
res
```

         fun: 9.213801022894747e-18
        nfev: 48
         nit: 43
     success: True
           x: 0.0975865281411968

The estimated CEV suggests that consumers would be indifferent between the baseline economy and a 100% increase in the permanent income shock variance along with a 10% increase in consumption in all periods.

DynamicProgramming/extra/Consumption Equivalent Variation (CEV).ipynb


# Approximate PDF for the invariant mass distribution of combinatorial background

We derive an approximate pdf for the invariant mass distribution from combinatorial background. Combinatorial background refers to random pairs of unrelated particles that are combined to compute the invariant mass of the hypothetical ancestor particle.

It is known empirically that an exponential distribution often provides a good description of this background. We show here why that is. We work in coordinates where $c=1$.

We express the 3-momentum of a particle in $p_T, \eta, \phi$ since the probability distributions approximately factorize:

$$
g(p_T) \, h_1(\phi) \, h_2(\eta)
$$

where $h_1(\phi)$ is a uniform distribution from $-\pi, \pi$ and $h_2(\eta)$ is a somewhat bell-shaped pdf around 0 with a broadened center and with a dip at zero. We will see that the result does not depend on $h_2(\eta)$ within the limits of the approximation.

The distribution $g(p_T)$ is approximately exponential for $p_T \gg 1\,\text{GeV}c^{-1}$. We will assume it is exponential, which is the first approximation.

We now express the invariant mass $m$ of the hypothetical ancestor particle in these coordinates.

```python
from sympy import *
```

```python
def ancestor_mass(pt1, eta1, phi1, pt2, eta2, phi2, m1, m2):
    # see e.g. Wikipedia https://en.wikipedia.org/wiki/Pseudorapidity

    px1 = cos(phi1) * pt1
    py1 = sin(phi1) * pt1
    pz1 = sinh(eta1) * pt1

    px2 = cos(phi2) * pt2
    py2 = sin(phi2) * pt2
    pz2 = sinh(eta2) * pt2

    # momentum squared of ancestor
    p_sq = (px1 + px2) ** 2 + (py1 + py2) ** 2 + (pz1 + pz2) ** 2

    # momentum squared of particles 1 and 2
    p1_sq = px1 ** 2 + py1 ** 2 + pz1 ** 2
    p2_sq = px2 ** 2 + py2 ** 2 + pz2 ** 2

    # energies of particles 1 and 2
    e1 = sqrt(p1_sq + m1 ** 2)
    e2 = sqrt(p2_sq + m2 ** 2)

    return sqrt((e1 + e2) ** 2 - p_sq)
```

We use SymPy to transform and simplify the formula for the ancestor mass. First we declare the variables.

```python
pt1, pt2, m1, m2, lambd, m = symbols("p_T_1 p_T_2 m_1 m_2 lambda m", positive=True, real=True)

eta1, eta2, phi1, phi2 = symbols("eta_1 eta_2 phi_1 phi_2", real=True)
```

```python
ancestor_mass(pt1, eta1, phi1, pt2, eta2, phi2, m1, m2)
```

$\displaystyle \sqrt{- \left(p_{T 1} \sin{\left(\phi_{1} \right)} + p_{T 2} \sin{\left(\phi_{2} \right)}\right)^{2} - \left(p_{T 1} \cos{\left(\phi_{1} \right)} + p_{T 2} \cos{\left(\phi_{2} \right)}\right)^{2} - \left(p_{T 1} \sinh{\left(\eta_{1} \right)} + p_{T 2} \sinh{\left(\eta_{2} \right)}\right)^{2} + \left(\sqrt{m_{1}^{2} + p_{T 1}^{2} \sin^{2}{\left(\phi_{1} \right)} + p_{T 1}^{2} \cos^{2}{\left(\phi_{1} \right)} + p_{T 1}^{2} \sinh^{2}{\left(\eta_{1} \right)}} + \sqrt{m_{2}^{2} + p_{T 2}^{2} \sin^{2}{\left(\phi_{2} \right)} + p_{T 2}^{2} \cos^{2}{\left(\phi_{2} \right)} + p_{T 2}^{2} \sinh^{2}{\left(\eta_{2} \right)}}\right)^{2}}$

The raw expression for the ancestor mass is rather complex. We ask SymPy to simplify it.

```python
m_expr = ancestor_mass(pt1, eta1, phi1, pt2, eta2, phi2, m1, m2).simplify(); m_expr
```

$\displaystyle \sqrt{m_{1}^{2} + m_{2}^{2} - 2 p_{T 1} p_{T 2} \cos{\left(\phi_{1} - \phi_{2} \right)} - 2 p_{T 1} p_{T 2} \sinh{\left(\eta_{1} \right)} \sinh{\left(\eta_{2} \right)} + 2 \sqrt{m_{1}^{2} + p_{T 1}^{2} \cosh^{2}{\left(\eta_{1} \right)}} \sqrt{m_{2}^{2} + p_{T 2}^{2} \cosh^{2}{\left(\eta_{2} \right)}}}$

This looks much better.

We now introduce the second approximation. We assume that $m_i \ll p_{T,i}$, which plays along nicely with the first approximation $p_{T,i} \gg 1\,\text{GeV}c^{-1}$ if the outgoing particles are light. In other words, this approximation can be expected to work well for decays into particles with masses $m_i \ll 1\,\text{GeV}c^{-2}$.

We expand the expression to first order in $m_{i}$ in a Taylor series. The first order happens to be zero, so we simply get:

```python
m_approx_expr = m_expr.subs(m1, 0).subs(m2, 0).simplify(); Eq(m, m_approx_expr)
```

$\displaystyle m = \sqrt{2} \sqrt{p_{T 1}} \sqrt{p_{T 2}} \sqrt{- \cos{\left(\phi_{1} - \phi_{2} \right)} + \cosh{\left(\eta_{1} - \eta_{2} \right)}}$

To compute the pdf of the ancestor mass $m$, we need to perform a change of variable and then integrate over the remaining coordinates.

We first make the change of variables.

The old variables are $p_{T,1}, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2$.

The new variables are $m, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2$.

The pdf in the new variables is

$$
f'(m, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2) = g(p_{T,1}) \, g(p_{T,2}) \, h_1(\phi_1) \, h_1(\phi_2) \, h_2(\eta_1) \, h_2(\eta_2) \,
\left|\frac{\partial(m, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2)}{\partial(p_{T,1}, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2)} \right|^{-1} \\
f'(m, p_{T,2}, \eta_1, \eta_2, \phi_1, \phi_2) = g(p_{T,1}(m, p_{T,2}, \dots)) \, g(p_{T,2}) \, h_1(\phi_1) \, h_1(\phi_2) \, h_2(\eta_1) \, h_2(\eta_2) \, \left| \frac{\partial m}{\partial p_{T, 1}} \right|^{-1}
$$

We manually express $\sqrt{p_{T, 1}}$ as a function of $m$, to replace it later. The SymPy solver has trouble doing this, as it tends to break down the $\cosh$ into exponentials, but we want to keep it intact.

```python
sqrt_pt1 = m / m_approx_expr * sqrt(pt1); Eq(sqrt(pt1), sqrt_pt1)
```

$\displaystyle \sqrt{p_{T 1}} = \frac{\sqrt{2} m}{2 \sqrt{p_{T 2}} \sqrt{- \cos{\left(\phi_{1} - \phi_{2} \right)} + \cosh{\left(\eta_{1} - \eta_{2} \right)}}}$

```python
pt1_m = sqrt_pt1 ** 2; Eq(pt1, pt1_m)
```

$\displaystyle p_{T 1} = \frac{m^{2}}{2 p_{T 2} \left(- \cos{\left(\phi_{1} - \phi_{2} \right)} + \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)}$

We compute $\left| \frac{\partial m}{\partial p_{T, 1}} \right|^{-1}$ and express it as a function of $m$.

```python
dpt1_dm = 1/diff(m_approx_expr, pt1); dpt1_dm
```

$\displaystyle \frac{\sqrt{2} \sqrt{p_{T 1}}}{\sqrt{p_{T 2}} \sqrt{- \cos{\left(\phi_{1} - \phi_{2} \right)} + \cosh{\left(\eta_{1} - \eta_{2} \right)}}}$

```python
dpt1_dm = dpt1_dm.subs(sqrt(pt1), sqrt_pt1); dpt1_dm
```

$\displaystyle \frac{m}{p_{T 2} \left(- \cos{\left(\phi_{1} - \phi_{2} \right)} + \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)}$

```python
# exponential pdf
def g(pt):
    return lambd * exp(-lambd * pt)
```

```python
# uniform pdf from -pi, pi
def h1(phi):
    return 1/(2 * pi)
```

```python
# unspecified pdf for eta
h2 = Function("h2")
```

```python
f_prime = simplify(dpt1_dm * g(pt1_m) * g(pt2) * h1(phi1) * h1(phi2) * h2(eta1) * h2(eta2)); f_prime
```

$\displaystyle - \frac{\lambda^{2} m \operatorname{h_{2}}{\left(\eta_{1} \right)} \operatorname{h_{2}}{\left(\eta_{2} \right)} e^{\lambda \left(\frac{m^{2}}{2 p_{T 2} \left(\cos{\left(\phi_{1} - \phi_{2} \right)} - \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)} - p_{T 2}\right)}}{4 \pi^{2} p_{T 2} \left(\cos{\left(\phi_{1} - \phi_{2} \right)} - \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)}$

To obtain $f(m)$, we need to integrate $f'$ over $p_{T,2}, \phi_1, \phi_2, \eta_1, \eta_2$.

```python
Integral(f_prime,
         (pt2, 0, oo),
         (eta1, "eta_a", "eta_b"), (eta2, "eta_a", "eta_b"),
         (phi1, -pi, pi), (phi2, -pi, pi))
```

$\displaystyle \int\limits_{- \pi}^{\pi}\int\limits_{- \pi}^{\pi}\int\limits_{\eta_{a}}^{\eta_{b}}\int\limits_{\eta_{a}}^{\eta_{b}}\int\limits_{0}^{\infty} \left(- \frac{\lambda^{2} m \operatorname{h_{2}}{\left(\eta_{1} \right)} \operatorname{h_{2}}{\left(\eta_{2} \right)} e^{\lambda \left(\frac{m^{2}}{2 p_{T 2} \left(\cos{\left(\phi_{1} - \phi_{2} \right)} - \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)} - p_{T 2}\right)}}{4 \pi^{2} p_{T 2} \left(\cos{\left(\phi_{1} - \phi_{2} \right)} - \cosh{\left(\eta_{1} - \eta_{2} \right)}\right)}\right)\, dp_{T 2}\, d\eta_{1}\, d\eta_{2}\, d\phi_{1}\, d\phi_{2}$

This integral does not look analytically tractable. Since it contains an exponential function, Laplace's method comes to mind as an approximation. To apply it, the function $g(\vec x)$ in the exponential $e^{M g(\vec x)}$ must have a global maximum inside the integration bounds. We investigate $g(\vec x)$ numerically to see whether that is the case.

```python
def g(x, y, z):
    return 1/(x * (np.cos(z) - np.cosh(y) + 1e-100)) - x
```

```python
import numpy as np
import matplotlib.pyplot as plt

xn = np.linspace(0.05, 3, 100)
yn = np.linspace(2, 5)
zn = np.linspace(-2*np.pi, 2* np.pi)

fig, ax = plt.subplots(1, 3, figsize=(14, 4))
plt.sca(ax[0])
Y, Z = np.meshgrid(yn, zn)
plt.pcolormesh(Y, Z, g(0.5, Y, Z), shading="nearest")
plt.colorbar()
plt.sca(ax[1])
X, Y = np.meshgrid(xn, yn)
plt.pcolormesh(X, Y, g(X, Y, 0), shading="nearest")
plt.colorbar()
plt.sca(ax[2])
for yi in yn[::10]:
    plt.plot(xn, g(xn, yi, 0))
```

There is indeed a global maximum as a function of $p_{T,2}$ between 0 and infinity, but not with respect to $\Delta\eta$. The maximum for $\Delta\eta$ is located at the upper bound of the $\eta$ interval, which means that Laplace's method cannot be applied to the integral as a whole.

We cannot solve the integral completely, so we do the next-best thing. We partially solve it by approximately carrying out the integration over $p_{T,2}$ for each value of $C(\Delta\eta, \Delta\phi) = \cosh\Delta\eta - \cos\Delta\phi$ with Laplace's method:

$$
\int h(x) e^{M g(x)} \text{d}x \approx \sqrt{\frac{2\pi}{M |g''(x_0)|}} h(x_0) e^{M g(x_0)}
$$

with $M > 0, M \rightarrow \infty, h(x) > 0$ and $x_0$ being the maximum of $g(x)$.

To obtain simpler formulas, we make a bunch of substitutions:

* $x := p_{T,2}$
* $\Delta \eta := \eta_1 - \eta_2$
* $\Delta \phi := \phi_1 - \phi_2$
* $C(\Delta\eta, \Delta\phi) := \cosh \Delta\eta - \cos\Delta\phi$

```python
deta, dphi = symbols(r"\Delta\eta \Delta\phi")

x = symbols("x", real=True, positive=True)
C = Function("C", real=True, positive=True)(deta, dphi)

f_prime = f_prime.subs((cosh(eta1 - eta2)-cos(phi1 - phi2)), C).subs(eta1, eta2 + deta).subs(phi1, phi2 + dphi).subs(pt2, x).simplify(); f_prime
```

$\displaystyle \frac{\lambda^{2} m \operatorname{h_{2}}{\left(\eta_{2} \right)} \operatorname{h_{2}}{\left(\Delta\eta + \eta_{2} \right)} e^{- \lambda \left(\frac{m^{2}}{2 x C{\left(\Delta\eta,\Delta\phi \right)}} + x\right)}}{4 \pi^{2} x C{\left(\Delta\eta,\Delta\phi \right)}}$

```python
M = lambd
```

```python
g = -m ** 2 / (2 * x * C) - x; g
```

$\displaystyle - \frac{m^{2}}{2 x C{\left(\Delta\eta,\Delta\phi \right)}} - x$

```python
h = f_prime / exp(M * g).simplify(); h
```

$\displaystyle \frac{\lambda^{2} m \operatorname{h_{2}}{\left(\eta_{2} \right)} \operatorname{h_{2}}{\left(\Delta\eta + \eta_{2} \right)}}{4 \pi^{2} x C{\left(\Delta\eta,\Delta\phi \right)}}$

```python
assert simplify(h * exp(M * g)) == f_prime
```

```python
gx = diff(g, x); gx
```

$\displaystyle \frac{m^{2}}{2 x^{2} C{\left(\Delta\eta,\Delta\phi \right)}} - 1$

```python
gxx = diff(g, x, 2); gxx
```

$\displaystyle - \frac{m^{2}}{x^{3} C{\left(\Delta\eta,\Delta\phi \right)}}$

```python
x0 = solve(Eq(gx, 0), x)[0]; x0 # there is only one solution, since x > 0
```

$\displaystyle \frac{\sqrt{2} m}{2 \sqrt{C{\left(\Delta\eta,\Delta\phi \right)}}}$

```python
gxx0 = gxx.subs(x, x0); gxx0
```

$\displaystyle - \frac{2 \sqrt{2} \sqrt{C{\left(\Delta\eta,\Delta\phi \right)}}}{m}$

```python
h0 = h.subs(x, x0); h0
```

$\displaystyle \frac{\sqrt{2} \lambda^{2} \operatorname{h_{2}}{\left(\eta_{2} \right)} \operatorname{h_{2}}{\left(\Delta\eta + \eta_{2} \right)}}{4 \pi^{2} \sqrt{C{\left(\Delta\eta,\Delta\phi \right)}}}$

```python
g0 = g.subs(x, x0); g0
```

$\displaystyle - \frac{\sqrt{2} m}{\sqrt{C{\left(\Delta\eta,\Delta\phi \right)}}}$

Now we have everything to get Laplace's approximate solution.

```python
sqrt(2 * pi / (M * -gxx0)) * h0 * exp(M * g0).simplify()
```

$\displaystyle \frac{\sqrt[4]{2} \lambda^{\frac{3}{2}} \sqrt{m} \operatorname{h_{2}}{\left(\eta_{2} \right)} \operatorname{h_{2}}{\left(\Delta\eta + \eta_{2} \right)} e^{- \frac{\sqrt{2} \lambda m}{\sqrt{C{\left(\Delta\eta,\Delta\phi \right)}}}}}{4 \pi^{\frac{3}{2}} C^{\frac{3}{4}}{\left(\Delta\eta,\Delta\phi \right)}}$

In other words, the approximate solution for each value of $C(\Delta\eta, \Delta\phi)$ is

$$
f(m, C) \propto m^{1/2} \, e^{- m \, \lambda_C}
$$

where $\lambda_C$ is proportional to $C^{-1/2}$. Since the function has a broad maximum, the actual function $f(m)$ will be a superposition of many $f(m, C)$ with similar slopes.

```python
import numpy as np
import matplotlib.pyplot as plt

yn = np.linspace(2, 5)
plt.plot(yn, 1/np.sqrt(np.cosh(yn) - 1), label=r"$C^{-1/2}(\eta, 0)$")
plt.legend()
plt.xlabel(r"$\eta$");
```

## Simulation study

We confirm the analytical results in a toy simulation. We first generate some toy data. The $p_T$ distribution is exponential. The distribution in $\eta$ and $\phi$ are uniform. We then form all unique pairs and compute the invariant mass for the case with candidate masses assumed to be zero.

```python
import numpy as np

rng = np.random.default_rng(1)

pt = rng.exponential(size=1000)
eta = rng.uniform(2, 5, size=1000)
phi = rng.uniform(-np.pi, np.pi, size=1000)

f = lambdify((pt1, eta1, phi1, pt2, eta2, phi2), m_approx_expr, "math")

# this is slow in pure Python, but we don't care for a one-shot
m = []
for i in range(1000):
    for j in range(i):
        m.append(f(pt[i], eta[i], phi[i], pt[j], eta[j], phi[j]))
m = np.array(m)
```

We make a histogram of the resulting distribution.

```python
n, xe = np.histogram(m, bins=50)

from pyik.mplext import plot_hist

plot_hist(xe, n)
plt.xlabel("m")
plt.ylabel("counts")
plt.semilogy();
```

It looks approximately like an exponential, but there are differences. The peak at zero is cut off and the slope varies, being steeper near zero and flatter in the tail. In a narrow interval, however, the exponential distribution is expected to give a good description.

We fit a simple exponential model and a more complex model that takes our analytical solution in account. It is a superposition of three distributions of the form $A_i \sqrt{m} e^{-B_i m}$ with approximate the integral that we cannot compute analytically.

```python
from iminuit import Minuit
from iminuit.cost import LeastSquares

def model_A(x, a, b):
    return a * np.exp(- x * b)

def model_B(x, par):
    npar = len(par) // 2
    par = np.asarray(par)[:, None]
    return x ** 0.5 * np.sum(par[:npar] * np.exp(-x * par[npar:]), axis=0)

cx = (xe[1:] + xe[:-1]) / 2
nerr = (n + 0.5) ** 0.5
```

```python
min1 = Minuit(LeastSquares(cx, n, nerr, model_A), 1, 1)
min1.limits = (0, None)
min1.migrad()
```

         FCN = 7712
         Nfcn = 177

         EDM = 4.59e-07 (Goal: 0.0002)

         Valid Minimum
         Valid Parameters
         No Parameters at limit

         Below EDM threshold (goal x 10)
         Below call limit

         Covariance
         Hesse ok
         Accurate
         Pos. def.
         Not forced

         Name
         Value
         Hesse Error
         Minos Error-
         Minos Error+
         Limit-
         Limit+
         Fixed

         0
         a
         153.21e3
         0.30e3

         0

         1
         b
         691.0e-3
         0.9e-3

         0

         a
         b

         a
         8.84e+04
         0.182 (0.678)

         b
         0.182 (0.678)
         8.14e-07

```python
min2 = Minuit(LeastSquares(cx, n, nerr, model_B), np.ones(6))
min2.limits = (0, None)
min2.migrad(ncall=10000)
```

         FCN = 149
         Nfcn = 1583

         EDM = 8.62e-06 (Goal: 0.0002)

         Valid Minimum
         Valid Parameters
         No Parameters at limit

         Below EDM threshold (goal x 10)
         Below call limit

         Covariance
         Hesse ok
         Accurate
         Pos. def.
         Not forced

         Name
         Value
         Hesse Error
         Minos Error-
         Minos Error+
         Limit-
         Limit+
         Fixed

         0
         x0
         3.2e3
         1.5e3

         0

         1
         x1
         197e3
         6e3

         0

         2
         x2
         123e3
         5e3

         0

         3
         x3
         0.500
         0.033

         0

         4
         x4
         1.744
         0.033

         0

         5
         x5
         0.864
         0.017

         0

         x0
         x1
         x2
         x3
         x4
         x5

         x0
         2.35e+06
         -6.8e+06 (-0.769)
         4.87e+06 (0.607)
         49.3 (0.982)
         29.4 (0.583)
         23.5 (0.900)

         x1
         -6.8e+06 (-0.769)
         3.32e+07
         -2.85e+07 (-0.946)
         -128 (-0.680)
         -163 (-0.859)
         -93.8 (-0.956)

         x2
         4.87e+06 (0.607)
         -2.85e+07 (-0.946)
         2.74e+07
         86.3 (0.503)
         165 (0.958)
         78.7 (0.884)

         x3
         49.3 (0.982)
         -128 (-0.680)
         86.3 (0.503)
         0.00107
         0.000538 (0.499)
         0.00046 (0.825)

         x4
         29.4 (0.583)
         -163 (-0.859)
         165 (0.958)
         0.000538 (0.499)
         0.00108
         0.000464 (0.830)

         x5
         23.5 (0.900)
         -93.8 (-0.956)
         78.7 (0.884)
         0.00046 (0.825)
         0.000464 (0.830)
         0.00029

```python
fig, ax = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)
plt.sca(ax[0])
plot_hist(xe, n)
xm = np.linspace(xe[0], xe[-1], 1000)
plt.plot(xm, model_A(xm, *min1.values), label=f"exponential: $\\chi^2/n_{{dof}} = {min1.fval:.1f}/{len(cx) - min1.nfit}$")
plt.plot(xm, model_B(xm, min2.values), label=f"superposition: $\\chi^2/n_{{dof}} = {min2.fval:.1f}/{len(cx) - min2.nfit}$")
plt.semilogy()
plt.legend()
plt.xlabel("m")
plt.sca(ax[1])
plot_hist(xe, n)
plt.plot(xm, model_A(xm, *min1.values))
plt.plot(xm, model_B(xm, min2.values))
plt.xlabel("m")
```

The sophisticated superposition model globally describes the distribution much better than the plain exponential model. The cut-off near zero is modeled by the term $\sqrt{m}$, which suppresses the rise of the pure exponentials near zero. The superposition of three exponentials with different slopes is able to match the changing slope.

Even for the sophisticated model, $\chi^2$ does not look so good, but that is because  we use the approximate least-squares method here to keep it simple, in which we do not integrate the model over each bin. The implied approximation is bad near $m = 0$, where the distribution has a sharp peak.

For an approximation over a local interval not too close to $m = 0$, a simple exponential distribution will also perform well, since locally one of the exponentail distributions is dominant.

invariant mass combinatorial background.ipynb


# Determine the 2D Basis of a Plane

The first step in simulating interfaces is the determination of the two-dimensional periodicity (i.e. the basis vectors) of the plane. The interfaces are two-dimensional sections of the underlying three-dimensional lattice, and hence, the interface will exhibit the periodicity of the corresponding 2D lattice of the plane. The technique to determine the basis vectors is outlined in the following article:

[**An efficient algorithm for computing the primitive bases of a general lattice plane.**](https://scripts.iucr.org/cgi-bin/paper?rg5087)
Journal of Applied Crystallography
Banadaki, A. D., & Patala, S. (2015). , 48(2), 585-588.

In this tutorial, we will discuss the steps involved in determining the basis vectors of the 2D plane using the **byxtal** package. Please follow the installation steps (link needed!!) to acquire the byxtal package and import all the required packages that we need for completing this tutorial.

## Miller Indices and Conventions:

1. Miller Indices are often used to refer to a given crystallographic plane in crystals.
2. However, various conventions are commonly used in determining the Miller Indices that can change the indices of the plane. For example, in the FCC lattice, one could either use the cubic unit-cell or the primitive cell to index planes and directions, resulting in completely different indices for the same plane. Therefore, we would like to declare our conventions in defining the Miller Indices to avoid potential confusion.
3. By definition, the Miller indices of a plane, denoted by $(h \, k \, l)$, refer to the indices of the lattice vector perpendicular to the plane expressed in the reciprocal lattice. Therefore, the indices will depend on the reference lattice used (e.g. the cubic unit-cell or the primitive cell).
4. In the **byxtal** package, we perform the calculations in the primitve cell. The reason simply is that, in the primitive lattice, all the lattice points are expressed using integers. This helps with some of the algebraic manipulations that are preformed in the package.
5. **Unless otherwise specified, the indices in byxtal package are in reference to the primitve cell.**
6. In the present tutorial, we also discuss how to convert the indices from one reference frame to the other (e.g. from the primitive cell to the cubic unit-cell).

[//]: # "Miller Indices are defined as the reciprocal of intercepts of a crystallographic plane with the Unit Cell Basis Vectors. The confusion is often caused by the definition of the Unit Cell. In f.c.c and b.c.c lattices primitive basis vectors are non-orthogonal while the supercell basis vectors are orthogonal. Most importantly since the reciprocal and direct lattices are the identical in f.c.c and b.c.c, the Miller Indices (defined in supercell unit cell) are the same as the plane normal indices. This unique property and convenience of using an orthogonal set of basis vectors is the root cause of the bipartisan approach to defining the Miller Indices. The downfall of such an approach is the fact that it does not have much utility in other types of lattices e.g. hcp. Therefore in GBpy whenever we use the term Miller Indices, we are referring to the reciprocals of intercepts of a crystallographic plane with the primitive Basis Vectors."
[//]: # (Miller Indices: reciprocal of intercepts of a crystallographic plane with the primitive Basis Vectors, or alternatively, Miller Indices: normal vector indices of the plane defined in primitive reciprocal lattice. The above interchangeable definitions are consistently used in GBpy for referring to Miller Indices. Other conventions in defining the Miller Indices are ultimately converted to the above definition. In the present tutorial we manually extract various Indices and demonstrate how they can be fed to the GBpy. In practice such conversions are not necessary as long as the user is consistent with the definitions of GBpy, and can be used for verifying the answer.)

Let's start with importing the **byxtal** package and other modules that we will use in this tutorial.

```python
import sys
sys.path.insert(0, './../../../../gbpy/')
import numpy as np
from sympy.matrices import Matrix, eye, zeros;
```

## Problem Definition:

In the tutorial, we will determine the planar basis of a crystallographic plane in the FCC lattice. Consider the plane whose normal vector is along the direction $[2 3 1]$ expressed in the cubic unit-cell reference frame. Since the normal vector to the plane is provided in the orthogonal unit-cell basis, we will first determine the Miller Indices of the plane (using the primitive cell bases).

### Finding Miller Indices:

A vector in the space can be expressed in any basis, of course with varying components. Vector $\vec{v}$ in basis A can be expressed as:

\begin{equation}
\vec{v} = \mathcal{B}_A v_A
\end{equation}

Similarly we can define the plane normal $\vec{n}$ in any basis. For instance we can define $\vec{n}$ in unit-cell basis ($\mathcal{B}_{PO}$) or in primitive-cell basis $\mathcal{B}_P$; we can write:

\begin{equation}
\vec{n} = \mathcal{B}_{PO} n_{PO} = \mathcal{B}_{P} n_{P}
\end{equation}

The conversion from one basis to the other can be determined by using the components of the basis vectors of one of the frames (e.g. $P$) in the other frame (e.g. $PO$):

\begin{equation}
\mathcal{B}_{P} = \mathcal{B}_{PO} \Lambda_{P}^{PO}
\end{equation}

where, $\Lambda_P^{PO}$ is a $3 \times 3$ matrix with its columns representing the components of basis vectors of $P$ frame in the $PO$ basis. For example, for an FCC lattice, $\Lambda_P^{PO}$ is given below.

```python
l_p_po = 1.0 * Matrix([[0.,0.5,0.5],[0.5,0.,0.5],[0.5,0.5,0.]])
l_p_po
```

$\displaystyle \left[\begin{matrix}0 & 0.5 & 0.5\\0.5 & 0 & 0.5\\0.5 & 0.5 & 0\end{matrix}\right]$

We can now determine the components of the vector $\vec{n}$ in the $P$ reference frame as follows:

\begin{align}
\mathcal{B}_{P} n_{P} &= \mathcal{B}_{PO} n_{P0} \\ \nonumber
\mathcal{B}_{PO} \Lambda_{P}^{PO} n_{P} &= \mathcal{B}_{PO} n_{P0} \\ \nonumber
\Lambda_{P}^{PO} n_{P} &= \mathcal{B}_{PO} n_{P0} \\ \nonumber
n_{P} &= \Lambda_{PO}^{P} n_{P0}
\end{align}

where $\Lambda_{P}^{PO} = \left( \Lambda_{PO}^{P} \right)^{-1}$.

1. **To determine the Miller indices**, we have to express the components of the normal vector $\vec{n}$ in the reference frame of the reciprocal lattice (the reciprocal of the primitive cell).

2. The basis vectors of the reciprocal of the primitve lattice are denoted using the symbol $\mathcal{B}^*_{P}$, and are given in the $PO$ reference frame as:

\begin{equation}
\mathcal{B}^*_{P} = \mathcal{B}_{PO} \Lambda_{P*}^{PO}
\end{equation}

3. $\Lambda_{P*}^{PO}$ can be computed using the byxtal package using the function `bxt.find_csl_dsc.reciprocal_mat()`. For the sake of convenience we abbreviate the imported module `bxt.find_csl_dsc` as `fcd`. The code is shown below.

```python
import byxtal.find_csl_dsc as fcd
l_rp_po = fcd.reciprocal_mat(l_p_po)
l_rp_po
```

where we use the variable `l_rp_po` to represent $\Lambda_{P*}^{PO}$. Now, we can determine the indices of $\vec{n}$ in the $P^*$ reference frame, using equation (4) as:

\begin{equation}
n_{P^*} = \Lambda_{PO}^{P*} n_{P0}
\end{equation}

Use the following code to determine the components $n_{P^*}$:

```python
l_po_rp = (l_rp_po).inv()
n_po = Matrix([[2], [3], [1]])
n_rp = l_po_rp*n_po
n_rp
```

$\displaystyle \left[\begin{matrix}2.0\\1.5\\2.5\end{matrix}\right]$

Remember, that the Miller Indices ought to be integers (without common factors). We have to find a common scaling factor for all the components such that the result is going to be scalar. We have implemented a function named `int_finder` that performs this task for a variety of input types (e.g. rows and columns of matrices). For irrational numbers int_finder accepts a tolerance and performs the same operation on the closest rational number within the specified tolerance. You can find this function in the package as: `byxtal.integer_manipulations.int_finder()`. Therefore, we repeat the previous steps and pass the results to the `int_finder` function to obtain the integer Miller indices.

```python
import byxtal.integer_manipulations as iman
ni_rp = iman.int_finder(n_rp)
ni_rp
```

## Finding the Planar Basis:

1. From the previous section, we found the Miller Indices of an FCC plane with the normal along $n_{PO} = [2 3 1]$ to be $(4 3 5)$.

2. Now all we have to do is to pass the obtained indices to `bp_basis`, which is a function that gets the Miller Indices (expressed using the primitive cell) as the input and returns a $3 \times 2$ matrix, where the columns represent the components of the basis vectors in the primitive $(P)$ reference frame.

3. Also the obtained vectors are in the [reduced form](https://en.wikipedia.org/wiki/Lenstra%E2%80%93Lenstra%E2%80%93Lov%C3%A1sz_lattice_basis_reduction_algorithm). You can find the bp_basis function in the following path: `byxtal.bp_basis.bp_basis()`. To find the basis vector of a plane with the Miller Indices of $(4 3 5)$ use the following syntax:

```python
import byxtal.bp_basis as bpb
l_2D_p = Matrix(bpb.bp_basis(ni_rp))
l_2D_p
```

To express the obtained basis in the orthogonal basis (i.e. supercell f.c.c) one needs to perform the following conversion of bases:

\begin{equation}
\Lambda_{2D}^{PO} = \Lambda_{P}^{PO} \times \Lambda_{2D}^{P}
\end{equation}

```python
l_2D_po = l_p_po*l_2D_p
l_2D_po
```

$\displaystyle \left[\begin{matrix}-1.0 & 0\\0.5 & -0.5\\0.5 & 1.5\end{matrix}\right]$

## Summary

1. At the interface of a bicrystal, the $\Lambda_{2D}^{PO}$ provides a basis for the interface.
2. If the two crystals are related to each other by a $\Sigma$-rotation, the obtained $\Lambda_{2D}^{po}$ is the two-dimensional basis for the two-dimensional coincidence site lattice at the interface. Therefore, the bicrystal conserves its periodicity in the obtained 2D-basis.
3. In other words the obtained basis is in fact the basis for the unit cell of the bicrystal and since it is in the reduced form, it is going to have the least skewness, hence ideal for constructing a periodic simulation box.

The above process is frquently repeated for simulation of grain boundaries. Therefore, we have developed a set of functions that make the conversion of indices more convenient and will accept various conventions for the Miller Indices. Please refer to the grain boundary 2D-CSL tutorial for how to use these functions.