# Import Libraries

```python
from __future__ import print_function
import neptune
import os, sys, math
import matplotlib.pyplot as plt
root   = os.path.abspath(os.getcwd())
libs   = root + '/libs'
data   = root + '/data'
sys.path.append(libs)
sys.path.append(data)
from nn_utils import *
from utilsData import *
from utilsData import read_dataset
from utilsData import transform_labels
import keras
from keras.utils import plot_model
from keras.models import Sequential, load_model
from keras.models import Model
from keras.datasets import mnist
from keras.layers import *
from keras.utils import to_categorical
import pandas as pd
import json
import numpy as np
import ctypes
import sklearn
%matplotlib inline
import plotly.graph_objects as go
import time
```

# Define Constants and Variables

```python
# Define Path Constants
dataRoot              = "data"
dataset_name          = 'Cloth'
rootWithSlash         = dataRoot + '/'
dataTest              = dataset_name + "_TEST.txt"
dataTrain             = dataset_name + "_TRAIN.txt"

# Skript Constants
INTERPOLATION         = 2
NUMBER_OF_SAMPLES_CNN = 250
ADM_LOWER_LIMIT_INT_8 = 127
ADM_UPPER_LIMIT_INT_8 = ADM_LOWER_LIMIT_INT_8
MAX_NUMBER_OF_DATA_POINTS = 2927
budget                = 100
alpha                 = 0.5
window                = 10

pickleObjectNameW      = '/lowstandardhighvolumetipswater.pkl'
pickleObjectNameG      = '/lowstandardhighvolumetipsglycerin.pkl'
```

# Load Acquired Aspiration Data

```python
# Load data from pickle Objects
sql_DF_Water          = pd.read_pickle(dataRoot+pickleObjectNameW)
sql_DF_Glycerol       = pd.read_pickle(dataRoot+pickleObjectNameG)
sql_DF                = pd.concat([sql_DF_Glycerol, sql_DF_Water])

# Select just pressure datas
sql_DFFilteredAsp        = sql_DF.loc[sql_DF.loc[:,'J_Read Pressure Data Active'] == 1,:]
# Select just class ids and pressure datas
sql_DFFilteredAspColumns = sql_DFFilteredAsp.iloc[:,-MAX_NUMBER_OF_DATA_POINTS:]
# Add Class id to dataframe
sql_DFFilteredAspColumns['ClassId'] = sql_DFFilteredAsp.loc[:,'ClassId']
# Reset index
sql_DFFilteredAspColumns = sql_DFFilteredAspColumns.reset_index()
sql_DFFilteredAspColumns = sql_DFFilteredAspColumns.drop(['index'], axis=1)
```

# Different Aspiration Curves Raw Data

```python
plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Raw Data'
# Add traces

for i in range(0, sql_DFFilteredAspColumns.shape[0]):
    # Normal Aspiration
    if int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Normal',
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Air',
                                 line=dict(color='rgb(255,0,0)', width=1)))
    # Cloth Aspiration
    elif int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Cloth',
                                 line=dict(color='rgb(0,0,255)', width=1)))

annotations = []

annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()
```

# Scale Time Data to 250 Points

Two different pressure data cases are possible
 - Aspiration pressure curve contains less than 250 points -> upsampling case
 - Aspiration pressure curve contains more than 250 points -> Downsampling case

The idea for this problem is to transform a digital signal to a nearly continuous signal by interpolation and resample it such that the whole curve in time contains 250 samples.

# Linear Interpolation of Curve

```python
sql_DFFilteredAspColumnsCopy                = sql_DFFilteredAspColumns.copy()
sql_DFFilteredAspColumnsCopy                = sql_DFFilteredAspColumnsCopy.drop(['ClassId'], axis= 1)
newDataframe                                = sql_DFFilteredAspColumnsCopy
for interpol in range(0,INTERPOLATION):
    newDataframe                            = pd.DataFrame()
    for i in range(0, sql_DFFilteredAspColumns.shape[0]):
        sql_DFFilteredAspColumnsWithoutNan  = np.array(sql_DFFilteredAspColumnsCopy.iloc[i,:].dropna())
        dataNumberingArray                  = np.arange(len(sql_DFFilteredAspColumnsWithoutNan)) + 1
        zeroPaddedData                      = np.insert(sql_DFFilteredAspColumnsWithoutNan, dataNumberingArray, 0)
        x = 0
        while x < (len(zeroPaddedData) - 2):
            zeroPaddedData[x + 1]           = ((zeroPaddedData[x] - zeroPaddedData[x + 2]) / 2.0) + zeroPaddedData[x + 2]
            x = x + 2

        interpolatedData                    = pd.DataFrame(zeroPaddedData)
        interpolatedDataT                   = interpolatedData.T

        newDataframe                        = pd.concat([newDataframe, interpolatedDataT])

    newDataframe                            = newDataframe.reset_index()
    newDataframe                            = newDataframe.drop(['index'], axis=1)

    sql_DFFilteredAspColumnsCopy            = newDataframe

```

# Resample Curves Such That Wohle Curve Is Sampled Equally

```python
totalResampledData                          = pd.DataFrame()
for i in range(0, sql_DFFilteredAspColumns.shape[0]):
    sampleLengthInterpolatedCurve           = len(sql_DFFilteredAspColumnsCopy.iloc[i, :].dropna())
    numberOfOffsetSamples                   = sampleLengthInterpolatedCurve/NUMBER_OF_SAMPLES_CNN
    sampleIteration                         = 0
    newDataframeWithoutNan                  = sql_DFFilteredAspColumnsCopy.iloc[i, :].dropna()
    resampledData                           = []
    for k in range(0, NUMBER_OF_SAMPLES_CNN):
        resampledData.append(newDataframeWithoutNan[int(sampleIteration)])
        sampleIteration                    +=numberOfOffsetSamples

    resampledDataPd                         = pd.DataFrame(np.array(resampledData))
    totalResampledData                      = pd.concat([totalResampledData, resampledDataPd.T])

totalResampledData                          = totalResampledData.reset_index()
totalResampledData                          = totalResampledData.drop(['index'], axis=1)
sql_DFFilteredAspColumnsResIndex            = sql_DFFilteredAspColumns.reset_index()
sql_DFFilteredAspColumnsResIndex            = sql_DFFilteredAspColumnsResIndex.drop(['index'], axis=1)
totalResampledData['ClassId']               = list(sql_DFFilteredAspColumnsResIndex.loc[:,'ClassId'])
# Make sure ClassId stands at first position
totalResampledDataColumns                   = totalResampledData.columns.tolist()
totalResampledDataColumns                   = totalResampledDataColumns[-1:] + totalResampledDataColumns[:-1]
totalResampledData                          = totalResampledData[totalResampledDataColumns]
```

# Save to CSV

```python
totalResampledData.to_csv(rootWithSlash+dataTest, index=False, header=False)
totalResampledData.to_csv(rootWithSlash+dataTrain, index=False, header=False)
```

# Convert Test/Train Cap Level Json File to Dataframe and Numpy Array

```python
# Load text file
datasets_dict = read_dataset(dataRoot, dataset_name)
x_train       = datasets_dict[dataset_name][0]
y_train       = datasets_dict[dataset_name][1]
x_test        = datasets_dict[dataset_name][2]
y_test        = datasets_dict[dataset_name][3]
y_trainClass  = datasets_dict[dataset_name][1]
y_testClass   = datasets_dict[dataset_name][3]

nb_classes = len(np.unique(np.concatenate((y_train,y_test),axis =0)))
# make the min to zero of labels
y_train,y_test = transform_labels(y_train,y_test)

# save orignal y because later we will use binary
y_true = y_test.astype(np.int64)
# transform the labels from integers to one hot vectors
enc = sklearn.preprocessing.OneHotEncoder()
enc.fit(np.concatenate((y_train,y_test),axis =0).reshape(-1,1))
y_train = enc.transform(y_train.reshape(-1,1)).toarray()
y_test = enc.transform(y_test.reshape(-1,1)).toarray()

if len(x_train.shape) == 2: # if univariate
    # add a dimension to make it multivariate with one dimension
    x_train = x_train.reshape((x_train.shape[0],x_train.shape[1],1))
    x_test = x_test.reshape((x_test.shape[0],x_test.shape[1],1))

input_shape = x_train.shape[1:]
```

# Scale Aspiration Data

\begin{align}
\Delta_s = max(S(t)) - min(S(t)) \\
S_s(t) = \frac{\Delta_{s128}}{\Delta_s} \Bigg(S(t) - \bigg|\frac{|max(S(t))| - |min(S(t))|}{2}\bigg|\Bigg) \hspace{1cm} |max(S(t))| > |min(S(t))|\\
S_s(t) = \frac{\Delta_{s128}}{\Delta_s} \Bigg(S(t) + \bigg|\frac{|max(S(t))| - |min(S(t))|}{2}\bigg|\Bigg) \hspace{1cm} |min(S(t))| > |max(S(t))|
\end{align}

where $S_s(t)$ defines the scaled pressure data, $S(t)$ defines the unscaled pressure data, $\Delta_{s128}$ the max range of a int_8 datatype (255) and $\Delta_s$ the max range of the unscaled pressure data $S(t)$

```python
plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Test Data'
# Add traces

for i in range(0,x_test.shape[0]):
    # Normal Aspiration
    if int(y_testClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Normal',
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(y_testClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Air',
                                 line=dict(color='rgb(255,0,0)', width=1)))
    # Cloth Aspiration
    elif int(y_testClass[i]) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Cloth',
                                 line=dict(color='rgb(0,0,255)', width=1)))

annotations = []

annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()
```

# Display Train X (Time) and Y (Normal-, Cloth-, Air-Aspiration)

```python
fig = go.Figure()
title = 'Different Aspiration Curves Train Data'
scaledTrain = x_train
# Add traces
for i in range(0,x_train.shape[0]):
    # Scale data
    temp        = []
    data        = x_train[i]
    delta_s     = max(data) - min(data)
    if(abs(max(data)) > abs(min(data))):
        tempScaled          = 253.0/delta_s * np.array(data)
        scaledTrain[i,:]    = tempScaled + (ADM_UPPER_LIMIT_INT_8 - max(tempScaled))
    elif(abs(min(data)) > abs(max(data))):
        tempScaled          = 253.0/delta_s * np.array(data)
        scaledTrain[i,:]    = tempScaled + (-ADM_LOWER_LIMIT_INT_8 - min(tempScaled))

    # Normal Aspiration
    if int(y_trainClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Normal',
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration Aspiration
    elif int(y_trainClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Air',
                                 line=dict(color='rgb(255,0,0)', width=1)))
    # Cloth Aspiration
    elif int(y_trainClass[i]) == 2 :
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Cloth',
                                 line=dict(color='rgb(0,0,255)', width=1)))

annotations = []

annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.0,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')
fig.show()

```

# Display Test X (Time) and Y (Normal-, Cloth-, Air-Aspiration)

```python
plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Test Data'
scaledTest = x_test
# Add traces
for i in range(0,x_test.shape[0]):
    # Scale data
    temp        = []
    data        = x_test[i]
    delta_s     = max(data) - min(data)
    if(abs(max(data)) > abs(min(data))):
        tempScaled          = 253.0/delta_s * np.array(data)
        scaledTest[i,:]    = tempScaled + (ADM_UPPER_LIMIT_INT_8 - max(tempScaled))
    elif(abs(min(data)) > abs(max(data))):
        tempScaled          = 253.0/delta_s * np.array(data)
        scaledTest[i,:]    = tempScaled + (-ADM_LOWER_LIMIT_INT_8 - min(tempScaled))

for i in range(0,x_test.shape[0]):
    # Normal Aspiration
    if int(y_testClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Normal',
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(y_testClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Air',
                                 line=dict(color='rgb(255,0,0)', width=1)))
    # Cloth Aspiration
    elif int(y_testClass[i]) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Cloth',
                                 line=dict(color='rgb(0,0,255)', width=1)))

annotations = []

annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()
```

# Define Layers of CNN Algorithm

```python
#https://adventuresinmachinelearning.com/keras-tutorial-cnn-11-lines/
def KModel(x_train, y_train, x_test, y_test):
    # Shuffle
    permutation = np.random.permutation(y_train.shape[0])
    x_train = x_train[permutation, :, :]
    y_train = y_train[permutation]

    inputs = Input(shape=x_train.shape[1:])
    # Layer Block 1
    x = Conv1D(32, kernel_size=(9), strides=(2), padding='same')(inputs)
    x = BatchNormalization()(x)
    x = Dropout(0.2)(x)
    x = ReLU()(x)
    x = MaxPool1D(2, strides=2)(x)

    # Layer Block 2
    x1 = Conv1D(32, kernel_size=(5), strides=(1), padding="same")(x)
    x1 = BatchNormalization()(x1)
    x1 = Dropout(0.2)(x1)
    x1 = ReLU()(x1)
    x1 = MaxPool1D(2, strides=2)(x1)

    # Layer Block 3
    x2 = Conv1D(32, kernel_size=(3), strides=(1), padding="same")(x)
    x2 = BatchNormalization()(x2)
    x2 = Dropout(0.2)(x2)
    x2 = ReLU()(x2)
    x2 = MaxPool1D(2, strides=2)(x2)

    # Layer Block 4
    x3 = MaxPool1D(2, strides=2)(x)
    x3 = Dropout(0.2)(x3)

    # concate all inception layers
    x = concatenate([ x1, x2,x3], axis=-1)

    # conclusion
    x = Conv1D(48, kernel_size=(3), strides=(1), padding="same")(x)
    x = ReLU()(x)
    x = MaxPool1D(2, strides=2)(x)
    x = Dropout(0.2)(x)

    # our netowrk is not that deep, so a hidden fully connected layer is introduce
    x = Flatten()(x)
    x = Dense(64)(x)
    x = Dropout(0.2)(x)
    x = ReLU()(x)
    x = Dense(3)(x)
    y = Softmax()(x)

    model = Model(inputs=inputs, outputs=y)
    model.compile(loss="categorical_crossentropy", optimizer="Adam", metrics=['accuracy'])
    model.fit(x_train, y_train, batch_size=16, epochs=100, shuffle=True, verbose=2, validation_data=(x_test, y_test))
    return model
```

# Train CNN Algorithm

```python
if os.path.exists('.shift_list'):
    os.remove(".shift_list")
model = KModel(scaledTrain, y_train, scaledTest, y_test)
model.save(dataRoot+'model.h5')
```

# Generate C Code for NN Library and Transfer Weights File SVN

```python
generate_model(model, x_test, name='NN_weights.h')
```


```python
# setup and imports

# the usual suspects
%matplotlib inline
%config InlineBackend.figure_format = 'png'
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from matplotlib.colors import LogNorm
import seaborn as sns

from ipywidgets import interact

# used for generating graphs
from graphviz import Digraph

# data mining algorithms
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules
from sklearn_extra.cluster import KMedoids

# config
plt.rcParams["figure.figsize"] = (10,10)
```

```python
# load the data
# note: this if randomly generated fake data, not real enrolment data
df = pd.read_csv('sample.csv')
df.drop('UNIQUEID', axis=1, inplace=True)
```

# Course Co-occurence Analysis

Learning Analytics, Visual Analytics @ UBC

Craig Thompson, CTLT

Academic programs often prescribe some number of official pathways. However, students may also choose to take combinations of courses other than those we intend.

We'd like to reveal those patterns.

[Desire path](https://flic.kr/p/5kDxUt) by [wetwebwork](https://www.flickr.com/photos/wetwebwork/][/url]), on Flickr [(CC BY 2.0)](https://creativecommons.org/licenses/by/2.0/)

# Data we have

- For every student who earns a degree, we have a record of all the courses they took and counted towards their degree requirements.

- We've limited the dataset to courses from a single department.

- Our dataset is two dimensional **binary indicator matrix**:
  - Across the horizontal axis: all the courses offered by the department
  - Down the vertical axis: IDs for each student who earned a degree
  - For each student/course pair we indicate whether the student took the course
  - This is fake data

```python
df.head(20)
```

# Data we don't have

- We don't have data for non-majors or anyone who did not complete their degree.

- We don't have performance data, so we don't know how well any student did in any particular course.

- We don't have any temporal information, so we don't know:
  - Which courses students took in sequence
  - Whether they took a pair of courses in back to back terms or with gaps
  - Which courses they took in concurrently

# About the analysis

- We are doing an exploratory analysis. This is not experimental.

- We will try to answer questions about *what* has happened, but we are unable to address *why*.

- We'd like to say "students are discovering their own pathways through our planned degrees". The reality is that students may be taking these groupings of courses because they:
  - Fit nicely in their timetable
  - Are offered by instructors they like
  - Have a reputation of being easy or fun courses
  - Have free or inexpensive textbooks
  - Are the only courses left at registration time

# Analysis via clustering

Common questions:
- What does an "average" student look like, in terms of the courses they study?
- If there were $N$ prototypical students, what would they look like?

Answer:
- We can formulate a *mathematically* average student, but there is no *pedagogically meaningful* average student.
- This sort of analysis is messy and hard to interpret.
- We'll do it anyway just to see!

```python
im = plt.imshow(df.head(100), cmap=plt.cm.binary)
```

```python
# most common courses
df.sum().sort_values(ascending=False).head(10)
```

```python
# how many courses does each student take?
df.sum(axis=1).value_counts().sort_index().plot(kind='bar');
```

```python
# helper functions for clustering students in course-space

def cluster(n, df):
    kmedoids = KMedoids(n_clusters=n, metric='manhattan', random_state=0).fit(df)
    nearest_medoid = kmedoids.predict(df)
    distances = kmedoids.transform(df)
    nearest_distance = distances[[np.arange(distances.shape[0])],nearest_medoid].T
    return (kmedoids, nearest_medoid, distances, nearest_distance)

def describe_clusters(kmedoids, nearest_medoid, distances, nearest_distance):
    plt.figure(figsize=(10, 10))

    for i in range(kmedoids.cluster_centers_.shape[0]):
        print("cluster", i+1, "centroid:", list(df.columns[kmedoids.cluster_centers_[i,:] == 1]))
        print("number of students in this cluster:", (nearest_medoid == i).sum())
        cluster_member_distances = nearest_distance[nearest_medoid == i]
        if cluster_member_distances.size > 0:
            print("minimum distance to centroid:", cluster_member_distances.min())
            print("maximum distance to centroid:", cluster_member_distances.max())
            print("mean distance to centroid:", cluster_member_distances.mean())
            print("median distance to centroid:", np.median(cluster_member_distances))
            print()
        plt.plot(sorted(cluster_member_distances))

```

```python
describe_clusters(*cluster(4, df))
```

# Lessons (re-)learned

- Course enrolment datasets are big, and hard to construct a clear mental picture of.
- We're working with (only) 50 students and 22 courses.
- Within academic programs, there aren't usually clear, strong, non-prescribed patterns at the level of whole-enrolment histories

So, let's try something different...

# History

What other domains work with similarly shaped data? Consumer purchases!

- Each individual shopper collects a bunch of items
- When a customer checks out, a sales invoice is generated listing all the items that were purchased together

From all the sales invoices, we may wish to look for patterns in consumer behaviour:
- Are there items that are **frequently** purchased together?
- Are some items good **predictors** of other items being purchased?

Why would someone care?
- If consumers buy hot dogs whenever they buy hotdog buns, then grocery stores can attempt to manipulate custormers into buying hotdogs by putting hotdog buns on sale. **Profit!**

# Content warning

The following slides contain math.

# Set theory

- a *set* is an unordered collection of distinct objects.

- For this analysis, each student's course enrolment history is being treated as a set. Sets are often written like this: $\{a,b,c\}$

- All the student enrolment histories are jointly represented as a collection of sets.
  - They are not a set-of-sets, because sets have distinct elements, and two students are able to have exactly the same course enrolment history.
  - So, this collection of sets is called a *multiset* or a *bag*, to denote that it may contain duplicate elements.

# Frequent itemsets

Given a multiset (such as a stack of grocery store receipts, or a table of student-course enrolments), how do we find the frequently occurring subsets (or itemsets)?

Example: Given $[\{a\},\{a,b\},\{a,c\},\{a,b,c,d\}]$

We can see that:
- $\{a\}$ occurs in all 4 sets
- $\{a,b\}$ and $\{a,c\}$ each occur in 2 sets

<math>
\begin{align}
&\mathrm{Apriori}(T,\epsilon)\\
&\quad L_1 \gets \{\textrm{large 1 item sets}\}\\
&\quad k \gets 2\\
&\quad \textbf{while}\ L_{k-1} \neq \emptyset\\
&\quad \quad C_k \gets \{c = a \cup \{b\} \mid a \in L_{k-1} \land b \notin a, \{s \subseteq c \mid |s| = k - 1 \} \subseteq L_{k-1} \}\\
&\quad \quad \textbf{for}\ \textrm{transactions}\ t \in T\\
&\quad \quad \quad D_t \gets \{c \in C_k \mid c \subseteq t \}\\
&\quad \quad \quad \textbf{for}\ \textrm{candidates}\ c \in D_t\\
&\quad \quad \quad \quad count[c] \gets count[c] + 1\\
&\quad \quad L_k \gets \{c \in C_k \mid count[c] \geq \epsilon \}\\
&\quad \quad k \gets k + 1\\
&\quad \textbf{return} \bigcup_k L_k
\end{align}
</math>

Rakesh Agrawal and Ramakrishnan Srikant. 1994. Fast Algorithms for Mining Association Rules in Large Databases. In Proceedings of the 20th International Conference on Very Large Data Bases (VLDB ’94). Morgan Kaufmann Publishers Inc., San Francisco, CA, USA, 487–499. ( > 26k citations!)

Let $X$ be an itemset and $T$ be the set of transactions/records in the database

<math>
\begin{align}
&\textrm{Support}(X) = \frac{\mid \{t \in T \mid X \subseteq t \}\mid }{\mid T \mid}
\end{align}
</math>

*Support* indicates how frequently a given itemset appears in the transactions of the database.
- A support of 1 indicates the itemset appears in every transaction.
- A support of 0.5 indicates the itemset appears in half of the transactions.

```python
course_frequency = apriori(df, min_support=np.nextafter(0,1), max_len=1, use_colnames=True)
course_frequency['course'] = course_frequency['itemsets'].apply(lambda x: set(x).pop())
course_frequency['course_number'] = course_frequency['course'].apply(lambda x: x[4:])
course_frequency[['support', 'course']].sort_values(by='support',ascending=False)
```

```python
cf = course_frequency[['support', 'course']].set_index('course').sort_values(by='support',ascending=False)

def f(limit):
    cf.head(limit).plot(kind='bar')

i = interact(f, limit=(1,20))
```

```python
frequent_itemsets = apriori(df, min_support=np.nextafter(0,1), max_len=2, use_colnames=True)
frequent_itemsets.sort_values(by='support',ascending=False)
```

# Association rules

\begin{equation*}
X \Rightarrow Y, \textrm{where}\ X,Y \subseteq I
\end{equation*}

X is called the *antecedent* and Y is the *consequent*.

$I$ is the set of all items (e.g. courses).

example: $\textrm{Math110} \Rightarrow \textrm{Math210}$ would be read as "if Math110, then Math210".

Now we have a notation for a relationship between two itemsets (in this case, the two itemsets each contain a single item), but we need to describe the *qualities* of that relationship...

Rakesh Agrawal, Tomasz Imieliński, and Arun Swami. 1993. Mining association rules between sets of items in large databases. In Proceedings of the 1993 ACM SIGMOD international conference on Management of data (SIGMOD ’93). Association for Computing Machinery, New York, NY, USA, 207–216. DOI:https://doi.org/10.1145/170035.170072 (> 22k citations!)

# Metrics for quantifying association rules: Support

- *Antecedent Support*: indicates how frequently the antecedent item set appears in the database.

$$\textrm{Antecedent Support}(X \Rightarrow Y) = \frac{\mid \{t \in T \mid X \subseteq t \}\mid }{\mid T \mid}$$

- *Consequent Support*: indicates how frequently the consequent item set appears in the database.

$$\textrm{Consequent Support}(X \Rightarrow Y) = \frac{\mid \{t \in T \mid Y \subseteq t \}\mid }{\mid T \mid}$$

- *(Rule) Support*: indicates how frequently the all them items of the antecedent and consequent jointly appear in the database.

$$\textrm{Support}(X \Rightarrow Y) = \frac{\mid \{t \in T \mid X \cup Y \subseteq t \}\mid }{\mid T \mid}$$

## Metrics for quantifying association rules: Confidence

$$\textrm{Confidence}(X \Rightarrow Y) = \frac{ \textrm{Support}(X \Rightarrow Y) }{ \textrm{Support}(X) }$$

*Confidence*: the ratio of rule support to antecedent support.
  - Or, given that the antecedent has been observed, how likely are we to also observe the consequent?

If a rule has high confidence, and the antecedent is observed, then we can be fairly confident that the consequent will be observed as well.

## Metrics for quantifying association rules: Lift

$$\textrm{Lift}(X \Rightarrow Y) = \frac{ \textrm{Confidence}(X \Rightarrow Y) }{ \textrm{Support}(Y) }$$

*Lift*: ratio of confidence to consequent support. Lift is a measure of how much more often the antecedent and the consequent occur together than would be expected if they were statistically independent. When the antecedent of a rule with high lift is observed, we can be more confident that the consequent will also be observed.

Confidence and lift are both descriptors of the "power" of a rule.

```python
rules = association_rules(frequent_itemsets, metric="support", min_threshold=np.nextafter(0,1))

rules['antecedent_course'] = rules['antecedents'].apply(lambda x: set(x).pop())
rules['consequent_course'] = rules['consequents'].apply(lambda x: set(x).pop())

rules['antecedent_course_number'] = rules['antecedent_course'].apply(lambda x: int(x[4:]))
rules['consequent_course_number'] = rules['consequent_course'].apply(lambda x: int(x[4:]))

rules['antecedent_year_level'] = rules['antecedent_course_number'].apply(lambda x: x//100 )
rules['consequent_year_level'] = rules['consequent_course_number'].apply(lambda x: x//100)
```

```python
rules
```

```python
pairwise_rules = rules[(rules['antecedent_year_level']==3) & (rules['consequent_year_level']==3)]
pairwise_support = pairwise_rules.pivot(index='antecedent_course',columns='consequent_course',values='support').fillna(0)
ax = sns.heatmap(pairwise_support, xticklabels=True, yticklabels=True, cmap='BuPu')
```

```python
pairwise_rules = rules[(rules['antecedent_year_level']==3) & (rules['consequent_year_level']==3)]
pairwise_confidence = pairwise_rules.pivot(index='antecedent_course',columns='consequent_course',values='confidence').fillna(0)
ax = sns.heatmap(pairwise_confidence, xticklabels=True, yticklabels=True, cmap='BuPu')
```

```python
pairwise_rules = rules[(rules['antecedent_year_level']==3) & (rules['consequent_year_level']==3)]
pairwise_lift = pairwise_rules.pivot(index='antecedent_course',columns='consequent_course',values='lift').fillna(0.1)
#pairwise_lift = pairwise_lift.applymap(lambda x: x if x >=1 else 0.01)
ax = sns.heatmap(pairwise_lift, xticklabels=True, yticklabels=True, cmap='BuPu', norm=LogNorm())
```

```python
# exploring 'significant' rules
sig_rules = rules[
      (rules['support'] > 0.01)
    #& (rules['antecedent support'] > 0.01)
    #& (rules['consequent support'] > 0.01)
    & (rules['antecedent_year_level'] <= rules['consequent_year_level'])
    & (rules['confidence'] > 0.5)
    & (rules['lift'] > 1.5)
                 ].sort_values(by='lift',ascending=False)
sig_rules
```

```python
def plot_rules(sig_rules):
    antecedents = sig_rules[['antecedent_course','antecedent_course_number']]
    antecedents.columns = ['course','course_number']

    consequents = sig_rules[['consequent_course','consequent_course_number']]
    consequents.columns = ['course','course_number']

    figure_courses = pd.concat([antecedents, consequents]).drop_duplicates()

    dot = Digraph()
    for course in figure_courses.itertuples():
        dot.node(str(course.course_number),course.course)

    for association in sig_rules.itertuples():
        dot.edge(str(association.antecedent_course_number), str(association.consequent_course_number), label=f"{association.lift:.2f}")

    dot.graph_attr['overlap'] = 'False'
    dot.engine = 'neato'
    return dot
```

```python
dot = plot_rules(sig_rules)
dot
```

# What next?

- This analysis was (nearly) totally naive to the official structure of the courses/programs offered. That means we 'discovered' several results that are obvious and uninteresting to anyone with domain expertise.
  - For example, we may have 'discovered' that if you take a senior course, then you are very likely to also take the prerequisites.
  - We should filter the result set to remove any association rules that reflect formal prerequisite relationships.

- We've looked at *what* courses students take, but we have not investigated ordering, or the effects of ordering. An analysis of sequencing would be useful for planning connections across courses.

# Thank You

Questions?

Contact: craig.thompson@ubc.ca


```javascript
%%javascript
$('#appmode-leave').hide();
$('#copy-binder-link').hide();
$('#visit-repo-link').hide();
```

```python
import ipywidgets as ipw
import json
import random
import time
import pandas as pd
import os
import webbrowser
import math
from IPython.display import display, Markdown, Math, Latex, clear_output

# set kinetic parameters
with open("rate_parameters.json") as infile:
    jsdata = json.load(infile)

params = jsdata["equi4"]
```

The hydrolysis Sucrose into Glucose and Fructose is catalysed by the enzyme Invertase.
\begin{equation}
Sucrose + Invertase + \mathrm{H_2O} \to Glucose + Fructose
\end{equation}
There are however several substances that can inhibit the efficacy of the catalyst

Imagine performing a series of experiments using different initial concentration of Sucrose where you measure the rate of formation of Glucose with. The results of your experiments are affected by the presence of a contaminating substance that interferes with the catalytic reaction. Although you can somewhat control the concentration of the contaminant, you cannot completely eliminate it.

1. Determine whether the contaminating substance inhibits the catalytic reaction and the type of the inhibition mechanism, *e.g.* Competitive, Uncompetitive, Non-competitive or Mixed.

2. Determine the maximum rate achieved by the reaction, $V_{max}$ and the Michaelis constant, $K_M$, in the case you could completely eliminate the contamininat.

### Tips:
- Note that every time you restart the experiment the type of the inhibition mechanism may change.

### Instructions:

- Use the slide bar below to select temperature at which you perform the virtual experiment,
- Click `Perform measurement` to run the virtual experiment and obtain the result of the experiment,
- Click `Download CSV` to export the complete data set for all the experiments as a CSV file.

```python
# define path to results.csv file
respath = os.path.join(os.getcwd(), "..", "results.csv")

# delete existing result file and setup rng
if os.path.exists(respath):
    os.remove(respath)

class system:
    def __init__(self, vol=0, conc=0, press=0):
        self.vol = vol
        self.conc = conc
        self.press = press
        self.inhibition = 0
        self.seed = 0
        self.Vm = 0
        self.Km = 0
        self.Ki = 0
        self.Kip= 0

class data:
    def __init__(self, start=-1, error=0, label='none', units='pure', value=0,
                minval=-1, maxval=3, text='none'):
        self.start = start
        self.minval = minval
        self.maxval = maxval
        self.error = error
        self.label = label
        self.units = units
        self.value = value
        self.text = text

# Experiment setup (+ hidden paramters)
system = system()
def initialiseExperiment():
    global n
    global system
    global columns_list
    global scatter

    scatter = 0.01

    n = []
    columns_list = []

    n.append(len(args)) # number of input adjustable parameters
    n.append(len(result)) # number of results for the experiment

    for i in range(0, n[0]):
        columns_list.append(f"{args[i].label} [{args[i].units}]")
    for i in range(0, n[1]):
        columns_list.append(f"{result[i].label} [{result[i].units}]")

    # Random number seed
    t = int( time.time() * 1000.0 )
    system.seed = ((t & 0xff000000) >> 24) + ((t & 0x00ff0000) >>  8) +((t & 0x0000ff00) <<  8) +((t & 0x000000ff) << 24)
    random.seed(system.seed)

    # Random inhibition type
    rnd = random.random()
    system.inhibition = int(5 * rnd)
    if (system.inhibition > 4):
        system.inhibition = 4

    system.Vm = params["Vm"] * (1 + random.random()/2)
    system.Km = params["Km"] * (1 + random.random()/2)
    system.Ki = system.Km * random.random()
    system.Kip= system.Km * random.random()

```

```python
# Adjustable input parameters
def initialiseVariables():
    global logScale
    logScale = True
    global args
    args = []
    args.append(
        data(
            label = "[S]",
            minval = -3,
            maxval = 1,
            start = 0.001,
            units = "mol/L",
            value = 0.
        )
    )

    args.append(
        data(
            label = "[I]",
            minval = -3,
            maxval = 0,
            start = 0.001,
            units = "mol/L",
            value = 0.
        )
    )

# Results
def initialiseResults():
    global result
    result = []
    result.append(
        data(
            label = "Reaction Rate",
            start = 0.,
            error = random.random() / 10.,
            units = "mol/L·min"
        )
    )

def measure():
    concS = float(args[0].text.value)
    concI = float(args[0].text.value)

    Vm = system.Vm
    Km = system.Km
    Ki = system.Ki
    Kip= system.Kip

    # no inhibition
    a = 1
    ap = 1

    # competitive
    if (system.inhibition == 1):
        a  = 1 + concI / Ki
        ap = 1
        adp = 1

    # non-competitive
    elif (system.inhibition == 4):
        a  = 1
        ap = 1 + concI / Ki
        adp = 1

    # un-competitive
    elif (system.inhibition == 2):
        a  = 1
        ap = 1
        adp = 1. / (1 + concI / Kip)

    # mixed
    elif (system.inhibition == 3):
        a  = 1 + concI / Ki
        ap = 1
        adp = 1. / (1 + concI / Kip)

    res =  (ap * adp) * Vm * concS / ((a * adp)*Km + concS)
    return res

initialiseVariables()

```

```python
out_P = ipw.Output()
out_L = ipw.Output()
out_X = ipw.Output()

with out_L:
    display(Markdown("[Download CSV](../results.csv)"))

def calc(btn):
    out_P.clear_output()

    # Measurement result
    result[0].value = measure()

    # Random error
    result[0].error = result[0].value * scatter * (0.5 - random.random()) * 2

    # Output result
    out_R[0].value = f"{result[0].value + result[0].error:.3e}"

    # Read previous lines
    res = pd.read_csv(respath)

    var_list = []
    for i in range(0, n[0]):
        var_list.append(args[i].text.value)
    for i in range(0, n[1]):
        var_list.append(result[i].value + result[i].error)

    # Append result
    res.loc[len(res)] = var_list
    res.to_csv(respath, index=False)
    with out_P:
        display(res.tail(50))

def reset(btn):
    if os.path.exists(respath):
        os.remove(respath)

    initialiseResults()
    initialiseExperiment()

    res = pd.DataFrame(columns=columns_list)
    res.to_csv(respath, index=False)
    with out_P:
        out_P.clear_output()
        display(res.tail(1))

    with out_X:
        out_X.clear_output()

btn_reset = ipw.Button(description="Restart Laboratory", layout=ipw.Layout(width="150px"))
btn_reset.on_click(reset)

btn_calc = ipw.Button(description="Perform measurement", layout=ipw.Layout(width="150px"))
btn_calc.on_click(calc)
# ---

rows = []
reset(btn_reset)

args[0].text = ipw.Text(str(args[0].start))
rows.append(ipw.HBox([ipw.Label('Initial concentration of ' + args[0].label + '   :  '),args[0].text]))

args[1].text = ipw.Text(str(args[1].start))
rows.append(ipw.HBox([ipw.Label('Initial concentration of ' + args[1].label + '   :  '),args[1].text]))

out_R = []
for i in range(0, n[1]):
    out_R.append(ipw.Label(value=""))
    rows.append(ipw.HBox([ipw.Label(value=f"Measured {result[i].label} [{result[i].units}]:",
                                    layout=ipw.Layout(width="250px")),
                            out_R[i]]))

rows.append(ipw.HBox([btn_reset, btn_calc, out_L]))

def calc2(btn):
    random.seed(system.seed)
    rnd = random.random()
    iType = int(4 * rnd) + 1

    with out_X:
        out_X.clear_output()
        if (iType == 1):
            display(Markdown(r'Competitive inhibition'))
        elif (iType == 2):
            display(Markdown(r'Un-Competitive inhibition'))
        elif (iType == 3):
            display(Markdown(r'Mixed inhibition'))
        elif (iType == 4):
            display(Markdown(r'Non-Competitive inhibition'))
        else:
            display(Markdown(r'No inhibition'))

        display(Markdown(r'$K_M$ = 'rf'{system.Km:7.5}'))
        display(Markdown(r'$V_{max}$ = 'rf'{system.Ki:7.5}'))
        if (iType == 1) or (iType == 3) or (iType == 4):
            display(Markdown(r'$K_i$ = 'rf'{system.Ki:7.5}'))
        if (iType == 2) or (iType == 3) or (iType == 4):
            display(Markdown(r'$K_i^\prime$ = 'rf'{system.Kip:7.5}'))

display(out_X)

btn_calc2 = ipw.Button(description="Check Inhibition Type", layout=ipw.Layout(width="150px"))
btn_calc2.on_click(calc2)
rows.append(ipw.HBox([btn_calc2]))

rows.append(ipw.HBox([out_P]))
ipw.VBox(rows)

```

```python

```

```python

```