```python
%matplotlib inline
from IPython.core.display import display, HTML
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from pylab import rcParams
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from pylab import rcParams
rcParams['figure.figsize'] = 10, 6
plt.rc("font", size=14)
```

```python
import os
import calendar
import collections
from collections import defaultdict, OrderedDict

from datetime import datetime
from dateutil.relativedelta import *
import itertools
from dateutil import parser

import pandas as pd
pd.set_option('display.max_columns', 100)

import numpy as np
import scipy
import statsmodels
import statsmodels.api as sm
import statsmodels.formula.api as smf
import statsmodels.tsa.api as smt

import sympy
import requests
from bs4 import BeautifulSoup
from scipy.stats import mode
from scipy import interp

from sklearn import linear_model
from sklearn import preprocessing, linear_model, metrics
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.cross_validation import cross_val_score
from sklearn.model_selection import StratifiedKFold

from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, f1_score, classification_report, roc_curve, auc
from sklearn.pipeline import Pipeline, FeatureUnion
```

    /anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
      "This module will be removed in 0.20.", DeprecationWarning)

```python
df = pd.read_clipboard()
display(df)
```

      yyyy
      month
      tmax
      tmin

      0
      1908
      January
      5.0
      -1.4

      1
      1908
      February
      7.3
      1.9

      2
      1908
      March
      6.2
      0.3

      3
      1908
      April
      Missing_1
      2.1

      4
      1908
      May
      Missing_2
      7.7

      5
      1908
      June
      17.7
      8.7

      6
      1908
      July
      Missing_3
      11.0

      7
      1908
      August
      17.5
      9.7

      8
      1908
      September
      16.3
      8.4

      9
      1908
      October
      14.6
      8.0

      10
      1908
      November
      9.6
      3.4

      11
      1908
      December
      5.8
      Missing_4

      12
      1909
      January
      5.0
      0.1

      13
      1909
      February
      5.5
      -0.3

      14
      1909
      March
      5.6
      -0.3

      15
      1909
      April
      12.2
      3.3

      16
      1909
      May
      14.7
      4.8

      17
      1909
      June
      15.0
      7.5

      18
      1909
      July
      17.3
      10.8

      19
      1909
      August
      18.8
      10.7

## Treat missing values in a standard way

```python
df2 = df.copy(deep = True)
df2[["tmax_clean", "tmin_clean"]] = df[["tmax", "tmin"]].replace(to_replace= r'(?i)missing', value=np.nan, regex= True)

df2[["tmax_clean", "tmin_clean"]] = df2[["tmax_clean", "tmin_clean"]].apply(pd.to_numeric)
df2.head(5)
```

      yyyy
      month
      tmax
      tmin
      date_time
      tmax_clean
      tmin_clean

      0
      1908
      January
      5.0
      -1.4
      1908-01-01
      5.0
      -1.4

      1
      1908
      February
      7.3
      1.9
      1908-02-01
      7.3
      1.9

      2
      1908
      March
      6.2
      0.3
      1908-03-01
      6.2
      0.3

      3
      1908
      April
      Missing_1
      2.1
      1908-04-01
      NaN
      2.1

      4
      1908
      May
      Missing_2
      7.7
      1908-05-01
      NaN
      7.7

### Convert to datetime index

```python
df2["yyyy"] = df2["yyyy"].map(str)
df2["date_time"] = df2['month'] + "-" + df2["yyyy"]
df2["date_time"]  = df2["date_time"].apply(lambda x: pd.to_datetime(x,format = '%B-%Y'))
df2.set_index("date_time", inplace = True)
df2.head(5)
```

      yyyy
      month
      tmax
      tmin
      tmax_clean
      tmin_clean

      date_time

      1908-01-01
      1908
      January
      5.0
      -1.4
      5.0
      -1.4

      1908-02-01
      1908
      February
      7.3
      1.9
      7.3
      1.9

      1908-03-01
      1908
      March
      6.2
      0.3
      6.2
      0.3

      1908-04-01
      1908
      April
      Missing_1
      2.1
      NaN
      2.1

      1908-05-01
      1908
      May
      Missing_2
      7.7
      NaN
      7.7

# Apply Pandas built in interpolation methods
- https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.interpolate.html

Types of missing data:
- if upsampling is required:
      upsampled = df.series.resample('D')
- if the dates are missing
      df = df.reindex(pd.date_range("2011-01-01", "2011-10-31"), fill_value="NaN")

- methods: {‘linear’, ‘time’, ‘index’, ‘values’, ‘nearest’, ‘zero’, 'slinear’, ‘quadratic’, ‘cubic’, ‘barycentric’,
            ‘krogh’, ‘polynomial’, ‘spline’, ‘piecewise_polynomial’, ‘from_derivatives’, ‘pchip’, ‘akima’}

```python
df_interp = df2.copy(deep = True)

df_interp["tmin_hat"] = df_interp["tmin_clean"].interpolate(axis=0, method='spline', order=3, s=0.,\
                         limit=None, inplace=False, limit_direction='forward', limit_area=None, downcast=None).ffill().bfill()
df_interp["tmax_hat"] = df_interp["tmax_clean"].interpolate(axis=0, method='spline', order=3, s=0.,\
                        limit=None, inplace=False, limit_direction='forward', limit_area=None, downcast=None).ffill().bfill()
df_interp[["tmin", "tmin_clean", "tmin_hat", "tmax", "tmax_clean", "tmax_hat"]].head(20)
```

      tmin
      tmin_clean
      tmin_hat
      tmax
      tmax_clean
      tmax_hat

      date_time

      1908-01-01
      -1.4
      -1.4
      -1.400000
      5.0
      5.0
      5.000000

      1908-02-01
      1.9
      1.9
      1.900000
      7.3
      7.3
      7.300000

      1908-03-01
      0.3
      0.3
      0.300000
      6.2
      6.2
      6.200000

      1908-04-01
      2.1
      2.1
      2.100000
      Missing_1
      NaN
      8.715007

      1908-05-01
      7.7
      7.7
      7.700000
      Missing_2
      NaN
      13.548117

      1908-06-01
      8.7
      8.7
      8.700000
      17.7
      17.7
      17.700000

      1908-07-01
      11.0
      11.0
      11.000000
      Missing_3
      NaN
      18.580018

      1908-08-01
      9.7
      9.7
      9.700000
      17.5
      17.5
      17.500000

      1908-09-01
      8.4
      8.4
      8.400000
      16.3
      16.3
      16.300000

      1908-10-01
      8.0
      8.0
      8.000000
      14.6
      14.6
      14.600000

      1908-11-01
      3.4
      3.4
      3.400000
      9.6
      9.6
      9.600000

      1908-12-01
      Missing_4
      NaN
      0.712381
      5.8
      5.8
      5.800000

      1909-01-01
      0.1
      0.1
      0.100000
      5.0
      5.0
      5.000000

      1909-02-01
      -0.3
      -0.3
      -0.300000
      5.5
      5.5
      5.500000

      1909-03-01
      -0.3
      -0.3
      -0.300000
      5.6
      5.6
      5.600000

      1909-04-01
      3.3
      3.3
      3.300000
      12.2
      12.2
      12.200000

      1909-05-01
      4.8
      4.8
      4.800000
      14.7
      14.7
      14.700000

      1909-06-01
      7.5
      7.5
      7.500000
      15.0
      15.0
      15.000000

      1909-07-01
      10.8
      10.8
      10.800000
      17.3
      17.3
      17.300000

      1909-08-01
      10.7
      10.7
      10.700000
      18.8
      18.8
      18.800000

```python
df_interp[['tmin_clean', 'tmin_hat']].plot(figsize=(12, 8))
plt.show()

df_interp[['tmax_clean', 'tmax_hat']].plot(figsize=(12, 8))
plt.show()
```

```python
# Print the missing values
df_final = df_interp[df_interp['tmin'].str.startswith("Missing") | df_interp['tmax'].str.startswith("Missing")]
df_final
```

      yyyy
      month
      tmax
      tmin
      tmax_clean
      tmin_clean
      tmin_hat
      tmax_hat

      date_time

      1908-04-01
      1908
      April
      Missing_1
      2.1
      NaN
      2.1
      2.100000
      8.715007

      1908-05-01
      1908
      May
      Missing_2
      7.7
      NaN
      7.7
      7.700000
      13.548117

      1908-07-01
      1908
      July
      Missing_3
      11.0
      NaN
      11.0
      11.000000
      18.580018

      1908-12-01
      1908
      December
      5.8
      Missing_4
      5.8
      NaN
      0.712381
      5.800000

```python
my_dict = OrderedDict()
for idx, row in df_final.iterrows():
    if "Missing" in row["tmin"]:
        my_dict[row["tmin"]] = row["tmin_hat"]
    if "Missing" in row["tmax"]:
        my_dict[row["tmax"]] = row["tmax_hat"]
#print(my_dict)
my_list = list(my_dict.values())
print()
for elem in my_list:
    print(elem)
```

    OrderedDict([('Missing_1', 8.715006595720961), ('Missing_2', 13.548117071007688), ('Missing_3', 18.580018038923892), ('Missing_4', 0.7123811326789329)])
    8.715006595720961
    13.548117071007688
    18.580018038923892
    0.7123811326789329


## Subsurface Data Analytics

### Ridge Regression for Subsurface Data Analytics in Python

#### Michael Pyrcz, Associate Professor, University of Texas at Austin

##### [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)

### PGE 383 Exercise: Ridge Regression for Subsurface Modeling in Python

Here's a simple workflow, demonstration of ridge regression for subsurface modeling workflows. This should help you get started with building subsurface models with data analytics and machine learning. Here's some basic details about linear regression.

Ridge regression is an extension of linear regresion, so let's review some basic details about linear regression first.

#### Linear Regression

Linear regression for prediction.  Here are some key aspects of linear regression:

**Parametric Model**

* the fit model is a simple weighted linear additive model based on all the available features, $x_1,\ldots,x_m$.

* the parametric model takes the form of:

\begin{equation}
y = \sum_{\alpha = 1}^m b_{\alpha} x_{\alpha} + b_0
\end{equation}

**Least Squares**

* least squares optimization is applied to select the model parameters, $b_1,\ldots,b_m,b_0$

* we minize the error, residual sum of squares (RSS) over the training data:

\begin{equation}
RSS = \sum_{i=1}^n (y_i - (\sum_{\alpha = 1}^m b_{\alpha} x_{\alpha} + b_0))^2
\end{equation}

* this could be simplified as the sum of square error over the training data,

\begin{equation}
\sum_{i=1}^n (\Delta y_i)^2
\end{equation}

**Assumptions**

* **Error-free** - predictor variables are error free, not random variables
* **Linearity** - response is linear combination of feature(s)
* **Constant Variance** - error in response is constant over predictor(s) value
* **Independence of Error** - error in response are uncorrelated with each other
* **No multicollinearity** - none of the features are redundant with other features

#### Other Resources

This is a tutorial / demonstration of **Linear Regression**.  In $Python$, the $SciPy$ package, specifically the $Stats$ functions (https://docs.scipy.org/doc/scipy/reference/stats.html) provide excellent tools for efficient use of statistics.
I have previously provided this example in R and posted it on GitHub:

1. R https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.R
2. Rmd with docs https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.Rmd
3. knit as an HTML document(https://github.com/GeostatsGuy/geostatsr/blob/master/linear_regression_demo_v2.html)

#### Ridge Regression

With ridge regression we add a hyperparameter, $\lambda$, to our minimization, with a shrinkage penalty term.

\begin{equation}
\sum_{i=1}^n (y_i - (\sum_{\alpha = 1}^m b_{\alpha} x_{\alpha} + b_0))^2 + \lambda \sum_{j=1}^m b_{\alpha}^2
\end{equation}

As a result ridge regression has 2 criteria:

* set the model parameters to minimize the error with training data

* shrink the estimates of the slope parameters towards zero

Note: the intercept is not affected by lambda.

The $\lambda$ is a hyperparameter that controls the degree of fit of the model and may be related to the model variance and bias trade-off.

* for $\lambda \rightarrow 0$ the solution approaches linear regression, there is no bias (relative to a linear model fit), but the variance is high

* as $\lambda$ increases the model variance decreases and the model bias increases

* for $\lambda \rightarrow \infty$ the coefficients approach 0.0 and the model approaches the global mean

#### Workflow Goals

Learn the basics of ridge regression in Python to for analysis, modeling and prediction of porosity from density. This includes:

* Basic Python workflows and data preparation

* Training / fitting a ridge regression model

* Checking the model and learning about the impact of hyperparameters

#### Objective

In the PGE 383: Stochastic Subsurface Modeling class I want to provide hands-on experience with building subsurface modeling workflows. Python provides an excellent vehicle to accomplish this. I have coded a package called GeostatsPy with GSLIB: Geostatistical Library (Deutsch and Journel, 1998) functionality that provides basic building blocks for building subsurface modeling workflows.

The objective is to remove the hurdles of subsurface modeling workflow construction by providing building blocks and sufficient examples. This is not a coding class per se, but we need the ability to 'script' workflows working with numerical methods.

#### Getting Started

Here's the steps to get setup in Python with the GeostatsPy package:

1. Install Anaconda 3 on your machine (https://www.anaconda.com/download/).
2. From Anaconda Navigator (within Anaconda3 group), go to the environment tab, click on base (root) green arrow and open a terminal.
3. In the terminal type: pip install geostatspy.
4. Open Jupyter and in the top block get started by copy and pasting the code block below from this Jupyter Notebook to start using the geostatspy functionality.

There are examples below with these functions. You can go here to see a list of the available functions, https://git.io/fh4eX, other example workflows and source code.

#### Import Required Packages

Let's import the GeostatsPy package.

```python
import os                                                   # to set current working directory
import numpy as np                                          # arrays and matrix math
import scipy.stats as st                                    # statistical methods
import pandas as pd                                         # DataFrames
import matplotlib.pyplot as plt                             # for plotting
from sklearn.metrics import mean_squared_error, r2_score    # specific measures to check our models
from sklearn.linear_model import Ridge                      # ridge regression implemented in scikit learn
from sklearn.model_selection import cross_val_score         # multi-processor K-fold crossvalidation
```

If you get a package import error, you may have to first install some of these packages. This can usually be accomplished by opening up a command window on Windows and then typing 'python -m pip install [package-name]'. More assistance is available with the respective package docs.

#### Set the working directory

I always like to do this so I don't lose files and to simplify subsequent read and writes (avoid including the full address each time).  Also, in this case make sure to place the required (see below) data file in this working directory.

```python
#os.chdir("C:\PGE337")                                       # set the working directory
```

#### Loading Data

Let's load the provided dataset. 'Density_Por_data.csv' is available at https://github.com/GeostatsGuy/GeoDataSets. It is a comma delimited file with 20 density ($\frac{g}{cm^3}$) and porosity (as a fraction) measures from the subsurface. We load the data file with the pandas 'read_csv' function into a data frame we called 'df' and then separate it into train and test datasets.  The smaples are in random order so we just split the dataset at the 80th sample.  We preview each with the head function from Pandas DataFrames.

```python
#df = pd.read_csv("Density_Por_data.csv")                    # read a .csv file in as a DataFrame
df = pd.read_csv("https://raw.githubusercontent.com/GeostatsGuy/GeoDataSets/master/Density_Por_data.csv") # load from Prof. Pyrcz's GitHub
df_train = df.iloc[0:80,:]                                  # extract a training set, note samples are random ordered
df_train.head()                                             # preview the DataFrame
```

      Density
      Porosity

      0
      1.281391
      16.610982

      1
      1.404932
      13.668073

      2
      2.346926
      9.590092

      3
      1.348847
      15.877907

      4
      2.331653
      4.968240

```python
df_test = df.iloc[80:]                                      # extract a testing set, note samples are random ordered
df_test.head()                                              # preview the DataFrame
```

      Density
      Porosity

      80
      1.750352
      11.325941

      81
      1.666285
      15.609445

      82
      1.466517
      17.066529

      83
      1.650921
      13.876841

      84
      0.996736
      20.964941

It is useful to review the summary statistics of our loaded DataFrame.  That can be accomplished with the 'describe' DataFrame member function.  We transpose to switch the axes for ease of visualization.  We will summarize over the training and testing subsets separately.

```python
df_train.describe().transpose()
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      Density
      80.0
      1.759945
      0.295260
      1.067960
      1.552707
      1.767908
      1.953262
      2.410560

      Porosity
      80.0
      12.187127
      3.125755
      4.966421
      10.011526
      12.170687
      14.063670
      19.600717

```python
df_test.describe().transpose()
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      Density
      25.0
      1.667429
      0.257608
      0.996736
      1.568988
      1.718085
      1.791432
      2.339324

      Porosity
      25.0
      13.632564
      2.948887
      9.489298
      11.325941
      13.767060
      15.051759
      20.964941

Here we extract the ndarrays with porsity and density, training and testing datasets separate arrays for convenience.

```python
por_train = df_train['Porosity'].values                     # make a shallow copy of the features for convenvience
df = pd.read_csv("Density_Por_data.csv")
den_train = df_train['Density'].values
por_test = df_test['Porosity'].values
den_test = df_test['Density'].values
n_train = len(df_train); n_test = len(df_test)              # get the number of data in training and testing
```

Let's look at the training and testing data.

```python
plt.subplot(111)
plt.scatter(df_train["Density"].values, df_train["Porosity"],  color='black', s = 20, alpha = 0.3, label = 'training')
plt.scatter(df_test["Density"].values, df_test["Porosity"],  color='red', s = 20, alpha = 0.3, label = 'testing')
plt.title('Linear Regression Porosity from Density with Training Data'); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
plt.legend()
plt.xlim(1.,2.6)#; plt.ylim(0,1500000)
plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.0, wspace=0.2, hspace=0.2)
plt.show()
```

#### Linear Regression Model

Let's first calculate the linear regression model. We use scikit learn and then extend the same workflow to ridge regression.

```python
from sklearn import linear_model

# Step 1. Instantiate the Model
linear_reg = linear_model.LinearRegression()

# Step 2: Fit the Data on Training Data
linear_reg.fit(df_train["Density"].values.reshape(n_train,1), df_train["Porosity"]) # fit model
density_model = np.linspace(1.2,2.4,10)

# Print the model parameters
porosity_model = linear_reg.predict(density_model.reshape(10,1)) # predict with the fit model
print('Coefficients: ', str(round(linear_reg.coef_[0],3)) + ', Intercept: ', str(round(linear_reg.intercept_,3)))

# Plot model fit
plt.subplot(111)
plt.scatter(df_train["Density"].values, df_train["Porosity"],  color='black', s = 20, alpha = 0.3)
plt.plot(density_model,porosity_model, color='red', linewidth=1)
plt.title('Linear Regression Porosity from Density with Training Data'); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
plt.xlim(1.,2.6)#; plt.ylim(0,1500000)
plt.subplots_adjust(left=0.0, bottom=0.0, right=1.0, top=1.0, wspace=0.2, hspace=0.2)
plt.show()
```

Let's run some quick model checks.  Much more could be done, but I limit this for breviety here.

```python
# Step 3: - Make predictions using the testing dataset
y_pred = linear_reg.predict(df_test['Density'].values.reshape(n_test,1))

# Report the goodness of fit
print('Variance explained: %.2f' % r2_score(df_test['Porosity'].values, y_pred))

# Plot testing diagnostics
plt.subplot(121)
plt.scatter(df_test['Density'].values, df_test['Porosity'].values,  color='black', s = 20, alpha = 0.3)
plt.scatter(df_test['Density'], y_pred, color='blue', s = 20, alpha = 0.3)
plt.title('Linear Regression Model Testing - Production from Porosity'); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
plt.xlim(1.0,2.6); plt.ylim(5,24)

y_res = y_pred - df_test['Porosity'].values

print('Residual: mean = ' + str(round(np.average(y_res),2)) + ', standard deviation = ' + str(round(np.var(y_res),2)))

plt.subplot(122)
plt.hist(y_res, alpha = 0.2, color = 'red', edgecolor = 'black', bins=20)
plt.title('Linear Regression Model Prediction Error - Porosity form Density'); plt.xlabel('Porosity Estimation Error (%) (Estimate - Truth)'); plt.ylabel('Frequency')
plt.xlim(-4,4)#; plt.ylim(0,1500000)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.2, top=1.2, wspace=0.3, hspace=0.2)
plt.show()
```

#### Ridge Regression

Let's replace the scikit learn linear regression method with the scikit learn ridge regression method.  Note, we must now set the lambda hyperparameter.

* the hyperparameter is set with the instantiation of the model

```python
lam = 1.0

# Step 1. Instantiate the Model
ridge_reg = Ridge(alpha=lam)

# Step 2: Fit the Data on Training Data
ridge_reg.fit(df_train["Density"].values.reshape(n_train,1), df_train["Porosity"]) # fit model
density_model = np.linspace(1.2,2.4,10)

# Print the model parameters
porosity_model = ridge_reg.predict(density_model.reshape(10,1)) # predict with the fit model
print('Coefficients: ', str(round(linear_reg.coef_[0],3)) + ', Intercept: ', str(round(linear_reg.intercept_,3)))

# Plot model fit
plt.subplot(111)
plt.scatter(df_train["Density"].values, df_train["Porosity"],  color='black', s = 20, alpha = 0.3)
plt.plot(density_model,porosity_model, color='red', linewidth=1)
plt.title('Ridge Regression Porosity from Density with Training Data'); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
plt.xlim(1.,2.6); plt.ylim(8,17)
plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.0, wspace=0.2, hspace=0.2)
plt.show()
```

Let's repeat the simple model checks that we applied with our linear regression model.

```python
# Step 3: - Make predictions using the testing dataset
y_pred = ridge_reg.predict(df_test['Density'].values.reshape(n_test,1))

# Report the goodness of fit
print('Variance explained: %.2f' % r2_score(df_test['Porosity'].values, y_pred))

# Plot testing diagnostics
plt.subplot(121)
plt.scatter(df_test['Density'].values, df_test['Porosity'].values,  color='black', s = 20, alpha = 0.3)
plt.scatter(df_test['Density'], y_pred, color='blue', s = 20, alpha = 0.3)
plt.title('Linear Regression Model Testing - Production from Porosity'); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
plt.xlim(1.0,2.6); plt.ylim(5,24)

y_res = y_pred - df_test['Porosity'].values

print('Residual: mean = ' + str(round(np.average(y_res),2)) + ', standard deviation = ' + str(round(np.var(y_res),2)))

plt.subplot(122)
plt.hist(y_res, alpha = 0.2, color = 'red', edgecolor = 'black', bins=20)
plt.title('Linear Regression Model Prediction Error - Porosity form Density'); plt.xlabel('Porosity Estimation Error (%) (Estimate - Truth)'); plt.ylabel('Frequency')
plt.xlim(-4,4)#; plt.ylim(0,1500000)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.2, top=1.2, wspace=0.3, hspace=0.2)
plt.show()
```

Interesting, we explained less variance and have a larger residual standard deviation (more error).

* we see we actually reduced both testing variance explained and accuracy

#### Investigating the Lambda Hyperparameter

Let's loop over multiple lambda values - from 0 to 100 and observe the change in:

* training and testing, mean square error (MSE) and variance explained

```python
# Arrays to store the results
ncases = 100
lamd_mat = np.linspace(0.0,100.0,ncases)
density_model = np.linspace(1.2,2.4,10)
var_explained_train = np.zeros(ncases); var_explained_test = np.zeros(ncases)
mse_train = np.zeros(ncases); mse_test = np.zeros(ncases)

for ilam in range(0,len(lamd_mat)):                         # Loop over all lambda values
    ridge_reg = Ridge(alpha=lamd_mat[ilam])
    ridge_reg.fit(df_train["Density"].values.reshape(n_train,1), df_train["Porosity"]) # fit model

    porosity_model = ridge_reg.predict(density_model.reshape(10,1)) # predict with the fit model
    porosity_pred_train = ridge_reg.predict(df_train['Density'].values.reshape(n_train,1)) # predict with the fit model
    var_explained_train[ilam] = r2_score(df_train['Porosity'].values, porosity_pred_train)
    mse_train[ilam] = mean_squared_error(df_train['Porosity'].values, porosity_pred_train)

    porosity_pred_test = ridge_reg.predict(df_test['Density'].values.reshape(n_test,1))
    var_explained_test[ilam] = r2_score(df_test['Porosity'].values, porosity_pred_test)
    mse_test[ilam] = mean_squared_error(df_test['Porosity'].values, porosity_pred_test)

    if ilam <= 7:
        plt.subplot(4,2,ilam+1)
        plt.scatter(df_train["Density"].values, df_train["Porosity"],  color='black', s = 20, alpha = 0.3)
        plt.plot(density_model,porosity_model, color='red', linewidth=1)
        plt.title('Ridge Regression Porosity from Density with Training Data - Lambda = ' + str(round(lamd_mat[ilam],2))); plt.xlabel('Density (g/cm^3)'); plt.ylabel('Porosity (%)')
        plt.xlim(1.,2.6); plt.ylim(5,24)

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=4.2, wspace=0.2, hspace=0.3)
plt.show()
```

We can observed from the first 8 cases above of ridge regression model fit that increase in the lambda hyper parameter decreases the slope of the linear fit.

Let's plot the MSE and variance explained over training and testing datasets.

```python
plt.subplot(121)
plt.plot(lamd_mat, var_explained_train,  color='blue', linewidth = 2, label = 'Training')
plt.plot(lamd_mat, var_explained_test,  color='red', linewidth = 2, label = 'Test')
plt.title('Variance Explained vs. Lambda'); plt.xlabel('Lambda'); plt.ylabel('Variance Explained')
plt.xlim(0.,100.); plt.ylim(0,1.0)
plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.0, wspace=0.2, hspace=0.2)
plt.legend()

plt.subplot(122)
plt.plot(lamd_mat, mse_train,  color='blue', linewidth = 2, label = 'Training')
plt.plot(lamd_mat, mse_test,  color='red', linewidth = 2, label = 'Test')
plt.title('MSE vs. Lambda'); plt.xlabel('Lambda'); plt.ylabel('Mean Square Error')
plt.xlim(0.,100.); plt.ylim(0,10.0)
plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.0, wspace=0.2, hspace=0.2)
plt.legend()

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.2, wspace=0.2, hspace=0.3)
plt.show()
```

We observe that as we increase the lambda parameter the variance explained decreases and the mean square error increases.

* this makes sense as the data has a consistent linear trend and as the slope 'shrinks' to zero the error increases and the variance explained decreases

* there could be other cases where the reduced slope actually performs better in testing.  For example with sparce and noisy data.

#### Model Variance

Now let's explore the concept of model variance, an important part of machine learning accuracy in testing.

* the sensitivity of the model to the specfic training data

* as lambda increases the sensitivity to the training data, model variance decreases

Let's demonstrate this with this workflow:

* loop over multiple lambda values
    * loop over multiple bootstrap samples of the data
        * calculate the ridge regression fit (slope)
    * calculate the variance of these bootstrap results

```python
L = 200                                                     # the number of bootstrap realizations
nsamples = 20                                               # the number of samples in each bootstrap realization
nlambda = 100                                               # number of lambda values to evaluate

coef_mat = np.zeros(L)                                      # declare arrays to store the results
variance_coef = np.zeros(nlambda)

lamd_mat = np.linspace(0.0,100.0,nlambda)
df = pd.read_csv("Density_Por_data.csv")
for ilam in range(0,len(lamd_mat)):                         # loop over all lambda values
    for l in range(0, L):                                   # loop over all bootstrap realizations
        df_sample = df.sample(n = nsamples)                 # random sample (1 bootstrap)
        ridge_reg = Ridge(alpha=lamd_mat[ilam])             # instatiate model
        ridge_reg.fit(df_sample["Density"].values.reshape(nsamples,1), df_sample["Porosity"]) # fit model
        coef_mat[l] = ridge_reg.coef_[0]                    # get the slope parameter

    variance_coef[ilam] = np.var(coef_mat)                  # calculate the variance of the slopes over the L bootstraps
```

Now let's plot the result.

```python
plt.subplot(111)
plt.plot(lamd_mat, variance_coef,  color='black', linewidth = 2, label = 'Slope Variance')
plt.title('Model Fit Variance vs. Lambda'); plt.xlabel('Lambda'); plt.ylabel('Model Fit Variance')
plt.xlim(0.,100.); plt.ylim(0.001,10.0); plt.yscale('log')
plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.0, wspace=0.2, hspace=0.2)
plt.legend()
```

The result is as expected, with increase in lambda hyperparameter the sensitivity of the model to the training data is decreased.

#### k-fold Cross Validation

It would be useful to conduct a complete k-fold validation to evaluate the testing error vs. the hyperparameter lambda for model tuning.

* the following code should do this

* but with a single feature as input for fitting the fit function requires a reshape

```python
my_array.reshape((nsample,1))
```

* this is not included in the scikit learn function 'cross_val_score' so we will skip this for now

I have left the code commented out below for reference:

```python
#score = []                                                  # code modified from StackOverFlow by Dimosthenis
#nlambda = 1
#lambd_mat = np.linspace(0.0,100.0,nlambda)
#for ilam in range(0,nlambda):
#    ridge_reg = Ridge(alpha=lambd_mat[ilam])
#    scores = cross_val_score(estimator=ridge_reg, X= df['Density'].values, y=df['Porosity'].values, cv=10, n_jobs=4, scoring = "neg_mean_squared_error") # Perform 10-fold cross validation
#    score.append(abs(scores.mean()))
```

#### Comments

Ridge regression is a variant of linear regression that includes a hyperparameter to constrain the degree of model fit.  This allow us to tune the variance-bias trade-off of our model. I hope this was helpful,

*Michael*

Michael Pyrcz, Ph.D., P.Eng. Associate Professor The Hildebrand Department of Petroleum and Geosystems Engineering, Bureau of Economic Geology, The Jackson School of Geosciences, The University of Texas at Austin
On twitter I'm the @GeostatsGuy.

***

#### More on Michael Pyrcz and the Texas Center for Geostatistics:

### Michael Pyrcz, Associate Professor, University of Texas at Austin
*Novel Data Analytics, Geostatistics and Machine Learning Subsurface Solutions*

With over 17 years of experience in subsurface consulting, research and development, Michael has returned to academia driven by his passion for teaching and enthusiasm for enhancing engineers' and geoscientists' impact in subsurface resource development.

For more about Michael check out these links:

#### [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)

#### Want to Work Together?

I hope this content is helpful to those that want to learn more about subsurface modeling, data analytics and machine learning. Students and working professionals are welcome to participate.

* Want to invite me to visit your company for training, mentoring, project review, workflow design and / or consulting? I'd be happy to drop by and work with you!

* Interested in partnering, supporting my graduate student research or my Subsurface Data Analytics and Machine Learning consortium (co-PIs including Profs. Foster, Torres-Verdin and van Oort)? My research combines data analytics, stochastic modeling and machine learning theory with practice to develop novel methods and workflows to add value. We are solving challenging subsurface problems!

* I can be reached at mpyrcz@austin.utexas.edu.

I'm always happy to discuss,

*Michael*

Michael Pyrcz, Ph.D., P.Eng. Associate Professor The Hildebrand Department of Petroleum and Geosystems Engineering, Bureau of Economic Geology, The Jackson School of Geosciences, The University of Texas at Austin

#### More Resources Available at: [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)

```python

```


# On the fundamental differences between quantum states - an overview.

### University of Basel. Department of Physics
#### Quantum Information. Frühjahrssemester 2020

**Professor**: James Wootton.

Bowei Wu - bowei.wu@stud.unibas.ch  (Responsible for the first part)
José A. Hernández - ja.hernandezsanchez@stud.unibas.ch  (Responsible for the second part)

**NOTE:** Both authors contributed with their respective parts. A consensus was reached in order to paint a concise picture and thus the presented work is a collaborative effort.

## First part - an introduction:

## What even is a "quantum state"?

Historically, the study of various microscopic phenomena have shown that our classical (and thus discreet) approach to the description and understanding of our reality is, at the microscopic level, inaccurate and missing "the bigger picture" at best and catastrophically off at worst.

A concise example is the so-called "photoelectric effect", that reveals that nature is not continuous. The unit part of the energy in a beam of light with frequency $v$, is $\epsilon=hv$. This is abnormal in classical physical. But when the scale become smaller and smaller, the physical phenomenon  become quantized. For a quantum system, the energy and the state are not arbitrary but discrete. For example, the particle in 1D box has eigenstates

$$\psi_n(x)=\sqrt\frac{2}{L}sin\left(\frac{n\pi x}{L}\right)$$

The state of the particle can be

$$|\psi\rangle=\sum c_n|\psi_n\rangle$$

Namely, the state of the particle can be different eigenstate at the same time. This is called superposition, which is a unique property given by quantum mechanics. If we do a measurement on the particle, we will find the particle is on one of the eigenstates. The case that the particle is at state $|\psi_n\rangle$ is of a probability $|c_n|^2$. So the coefficient should satisfy

$$\sum |c_n|^2=1$$

## The Qubit and Two-state quantum system

Qubit is the basic unit in quantum computing. The name of 'qubit' comes from 'bit', which is the basic unit in classical computing. In classical computer, a bit can be either 0 or 1. Here 0 and 1 can correspond to the off and on of a switch, or low and high voltage in real physics. Then we can use multiple bits (bit string) to encode information. In quantum computer, we also use 0 and 1 corresponding to the classical binary bit. However, the qubit can be both 0 and 1 at same time. If we write the qubit down, it can be expressed as

$$ |\psi\rangle=\alpha|0\rangle+\beta|1\rangle $$

Here $|0\rangle$ and $|1\rangle$ are 2 eigenstates, which are orthogonal to each other. For example, we can define a perpendicularly polarized photon as $|0\rangle$ and a horizontally polarized photon as $|1\rangle$. Or, a spin-up electron as $|0\rangle$ and a spin-down electron as $|1\rangle$.

The value of $\alpha$ and $\beta$ are complex numbers. And they need to implement $|\alpha|^2+|\beta|^2=1$, because $|\alpha|^2$ and $|\beta|^2$ are the probabilities of the result being $|0\rangle$ and $|1\rangle$. After the measurement the state will collapse to either $|0\rangle$ or $|1\rangle$, which means that the uncertainty disappear and the state would no longer change.

If we have multiple qubits, we can use tenser products to express the multi-qubit state. For example, 2-qubits state which contains qubit A and B can be expressed as

$$|\psi\rangle_{AB}=|\psi\rangle_A\otimes\psi\rangle_B=c_{00}|00\rangle+c_{01}|01\rangle+c_{10}|10\rangle+c_{11}|11\rangle$$

What mentioned above are just abstract expressions. To put it into practice, we need to prepare some real qubits. Since the qubit has 2 eigenstates, it is just a two-state quantum system.

### Experimental approach towards Two-state quantum system

The basic requirement of the two-state quantum system is that there are 2 stable enough states in the energy level where the physics process of our interest happens. That is to say, the half-life period of the two states should be far longer than their running time. The system even can have more than 2 states, as long as the other states have almost no influence on the two states during the physics process we are interested in.

A possible method to build a two-state system is by Nuclear Magnetic Resonance (NMR). The qubits are the nucleus with spin $1\over2$ (like $^1H$, $^13C$, $^19F$, $^{15}N$, etc.). Define the spin-up nucleus as $|0\rangle$ and spin-down nucleus as $|1\rangle$. The qubits can be controlled by alternating magnetic field, since the spin will precess under external magnetic field.

To build two-state systems, there are also some other methods like quantum dots (define spin-down dot as $|0\rangle$ and spin-up dot as $|1\rangle$), optical method (mentioned above), superconducting Josephson junction, etc.

## Quantum Gates

Mathematically, if we call the operation on qubits is $U$, and the state of qubits are written down as a vector, $U$ must be a unitary matrix. That is to say, $UU^+=I$. Let us use the two-state NMR system as example to see how the mathematical unitary transform, i.e. the quantum logic gate, is realized on a real physics system.

The gate we want to realzie is called $U(\theta,\phi)$.

$$U(\theta,\phi)= \begin{pmatrix} cos\theta & -isin\theta e^{i\phi} \\ -isin\theta e^{-i\phi} & cos\theta \end{pmatrix}
$$

To implement this, we need to create a magnetic field in X-Y plane existing in time interval $\tau$, which will let the spin precess in the orientation of the direction of magnetic field. If the magnetic field is $\textbf{B}=|\textbf{B}|(cos\phi, sin\phi, 0)$, the frequency of Larmor precession will be $\omega_L={2\over\hbar}|\mu||\textbf{B}|$. So the rotated angle is $\omega_L \tau$ and $\theta={\omega_L \tau\over2}$. Now we succeed to bulid the unitary transform $U(\theta,\phi)$. We can see that if we apply this on state $|0\rangle$, we get $cos\theta|0\rangle -ie^{i\phi}sin\theta|1\rangle$.

On different real quantum systems we will have different approaches to realize different quantum gates.

## Entanglement

Entanglement is a non-local property between sub-systems in a quantum system. If a state of several quantum states can't be expressed as the tenser product of each qubit, such state is called entangled state. For example, the state ${1\over\sqrt2}(|00\rangle+|10\rangle)$ can be expressed as ${1\over\sqrt2}(|0\rangle+|1\rangle)\otimes|0\rangle$. In contrast, the state ${1\over\sqrt2}(|00\rangle+|11\rangle)$ cannot be expressed as the tensor product of two qubits, which means this is an entangled state. Since entanglement is non-local, the state of one qubit will have influence on the state of other qubits which are entangled with it.

The property of entanglement gives the quantum computing ability of parallel processing. For instance, given decimal number 10 and 5, we express them as $|1010\rangle)$ and $|0101\rangle)$. Then create a entangled state of them,  $|1010\rangle)+|0101\rangle)$. For this state, if we apply some quantum algorithm, the number 10 and 5 can be computed separately. This is one of the reasons why quantum computer sometimes is faster than classical computer.

However, the property of entanglement means that the quantum system can be entangled with not only the target but also the environment. Once it is entangled with the environment, the quantum coherence will decay with time, which is called quantum decoherence.

## Second part - on the "most" and "least" quantum states:

## The many-body quantum system

All of these aforementioned characteristics and descriptions of quantum systems lay a solid foundation for even deeper questions to be posed; mainly concerning how we can exploit the seemingly erratic and unknowable features of the quantum world for our benefit.
A question one might ask is how we could in principle use qubits to convey information to others just as some scribbles on paper do in our daily lives; and so, the quest for communication using many of these qubits begins.

Consider then a system composed of $N$ identically prepared subsystems such that the Hilbert space where the complete system "lives" may be written in terms of the subsystem's Hilbert subspaces; namely:

$$\mathcal{H}_{C}=\mathcal{H}_{S}^{\otimes N}$$

Additionally, the Hamiltonian with which this complete system will evolve in time can be written in terms of the Hamiltonians describing the evolution of the subsystems, so:

$$H=\sum_{i=1}^{N}\mathbb{1}^{1}\otimes\ldots\otimes h^{i} \otimes\ldots\otimes\mathbb{1}^{N}$$

And the ith subsystem's Hamiltonian $h^{i}$ in turn may be written in some appropriate basis as:

$$h^{i}=\begin{pmatrix}\lambda_{min} & & \\ & \ddots & \\ & &\lambda_{max}=\lambda_{min}+\Delta  \end{pmatrix}$$

Assuming of course that there exists a finite spectral range and that the eigenvalues of said "sub Hamiltonian" are bounded and sorted from a minimum to a maximum. Now, from this, and by virtue of $h^{i}=h^{i\dagger}$. We can propose the decomposition of $h^{i}=g^{i}+\frac{\lambda_{min}+\lambda_{max}}{2}\mathbb{1}$, where:

$$g^{i}=\begin{pmatrix}-\Delta/2 & & \\ & \ddots & \\ & &\Delta/2\end{pmatrix}$$

Now, if we consider the case where the encoding of such qubits is done in a unitary (and thus reversible) fashion, then we can write the unitary operator that describes the evolution of the whole system in terms of the unitaries which describe the evolution of the individual subsystems:

$$U_{C}(t)=e^{-itH}=\bigotimes_{i=1}^{N}U_{S}^{i}(t)\longleftrightarrow U_{S}^{i}(t)=e^{-ith^{i}}$$

From this consideration and from the fact that the identity matrix $\mathbb{1}$ and $g^{i}$ commute, then the individual subsystem unitary may be rewritten as:

$$U_{S}(t)=e^{-it\frac{\lambda_{min}+\lambda_{max}}{2}}e^{-itg^{i}}$$

Where in the end, the scalar phase factor the subsystems accrue $e^{-it\frac{\lambda_{min}+\lambda_{max}}{2}}$ has no physical relevance. From this, we can assume then that $h^{i}=g^{i}, \forall i$.
The subsystem Hamiltonian's eigenvalues may be assumed to be within the interval $[-\Delta/2,\Delta/2]$ without loss of any generality. Moreover, the eigenstates corresponding to the extremal eigenvalues can be denoted to be:
$$
\begin{align}
h^{i}\left|s_{min}\right\rangle=&-\frac{\Delta}{2}\left|s_{min}\right\rangle\\
h^{i}\left|s_{max}\right\rangle=&\frac{\Delta}{2}\left|s_{max}\right\rangle
\end{align}
$$

## Product states, GHZ states and the Quantum Fisher information measure

To begin our discussion of quantum states, we begin with the "least" quantum state; the product state. Basically we consider the total state to be a direct product of the individual sub states. So for a fixed interaction time $t=1$ a product state can be written in a general way as follows:

$$\left|\Psi(t)\right\rangle=e^{-iH}\left|\Psi(0)\right\rangle=e^{-iH}\bigotimes_{i=1}^{N}\left|\psi^{(i)}(0)\right\rangle$$

In the case for qubits, the sub states may be written in density matrix form as:

$$\left|\psi^{(i)}\right\rangle\left\langle\psi^{(i)}\right|=\left|\vec{n}\right\rangle\left\langle\vec{n}\right|=\frac{1}{2}\left(\mathbb{1}+\vec{n}\cdot\vec{\sigma}\right)$$

Where $||\vec{n}||=1$ and $\vec{\sigma}=(\sigma_{X},\sigma_{Y},\sigma_{Z})$.

Resulting in a product state for qubits:

$$\left|\Psi(t)\right\rangle=e^{-iH}\frac{1}{\sqrt{N}}\left(\left|0\right\rangle+e^{i\varphi}\left|1\right\rangle\right)^{\otimes N}$$

On the other hand, the "most" quantum state we can prepare can be thought of as "the maximally entangled" state; such a state is usually called a GHZ state (as it was first studied for the case of at least three qubits by Greenberger, Horne and Zellinger in 1989). For qubits, this state may be written as follows:

$$\left|GHZ\right\rangle=\frac{1}{\sqrt{2}}\left(\left|0\right\rangle^{\otimes N}+e^{i\varphi}\left|1\right\rangle^{\otimes N}\right)$$

All this talk about states is good and all, but when it comes to actual devices and experiments, is there a way to distinguish between these two states? Meaning, which quantity can we extract from the outcome of an experiment in order to show that the state we're working with is of which kind? The answer comes to us from an information-theoretical quantity called the **Fisher information** and its quantum counterpart the quantum Fisher information (QFI). Many developments have studied this quantity throughout the years, and as such it has many different interpretations and representations; in this work however we focus on one particular definition of it, mathematically it may be written as:

$$\mathcal{F}(|\Psi(t)\rangle)=4\mathrm{Var}_{|\Psi(0)\rangle}(H)$$

I.e. four times the variance of a given Hamiltonian under an initial state quantifies the QFI of a given (strictly pure) quantum state. In short: the QFI is a measure for "how entangled" a quantum state is. To prove this let us calculate explicitly the QFI for the two types of states stated above, the product state and the GHZ state all with the Hamiltonian also previously stated. First, for the product state:

$$\begin{align*}
\mathcal{F}(|\Psi(t)\rangle)=&4\left(\left\langle\Psi(0)\right|H^{2}\left|\Psi(0)\right\rangle-(\left\langle\Psi(0)\right|H\left|\Psi(0)\right\rangle)^{2}\right)
\end{align*}$$

Where the first moment about the mean is:

$$\begin{align*}
\left\langle\Psi(0)\right|H\left|\Psi(0)\right\rangle=&\sum_{i=1}^{N}\left\langle\Psi(0)\right|h^{(i)}\left|\Psi(0)\right\rangle\\
=&\sum_{i=1}^{N}\left\langle\psi^{(i)}(0)\right|h^{(i)}\left|\psi^{(i)}(0)\right\rangle\\
=&\sum_{i=1}^{N}\left\langle h^{(i)}\right\rangle
\end{align*}$$

And the second moment.

$$\begin{align*}
\left\langle\Psi(0)\right|H^{2}\left|\Psi(0)\right\rangle=&\sum_{i,j=1}^{N}\left\langle\Psi(0)\right|h^{(i)}h^{(j)}\left|\Psi(0)\right\rangle\\
=&\sum_{i=j=1}^{N}\left\langle \left(h^{(i)}\right)^{2}\right\rangle+\sum_{i\neq j}^{N}\left\langle h^{(i)}\right\rangle\left\langle h^{(j)}\right\rangle
\end{align*}$$

All in all, the QFI for a product state may be shown to be simply equivalent to the sum of the QFI of the individual sub states w.r.t. their corresponding sub Hamiltonians. I.e.

$$\begin{align*}
\mathcal{F}(\left|\Psi(t)\right\rangle)=&4\sum_{i=1}^{N}\left(\left\langle \left(h^{(i)}\right)^{2}\right\rangle-\left\langle h^{(i)}\right\rangle^{2}\right)=4\mathrm{Var}_{|\psi^{(i)}(0)}\left(h^{(i)}\right)
\end{align*}$$

The best QFI attainable from this is then proportional up to a factor of $4$ to the maximum variance of a Hamiltonian about a particular state. For a product state considering the above expressions for the Hamiltonian and sub systems then:

$$\begin{align*}
\mathcal{F}_{\text{Product}}=&N\Delta^{2}
\end{align*}$$

In contrast to this result, following analogously but now for a GHZ state, it is straightforward to show that:

$$\begin{align*}
\mathcal{F}_{\text{GHZ}}=&N^{2}\Delta^{2}
\end{align*}$$

Which is indeed $N$ times larger than the QFI for that of a product state! Implying of course that our knowledge of the entanglement of the state scales linearly as the number of subsystems increases.

So, we know that the QFI for different states is different, effectively quantifying "how entangled" a state is. The next obvious question to ask after investigating the "most" and "least" quantum states is, does an "intermediately" entangled state exist? And if so, how can we characterize _any_ state by virtue of its "entanglement depth"? We address this next.

## Entanglement depth and a Qiskit proposal
Let us now consider an intermediate case between the completely product state and a GHZ "maximally" entangled state. Consider then that from the $N$ subsystems we form $K$ groups $G_{1}, G_{2},\ldots,G_{K}$ with at most $M$ subsystems in each group. By allowing a total state with this assumption to not be completely product but rather product among these different groups, then the total state may be written as:

$$\begin{align*}
\left|\Psi_{M}\right\rangle=&\bigotimes_{k=1}^{K}\left|\psi^{(G_{k})}\right\rangle
\end{align*}$$

Computing the QFI corresponding to this state with our usual Hamiltonian, and recalling that for product states (even among groups!) the QFI is additive across subsystems, then:

$$\begin{align*}
\mathcal{F}_{\left|\Psi_{M}\right\rangle}(H)=&\sum_{k=1}^{K}\mathcal{F}_{\left|\psi^{(G_{k})}\right\rangle}\left(H\right)\leq\sum_{k=1}^{K}\left|G_{k}\right|^{2}\Delta^{2}
\end{align*}$$

Where of course $\left|G_{k}\right|$ is the size of the $k$-th subgroup. One can then make the assertion that this QFI is then maximized for the case where $\left|G_{k}\right|=|M|,\forall k$ by construction of these groups. In this case then, the QFI may be rewritten as:

$$\begin{align*}
\mathcal{F}_{\left|\Psi_{M}\right\rangle}(H)\leq&\Delta^{2}\left(\frac{N}{M}\right)M^{2}=\Delta^{2}NM
\end{align*}$$

Since $M=N/k$. This yields then the QFI of a system where $M$ particles are entangled (at the most!) from an $N$ particle total system. Pure states are experimentally difficult to prepare, lifting the QFI restriction for pure states can be done by writing the density state for a particular mixed state with "$M$-bound" entanglement in the following way:

$$\begin{align*}
\varrho_{M}=&\sum_{i}^{N}p_{(i)}\left|\Psi_{M}^{(i)}\right\rangle\left\langle\Psi_{M}^{(i)}\right|
\end{align*}$$

The QFI may then be written for this more general state as:

$$\begin{align*}
\mathcal{F}_{\varrho_{M}}(H)\leq&\sum_{i}^{N}p_{(i)}\mathcal{F}_{\left|\Psi_{M}^{(i)}\right\rangle}(H)\leq\Delta^{2}NM
\end{align*}$$

From this we can define the entanglement depth ($\mathcal{ED}$) of a given general state $\varrho$ by acknowledging that a witness or a measure for it is indeed the QFI of said state $\varrho$ for a given Hamiltonian describing the evolution of the system. Thus,

$$\begin{equation*}
\boxed{
\mathcal{ED}(\varrho)\geq\frac{\mathcal{F}_{\varrho}(H)}{\Delta^{2}N}
}
\end{equation*}$$

Where in particular, if $M=N$, i.e. $N$ subsystems (all of them) are entangled, then we recover the GHZ state previously discussed. Conversely, if $M=1$, meaning that no subsystems are entangled, a product state is recovered.

In theory the characterization of any state by means of the entanglement of the particles conforming said state is rather straightforward:
1. **FIRST:** Define the Hamiltonian under which the system will evolve (possibility for error induction and characterization goes here)
2. **SECOND:** With knowledge of the state with which we are working with, define the spectral range $\Delta$ of the Hamiltonian by looking at the extremal cases.
3. **THIRD:** Compute the QFI for this state under said Hamiltonian.
4. **FOURTH:** Comparte the QFI with the spectral range and the number of subsystems in total.

After all of this one can say with certainty that:

$$\begin{align*}
\text{If$\ldots$}& &\text{$\ldots$ then.}\\
\mathcal{ED(\varrho)}&>1 &\text{$\varrho$ is entangled!}\\
\mathcal{ED(\varrho)}&<1 &\text{$\varrho$ is not entangled!}
\end{align*}$$

In Qiskit, this can be put to the test for the extreme cases (product state and the GHZ state).

```python
# Initialization
from qiskit import *
from qiskit.aqua.operators import *
import numpy as np
```

Setting the number of Qubits:

```python
# Number of qubits and thus, quantum registers

N = 8

if (N % 2) != 0:
    raise ValueError('The number of qubits is uneven!')

qr = QuantumRegister(N)
#cr = ClassicalRegister(N)
```

A GHZ state may of $N$ qubits may be prepared as follows:

```python
# GHZ state preparation for N qubits

GHZcircuit = QuantumCircuit(qr)
GHZcircuit.h(0)
for i in range(0,N-1):
    GHZcircuit.cx(i,i+1)
GHZcircuit.h((qr[N-2],qr[N-1]))
GHZcircuit.cx(N-1,N-2)
GHZcircuit.h((qr[N-2],qr[N-1]))

GHZcircuit.draw(output="mpl")
```

A randomly prepared but nonetheless product state can be prepared by using:

```python
# Random, N qubit product state preparation

Prodcircuit = QuantumCircuit(qr)

Prodcircuit.h(qr)
for i in range(0,N-1):
    if np.random.uniform(0.0,1.0) > 0.5:
        Prodcircuit.x(i)

Prodcircuit.draw(output="mpl")
```

For a particular Hamiltonian (which can be user defined) a simple and rather naive algorithm for our eigenvalue sorting scheme can be:

```python
# Eigenvalue sorting in the Hamiltonian's diagonal basis
def EV_sorting(arr):
    for i in range(0,N-1):
        for j in range(0,N-1):
            if arr[i][j] != 0 and i < N/2:
                arr[i][j] = -1*arr[i][j]
    return arr
```

```python
# Arbitrary local Hamiltonian
for i in range(0,N-1):
    hl = (1/2)*np.eye(N,N)
h = EV_sorting(hl)
h
```

    array([[-0.5,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. , -0.5,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. , -0.5,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. , -0.5,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0.5,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0.5,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0.5,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0.5]])

```python
# Arbitrary global Hamiltonian
for i in range(0,N-1):
    Hg = N*(1/2)*np.eye(N,N)
H = EV_sorting(Hg)
H
```

    array([[-4.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0., -4.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0., -4.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0., -4.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  4.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  4.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  4.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  4.]])

From this, it would be enough to compute the Variance of these Hamiltonians using the different state functions from the quantum circuits described above. Qiskit's own "StateFn" function does the trick with the only caveat being that one must take into account normalization of the states.

```python
# Quantum Fisher information estimation as per above equation

GHZ = StateFn(GHZcircuit,1/np.sqrt(2),True)
Prod = StateFn(Prodcircuit,1/np.sqrt(N),True)
```

Results on real IBM devices can be shown to yield results similar to:

```python
from IPython.display import Image
```

Where the plot in a) shows that as time progresses $Jt$ the state evolves and becomes entangled by virtue of $\mathcal{ED}>1$. Conversely the state analyzed in b) shows no sign of entanglement as $\mathcal{ED}<1$ for a particular time range after which it becomes entangled.

## Final remarks and sources
Evolution/interaction times are also a factor to consider in future work as it will play a significant role in further quantum state characterization since as seen above, after a particular time, an initially non-entangled state may evolve to be entangled.

Additionally, further research and application of Qiskit's own tools for QFI estimation is in order to be able to test these theoretical proceedings on real quantum devices. Additionally, this might serve as a way to calibrate and/or characterize errors on real quantum devices since the extremal cases were thoroughly studied.

[1] Ozaeta, Asier, and Peter L McMahon. “Decoherence of up to 8-Qubit Entangled States in a 16-Qubit Superconducting Quantum Processor.” Quantum Science and Technology 4.2 (2019): 025015. Crossref. Web.
[2] Suzuki, Yohichi et al. “Amplitude Estimation Without Phase Estimation.” Quantum Information Processing 19.2 (2020): n. pag. Crossref. Web.
[3] Smith, Adam et al. “Simulating Quantum Many-Body Dynamics on a Current Digital Quantum Computer.” npj Quantum Information 5.1 (2019): n. pag. Crossref. Web.
[4] Sekatski, Pavel et al. “Quantum Information: Applications to Metrology and Communications.” University of Basel lecture (FS 2020)
[5] Wootton, James. “Quantum Information.” University of Basel lecture (FS 2020)

```python
import qiskit
qiskit.__qiskit_version__
```

    {'qiskit-terra': '0.14.1',
     'qiskit-aer': '0.5.1',
     'qiskit-ignis': '0.3.0',
     'qiskit-ibmq-provider': '0.7.1',
     'qiskit-aqua': '0.7.1',
     'qiskit': '0.19.2'}

```python

```


The first three cells are a copy of the DarkPhoton.py files included in the DarkCapPy package.
I have changed the relative path to the configuration files (constants.py, atomicData.py, etc..), and some of the relative paths within those files so that it would work from this notebook, without having to install the package. I have also changed the parameters in PlanetData.py back to Earth's.

Somehow, the computing time for the kappa_0 function is excessively long when importing the package into jupyter notebook on my end. Running the code directly from the notebook makes computing time acceptable (about 2-3 minutes vs an hour). There might be an issue in multiple imports of the package within the configuration files that needs to be looked at in the original package.

```python
################################################################
# Import Python Libraries
################################################################
import numpy as np
import scipy.integrate as integrate
import scipy.interpolate as interpolate
import pandas as pd
import matplotlib.pyplot as plt
```

```python

from Configure.Constants  import *
from Configure.AtomicData import *
from Configure.PlanetData  import *
from Configure.Conversions import amu2GeV
```

```python

# import os                                          | Reference: https://stackoverflow.com/questions/779495/python-access-data-in-package-subdirectory
# this_dir, this_filename = os.path.split(__file__)  | This was a hack for importing the branching ratio inside of this file.
# DATA_PATH = os.path.join(this_dir, "brtoe.csv")    | It is not presently needed, but may be useful in the future.

################################################################
# Capture Rate Functions
################################################################

########################
# Nuclear Form Factor
########################
def formFactor2(element, E):
    '''
    formFactor2(element,E)

    Returns the form-factor squared of element N with recoil energy E
    [E] = GeV
    '''
    E_N = 0.114/((atomicNumbers[element])**(5./3))
    FN2 = np.exp(-E/E_N)
    return FN2

########################
# Photon Scattering Cross Sections
########################
def crossSection(element, m_A, E_R): # returns 1/GeV^3
	'''
	crossSection(element, m_A, E_R)

    Returns the differntial scattering cross section for a massive dark photon

    [m_A] = GeV
    [E_R] = GeV
    '''

	m_N = amu2GeV(atomicNumbers[element])
	FN2 = formFactor2(element, E_R)
	function = ( FN2 ) / ((2 * m_N * E_R + m_A**2)**2)
	return function

def crossSectionKappa0(element, E_R): # Dimensionless
	'''
	crossSectionKappa0(element, E_R)

	Returns the cross section used in the kappa0 calculation

	[E_R] = GeV
	'''

	FN2 = formFactor2(element, E_R)
	function = FN2
	return function

########################
# Kinematics
########################
def eMin(u, m_X):
	'''
	eMin(u, m_X)

	Returns the minimum kinetic energy to become Gravitationally captured by Earth

	[m_X] = GeV
	'''

	function = (0.5) * m_X * u**2
#     assert (function >=0), '(u, m_X): (%e,%e) result in a negative eMin' % (u, m_X)
	return function

def eMax(element, m_X, rIndex, u):
	'''
	eMax(element, m_X, rIndex, u)

	Returns the maximum kinetic energy allowed by the kinematics

	[m_X] = GeV

	rIndex specifies the index in the escape velocity array escVel2_List[rIndex]
	'''

	m_N = amu2GeV(atomicNumbers[element])
	mu = m_N*m_X / (m_N + m_X)
	vCross2 = (escVel2_List[rIndex])
	function = 2 * mu**2 * (u**2 + vCross2) / m_N
#     assert (function >= 0), '(element, m_X, rIndex, u): (%s, %e, %i, %e) result in negative eMax' %(element, m_X, rIndex, u)
	return function

########################
# Intersection Velocity
########################
def EminEmaxIntersection(element, m_X, rIndex):
	'''
	EminEmaxIntersection(element, m_X, rIndex):

	Returns the velocity uInt when eMin = eMax.

	[m_X] = GeV

	'''
	m_N = amu2GeV(atomicNumbers[element])
	mu = (m_N*m_X)/(m_N+m_X)

	sqrtvCross2 = np.sqrt(escVel2_List[rIndex])
	# Calculate the intersection uInt of eMin and eMax given a specific rIndex
	A = m_X/2.
	B = 2. * mu**2 / m_N
	uInt = np.sqrt( ( B ) / (A-B) ) * sqrtvCross2

	return uInt

########################
# Photon Velocity and Energy Integration
########################
def intDuDEr(element, m_X, m_A, rIndex):
	'''
	intDuDER(element, m_X, m_A, rIndex):

	Returns the evaluated velocity and recoil energy integrals for dark photon scattering

	[m_X] = GeV
	[m_A] = GeV
	'''

	def integrand(E,u):
		fu = fCrossInterp(u)
		integrand = crossSection(element, m_A, E) * u * fu

		return integrand

	# Calculate the intersection uInt of eMin and eMax given a specific rIndex
	uInt = EminEmaxIntersection(element, m_X, rIndex)

	uLow = 0
	uHigh = min(uInt, V_gal) # We take the minimal value between the intersection velocity and galactic escape velocity
	eLow = lambda u: eMin(u, m_X)
	eHigh = lambda u: eMax(element, m_X, rIndex, u)
	integral = integrate.dblquad(integrand, uLow, uHigh, eLow, eHigh)[0]
	return integral

def intDuDErKappa0(element, m_X, rIndex):
	'''
	intDuDErKappa0(element, m_X, rIndex):

	returns the evaluated velocity and recoil energy integration for dark photon scattering
		used in the kappa0 calculation

	[m_X] = GeV

	'''

	def integrand(E_R,u):
		fu = fCrossInterp(u)
		integrand = crossSectionKappa0(element, E_R) * u * fu
		return integrand

	uInt = EminEmaxIntersection(element, m_X, rIndex)

	uLow = 0
	uHigh = min(uInt, V_gal) # We take the minimal value between the intersection velocity and galactic escape velocity
	eLow = lambda u: eMin(u, m_X)
	eHigh = lambda u: eMax(element, m_X, rIndex, u)
	integral = integrate.dblquad(integrand, uLow, uHigh, eLow, eHigh)[0]
	return integral

########################
# Sum Over Radii
########################
def sumOverR(element, m_X, m_A):
	'''
	sumOverR(element, m_X, m_A)

	Returns the summation over radius of the velocity and recoil energy integration

	[m_X] = GeV
	[m_A] = GeV
	'''

	tempSum = 0

	for i in range(0, len(radius_List)):
		r = radius_List[i]
		deltaR = deltaR_List[i]

		n_N = numDensity_Func(element)[i]

		summand = n_N * r**2 * intDuDEr(element, m_X, m_A, i) * deltaR
		tempSum += summand
	return tempSum

def sumOverRKappa0(element, m_X):
	'''
	sumOverRKappa0(element, m_X)

	Returns the summation over radius of the velocity and recoil energy integration
		used in the kappa0 calculation

	[m_X] = GeV
	'''
	tempSum = 0

	for i in range(0,len(radius_List)):
		r = radius_List[i]
		deltaR = deltaR_List[i]

		n_N = numDensity_Func(element)[i]

		summand = n_N * r**2 * intDuDErKappa0(element, m_X, i) * deltaR
		tempSum += summand
	return tempSum

########################
# Single Element Capture Rate
########################
def singleElementCap(element, m_X, m_A, epsilon, alpha, alpha_X):
	'''
	singleElementCap(element, m_X, m_A, epsilon, alpha, alpha_X)

	Returns the capture rate due to a single element for the specified parameters

	[m_X] = GeV
	[m_A] = GeV
	'''
	Z_N = nProtons[element]
	m_N = amu2GeV(atomicNumbers[element])
	n_X = 0.3/m_X # GeV/cm^3

	conversion = (5.06e13)**-3 * (1.52e24) # Conversion to seconds (cm^-3)(GeV^-2) -> (s^-1)
	prefactors = (4*np.pi)**2
	crossSectionFactors = 2 * (4*np.pi) * epsilon**2 * alpha_X * alpha * Z_N**2 * m_N
	function = n_X * conversion* crossSectionFactors* prefactors * sumOverR(element, m_X, m_A)
	return function

def singleElementCapKappa0(element, m_X, alpha):
	'''
	singleElementCapKappa0(element, m_X, alpha):

	Returns a single kappa0 value for 'element' and the specified parameters

	[m_X] = GeV
	'''
	Z_N = nProtons[element]
	m_N = amu2GeV(atomicNumbers[element])
	n_X = 0.3/m_X # 1/cm^3

	conversion = (5.06e13)**-3 * (1.52e24) # (cm^-3)(GeV^-2) -> (s^-1)
	crossSectionFactors = 2 * (4*np.pi) * alpha * Z_N**2 * m_N

	prefactor = (4*np.pi)**2

	function = n_X * conversion * prefactor * crossSectionFactors * sumOverRKappa0(element, m_X)
	return function

########################
# Full Capture Rate
########################
def cCap(m_X, m_A, epsilon, alpha, alpha_X):
	'''
	cCap(m_X, m_A, epsilon, alpha, alpha_X)

	returns the full capture rate in sec^-1 for the specified parameters

	Note: This function is the less efficient way to perform this calculation. Every point in (m_A, epsilon) space
		involves peforming the full tripple integral over recoil energy, incident DM velocity, and Earth radius
		which is time consuming.

	[m_X] = GeV
	[m_A] = GeV
	'''
	totalCap = 0
	for element in element_List:
		elementCap = singleElementCap(element, m_X, m_A, epsilon, alpha, alpha_X)
		print ('Element:', element,',' 'Cap: ', elementCap)
		totalCap += elementCap
	return totalCap

########################
# Kappa0
########################
def kappa_0(m_X, alpha):
	'''
	kappa_0(m_X, alpha)

	Returns the kappa0 value for m_X and alpha

	[m_X] = GeV

	This funciton encodes how the capture rate depends on m_X and alpha.
	'''
	tempSum = 0 # tempSum = "temporary sum" not "temperature sum"
	for element in element_List:
		function = singleElementCapKappa0(element, m_X, alpha)
		tempSum += function

	return tempSum

########################
# Capture Rate the quick way
########################
def cCapQuick(m_X, m_A, epsilon, alpha_X, kappa0):
	'''
	cCapQuick(m_X, m_A, epsilon, alpha_X, kappa0):

	Returns the Capture rate in a much more computationally efficient way. For a given dark matter mass m_X and coupling
		constant alpha, we calculate the quantity kappa_0 once and multiply it by the differential cross section approximation
		for each point in (m_A, epsilon) space.

	[m_X] = GeV
	[m_A] = GeV

	Provides a quick way to calculate the capture rate when only m_A and epsilon are changing.

	All the m_X dependence, which is assumed to be fixed, is contianed in kappa0.
	Note that m_X is defined as an input to this function, but not actually used.
		This was made in light of keeping the function definitions in Python consistent.
	'''
	function = epsilon**2 * alpha_X * kappa0 / m_A**4
	return function

################################################################
# Thermal Relic
################################################################
def alphaTherm(m_X, m_A):
    '''
    alphaTherm(m_X,m_A)

    [m_X] = GeV
    [m_A] = GeV

    This function determines the dark matter fine structure constant alpha_X given the dark matter relic abundance.
    '''

    conversion = (5.06e13)**3/ (1.52e24) # cm^3 Sec -> GeV^-2
    thermAvgSigmaV = 2.2e-26 # cm^3/s from ArXiV: 1602.01465v3 between eqns (4) and (5)

    function = conversion * thermAvgSigmaV * (m_X**2/np.pi) \
    * (1 - 0.5*(m_A/m_X)**2)**2 / ((1 - (m_A/m_X)**2)**(3./2))
    return np.sqrt(function)

# Thermal Relic for m_X >> m_A Approximation
def alphaThermApprox(m_X):
    '''
    alphaThermApprox(m_X)

    This function determines the dark matter fine structure constant alpha_X given the dark matter relic abundance
    	in the m_X >> m_A approximation.
    '''
    conversion = (5.06e13)**3/ (1.52e24) # cm^3 Sec -> GeV^-2
    thermAvgSigmaV = 2.2e-26 # cm^3/s from ArXiV: 1602.01465v3 between eqns (4) and (5)

    function = conversion * thermAvgSigmaV * (5.06e13)**3/ (1.52e24) * (m_X**2/np.pi)
    return np.sqrt(function)

################################################################
# Annihilation Rate Functions
################################################################

########################
# V0 at center of Earth
########################
def v0func(m_X):
	'''
	v0func(m_X)

	Returns the characteristic velocity of a dark matter particle with mass m_X at the center of the Earth.

	[m_X] = GeV
	'''
	return np.sqrt(2*TCross/m_X)

########################
# Tree-level annihilation cross section
########################
def sigmaVtree(m_X, m_A, alpha_X):
	'''
	sigmaVtree(m_X, m_A, alpha_X)

	Returns the tree-level annihilation cross section for massive dark photons fixed by relic abundance

	[m_X] = GeV
	[m_A] = GeV
	'''
	numerator = (1 - (m_A/m_X)**2)**1.5
	denominator = ( 1 - 0.5 * (m_A/m_X)**2 )**2
	prefactor = np.pi*(alpha_X/m_X)**2

	function = prefactor * numerator/denominator
	return function

########################
# Sommerfeld Enhahcement
########################
def sommerfeld(v, m_X, m_A, alpha_X):
	'''
	sommerfeld(v, m_X, m_A, alpha_X)

	Returns the Sommerfeld enhancemement

	[m_X] = GeV
	[m_A] = GeV
	'''
	a = v / (2 * alpha_X)						# Variable substitution
	c = 6 * alpha_X * m_X / (np.pi**2 * m_A)    # Variable substitution

	# Kludge: Absolute value the argument of the square root inside Cos(...) two lines below.
	function = np.pi/a * np.sinh(2*np.pi*a*c) / \
		( np.cosh(2*np.pi*a*c) - np.cos(2*np.pi*np.abs(np.sqrt(np.abs(c-(a*c)**2)) ) ) )
	return function

########################
# Thermal Average Sommerfeld
########################
def thermAvgSommerfeld(m_X, m_A, alpha_X):
	'''
	thermAvgSommerfeld(m_X, m_A, alpha_X):

	Returns the thermally-averaged Sommerfeld enhancement

	[m_X] = GeV
	[m_A] = GeV
	'''

	v0 = v0func(m_X)

	def integrand(v):
		# We perform d^3v in spherical velocity space.
		# d^3v = v^2 dv * d(Omega)
		prefactor = 4*np.pi/(2*np.pi*v0**2)**(1.5)
		function = prefactor * v**2 * np.exp(-0.5*(v/v0)**2) * sommerfeld(v, m_X, m_A, alpha_X)
		return function

	lowV = 0
	# Python doesn't like it when you integrate to infinity, so we integrate to 10 standard deviations
	highV = 10*(v0func(m_X))

	integral = integrate.quad(integrand, lowV, highV)[0]
	return integral

########################
#  CAnnCalc
########################
def cAnn(m_X, sigmaVTree, thermAvgSomm = 1):
	'''
	CAnnCalc(m_X, sigmaVTree, thermAvgSomm = 1)

	Returns the Annihilation rate in sec^-1 without Sommerfeld effects.
	To include sommerfeld effects, set thermAvgSomm = thermAvgSommerfeld(m_X, m_A, alpha_X)

	[m_X] = GeV
	[sigmaVTree] = GeV^-2
	'''
	prefactor = (Gnat * m_X * rhoCross/ (3 * TCross) )**(3./2)
	conversion = (1.52e24) # GeV -> Sec^-1
	function = conversion * prefactor * sigmaVTree * thermAvgSomm
	return function

################################################################
# Annihilation Rate Functions
################################################################

########################
# Equilibrium Time
########################
def tau(CCap,CAnn):
	'''
	tau(CCap,CAnn)

	Returns the equilibrium time in sec

	[CCap] = sec^-1
	[CAnn] = sec^-1
	'''
	function = 1./(np.sqrt(CCap*CAnn))

	return function

########################
# Epsilon as a function of m_A
########################
def contourFunction(m_A, alpha_X, Cann0, Sommerfeld, kappa0, contourLevel):
	'''
	EpsilonFuncMA(m_A, alpha_X, Cann, Sommerfeld, kappa0, contourLevel)

	Returns the value of epsilon as a function of mediator mass m_A.

	[m_A]  = GeV
	Cann0  = sec^-1
	Kappa0 = GeV^5

	Note: The 10^n contour is input as contourLevel = n
	This function is used to quickly generate plots of constant tau/tau_Cross in (m_A, epsilon) space.
	'''
	function = 2 * np.log10(m_A) - (0.5)*np.log10(alpha_X * kappa0 * Cann0 * Sommerfeld) \
	            - contourLevel - np.log10(tauCross)
	return function

########################
# Annihilation Rate
########################
def gammaAnn(CCap, CAnn):
	'''
	gammaAnn(CCap, CAnn)

	Returns the solution to the differential rate equation for dark matter capture and annihilation

	[CCap] = sec^-1
	[CAnn] = sec^-1
	'''
	Tau = tau(CCap, CAnn)
	EQRatio = tauCross/Tau
	function = (0.5) * CCap * ((np.tanh(EQRatio))**2)

	return function

########################
# Decay Length
########################

def decayLength(m_X, m_A, epsilon, BR):
	'''
	DecayLength(m_X, m_A, epsilon, BR)

	Returns the characteristic decay length of dark photons in cm.

	[m_X] = GeV
	[m_A] = GeV
	BR = Branching Ratio
	'''
	function = RCross * BR * (3.6e-9/epsilon)**2 * (m_X/m_A) * (1./1000) * (1./m_A)
	return function

########################
# Decay Parameter
########################
def epsilonDecay(decayLength, effectiveDepth = 10**5): # Effective depth = 1 km
	'''
	epsilonDecay(decayLength, effectiveDepth = 10**5)

	Returns the probability for dark photons to decay inside the IceCube detector near the surface of Earth.

	[effectiveDepth] = cm, default value for the IceCube Neutrino Observatory is 1 km.
	'''
	arg1 = RCross                   # To make the arguments of the exponentials nice
	arg2 = RCross + effectiveDepth  # To make the arguments of the exponentials nice

	function = np.exp(-arg1/decayLength) - np.exp(-arg2/decayLength)
	return function

########################
# Ice Cube Signal
########################
def iceCubeSignal(gammaAnn, epsilonDecay, T, Aeff = 10**10):
	'''
	iceCubeSignal(gammaAnn, epsilonDecay, liveTime, Aeff = 10**10)

	Returns the number of signal events for IceCube.

	[gammaAnn] = sec^-1
	[liveTime] = sec
	[Aeff]     = cm^2
	'''
	function = 2 * gammaAnn * (Aeff/ (4*np.pi*RCross**2) ) * epsilonDecay * T
	return function

```

We're interested in computing the total capture $C_{cap}$ as a function of dark matter mas $m_X$. The total capture rate is defined as the sum of individual capture rates for each element $C_{Cap}^N$:

$$
\begin{equation}
    C_{Cap} = \sum_N C_{Cap}^N
\end{equation}
$$

Where the individual rates are defined as :

$$
\begin{equation}
    C_{Cap}^N = n_X\int_0^{R_\oplus}dr4\pi r^2 n_N(r)\int_0^{v_{gal}}du4\pi u^2f_\oplus(u)\frac{u^2 + v_\oplus^2(r)}{u}\int_{E_{min}}^{E_{max}}dE_R\frac{d\sigma_N}{dE_R}\Theta(\Delta E)
\end{equation}
$$

The details of the components of this expression are included in the DarkCapPy manual.

Using the same limits as detailed in that manual, we can factorise the expression of $C_{Cap}$ as :

$$
\begin{equation}
    C_{Cap} = \left(\frac{\varepsilon^2}{m_A^4}\alpha_X\right)\kappa_0(m_X, \alpha)
\end{equation}
$$

Therefore, the $C_{Cap}$ dependency in $m_X$ is enclosed in $\kappa_0$, which itself does not depends on neither the kinetic mixing parameter $\varepsilon$ nor the dark mediator mass $m_A$.

Most of the computing time for the capture rate lies in the $\kappa_0$ computation. Computing $\kappa_0$ over a range of $m_X$ value and saving it into a file allows us to quickly compute the capture rate for different values of $\varepsilon$ and $m_A$ without having to go through long and heavy computation.

Now we import the csv file containing all computed $\kappa_0$ values for $m_X$ ranging from 10 to 10 000 GeV, as a pandas DataFrame, and plot it.

The 200 values of $mx$ used within that range are distributed logarithmically (np.logspace(1, 4, 200)). $\alpha = \frac{1}{137}$ is the fine structure constant

```python
data = pd.read_csv('kappa0_200', index_col = 0)
data.columns = ['mx', 'kappa']
data
```

      mx
      kappa

      0
      10.000000
      1.614263e+27

      1
      10.353218
      1.847457e+27

      2
      10.718913
      2.139252e+27

      3
      11.097525
      2.514517e+27

      4
      11.489510
      3.013940e+27

      ...
      ...
      ...

      195
      8703.591361
      4.130521e+23

      196
      9011.018252
      3.853472e+23

      197
      9329.304026
      3.595007e+23

      198
      9658.832241
      3.353879e+23

      199
      10000.000000
      3.128925e+23

200 rows × 2 columns

```python
mx = data['mx'].values
kappa = data['kappa'].values
```

We define an array containing the main elements found in Earth's Mantle, and their mass expressed in GeV, which will be useful for annotating further graphs

```python
elementMass = []
elements = []
for i, element in enumerate(atomicNumbers) :
    elementMass.append(amu2GeV(atomicNumbers[element]))
    elements.append(element)
elementsMass = np.transpose(np.array([elements, elementMass]))
```

We use scipy's interpolate function to define a kapa function that interpolate values of $\kappa_0$ based on the computed data. This will be used to place the annotation regarding elements

```python
kapa = interpolate.interp1d(mx, kappa, kind = 'cubic')
```

We only keep elements that fall in the $m_X$ range

```python
elementsMass = elementsMass[np.where(np.array(elementMass) > 10)[0], :]
```

We plot $\kappa_0(m_X)$, and annotate it with the elements list

```python
fig, ax = plt.subplots(1, 1, figsize = (16, 9), dpi = 300)

ax.scatter(mx, kappa, label = '$\\kappa_0(m_X)$', marker = '^',
           color = 'g')
for i, elementMass in enumerate(elementsMass) :
    mass = float(elementMass[1])
    y = kapa(mass)
    if i%2 == 0 :

        offsetY = 10**26+y/10
    else :

        offsetY = 10**29 + y*10

    ax.annotate(elementMass[0], (mass, y), xytext = (mass, offsetY), arrowprops = dict(arrowstyle = '->'))

ax.legend()
ax.grid()
ax.set_ylim(1e23, 1e31)
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_ylabel('$\\kappa_0$ [GeV$^{5}$]')
ax.set_xlabel('$m_X$ [GeV]')

fig.tight_layout()
```

The shape of $\kappa_0(m_X)$ is as we expect it to be. We can now compute different $C_{Cap}(m_X)$ curves for different values of $(\varepsilon, m_A)$ quickly, as all the heavy computing bit has been done in computing $\kappa_0(m_X)$. Let's try it out for $m_A = 0.1$ GeV and $\varepsilon = 10^{-8}$.

```python
alpha_X = 0.035
alpha = 1/137
epsilon = 1e-8
m_A = 0.1
Cap = np.empty_like(mx)
```

It might be interesting to try using numba to vectorize the cCapQuick function, so that we can compute $C_{Cap}$ for the whole $m_X$ range at once, instead of having to loop through all $m_X$ values.

```python
for i, m in enumerate(mx) :
    Cap[i] = cCapQuick(m, m_A, epsilon, alpha_X, kappa[i])
```

We define a new function so that we can place our elements label, the same way we did for $\kappa_0$

```python
cap = interpolate.interp1d(mx, Cap, kind = 'cubic')
```

And plot it !

```python
fig, ax = plt.subplots(1, 1, figsize = (16, 9), dpi = 300)

ax.scatter(mx, Cap, label = '$C_{Cap}(m_X)$ for $m_A = 0.1$ GeV and $\\varepsilon = 10^{-8}$', marker = '^',
           color = 'g')
for i, elementMass in enumerate(elementsMass) :
    mass = float(elementMass[1])
    y = cap(mass)
    if i%2 == 0 :

        offsetY = 10**13+y/10
    else :

        offsetY = 10**15 + y*10

    ax.annotate(elementMass[0], (mass, y), xytext = (mass, offsetY), arrowprops = dict(arrowstyle = '->'))

ax.legend()
ax.grid()
ax.set_ylim(1e10, 1e17)
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_ylabel('$C_{Cap}$ [$s^{-1}$]')
ax.set_xlabel('$m_X$ [GeV]')

fig.tight_layout()
```

```python
fig.savefig('capture_rate')
```