# Neuromatch Academy: Week 1, Day 5, Tutorial 3
# Dimensionality Reduction and reconstruction

__Content creators:__ Alex Cayco Gajic, John Murray

__Content reviewers:__ Roozbeh Farhoudi, Matt Krause, Spiros Chavlis, Richard Gao, Michael Waskom

---
# Tutorial Objectives

In this notebook we'll learn to apply PCA for dimensionality reduction, using a classic dataset that is often used to benchmark machine learning algorithms: MNIST. We'll also learn how to use PCA for reconstruction and denoising.

Overview:
- Perform PCA on MNIST
- Calculate the variance explained
- Reconstruct data with different numbers of PCs
- (Bonus) Examine denoising using PCA

You can learn more about MNIST dataset [here](https://en.wikipedia.org/wiki/MNIST_database).

```python
# @title Video 1: PCA for dimensionality reduction
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="oO0bbInoO_0", width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=oO0bbInoO_0

---
# Setup
Run these cells to get the tutorial started.

```python
# Imports
import numpy as np
import matplotlib.pyplot as plt
```

```python
# @title Figure Settings
import ipywidgets as widgets   # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
# @title Helper Functions

def plot_variance_explained(variance_explained):
  """
  Plots eigenvalues.

  Args:
    variance_explained (numpy array of floats) : Vector of variance explained
                                                 for each PC

  Returns:
    Nothing.

  """

  plt.figure()
  plt.plot(np.arange(1, len(variance_explained) + 1), variance_explained,
           '--k')
  plt.xlabel('Number of components')
  plt.ylabel('Variance explained')
  plt.show()

def plot_MNIST_reconstruction(X, X_reconstructed):
  """
  Plots 9 images in the MNIST dataset side-by-side with the reconstructed
  images.

  Args:
    X (numpy array of floats)               : Data matrix each column
                                              corresponds to a different
                                              random variable
    X_reconstructed (numpy array of floats) : Data matrix each column
                                              corresponds to a different
                                              random variable

  Returns:
    Nothing.
  """

  plt.figure()
  ax = plt.subplot(121)
  k = 0
  for k1 in range(3):
    for k2 in range(3):
      k = k + 1
      plt.imshow(np.reshape(X[k, :], (28, 28)),
                 extent=[(k1 + 1) * 28, k1 * 28, (k2 + 1) * 28, k2 * 28],
                 vmin=0, vmax=255)
  plt.xlim((3 * 28, 0))
  plt.ylim((3 * 28, 0))
  plt.tick_params(axis='both', which='both', bottom=False, top=False,
                  labelbottom=False)
  ax.set_xticks([])
  ax.set_yticks([])
  plt.title('Data')
  plt.clim([0, 250])
  ax = plt.subplot(122)
  k = 0
  for k1 in range(3):
    for k2 in range(3):
      k = k + 1
      plt.imshow(np.reshape(np.real(X_reconstructed[k, :]), (28, 28)),
                 extent=[(k1 + 1) * 28, k1 * 28, (k2 + 1) * 28, k2 * 28],
                 vmin=0, vmax=255)
  plt.xlim((3 * 28, 0))
  plt.ylim((3 * 28, 0))
  plt.tick_params(axis='both', which='both', bottom=False, top=False,
                  labelbottom=False)
  ax.set_xticks([])
  ax.set_yticks([])
  plt.clim([0, 250])
  plt.title('Reconstructed')
  plt.tight_layout()

def plot_MNIST_sample(X):
  """
  Plots 9 images in the MNIST dataset.

  Args:
     X (numpy array of floats) : Data matrix each column corresponds to a
                                 different random variable

  Returns:
    Nothing.

  """

  fig, ax = plt.subplots()
  k = 0
  for k1 in range(3):
    for k2 in range(3):
      k = k + 1
      plt.imshow(np.reshape(X[k, :], (28, 28)),
                 extent=[(k1 + 1) * 28, k1 * 28, (k2+1) * 28, k2 * 28],
                 vmin=0, vmax=255)
  plt.xlim((3 * 28, 0))
  plt.ylim((3 * 28, 0))
  plt.tick_params(axis='both', which='both', bottom=False, top=False,
                  labelbottom=False)
  plt.clim([0, 250])
  ax.set_xticks([])
  ax.set_yticks([])
  plt.show()

def plot_MNIST_weights(weights):
  """
  Visualize PCA basis vector weights for MNIST. Red = positive weights,
  blue = negative weights, white = zero weight.

  Args:
     weights (numpy array of floats) : PCA basis vector

  Returns:
     Nothing.
  """

  fig, ax = plt.subplots()
  cmap = plt.cm.get_cmap('seismic')
  plt.imshow(np.real(np.reshape(weights, (28, 28))), cmap=cmap)
  plt.tick_params(axis='both', which='both', bottom=False, top=False,
                  labelbottom=False)
  plt.clim(-.15, .15)
  plt.colorbar(ticks=[-.15, -.1, -.05, 0, .05, .1, .15])
  ax.set_xticks([])
  ax.set_yticks([])
  plt.show()

def add_noise(X, frac_noisy_pixels):
  """
  Randomly corrupts a fraction of the pixels by setting them to random values.

  Args:
     X (numpy array of floats)  : Data matrix
     frac_noisy_pixels (scalar) : Fraction of noisy pixels

  Returns:
     (numpy array of floats)    : Data matrix + noise

  """

  X_noisy = np.reshape(X, (X.shape[0] * X.shape[1]))
  N_noise_ixs = int(X_noisy.shape[0] * frac_noisy_pixels)
  noise_ixs = np.random.choice(X_noisy.shape[0], size=N_noise_ixs,
                               replace=False)
  X_noisy[noise_ixs] = np.random.uniform(0, 255, noise_ixs.shape)
  X_noisy = np.reshape(X_noisy, (X.shape[0], X.shape[1]))

  return X_noisy

def change_of_basis(X, W):
  """
  Projects data onto a new basis.

  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors

  Returns:
    (numpy array of floats)   : Data matrix expressed in new basis
  """

  Y = np.matmul(X, W)

  return Y

def get_sample_cov_matrix(X):
  """
  Returns the sample covariance matrix of data X.

  Args:
    X (numpy array of floats) : Data matrix each column corresponds to a
                                different random variable

  Returns:
    (numpy array of floats)   : Covariance matrix
"""

  X = X - np.mean(X, 0)
  cov_matrix = 1 / X.shape[0] * np.matmul(X.T, X)
  return cov_matrix

def sort_evals_descending(evals, evectors):
  """
  Sorts eigenvalues and eigenvectors in decreasing order. Also aligns first two
  eigenvectors to be in first two quadrants (if 2D).

  Args:
    evals (numpy array of floats)    :   Vector of eigenvalues
    evectors (numpy array of floats) :   Corresponding matrix of eigenvectors
                                         each column corresponds to a different
                                         eigenvalue

  Returns:
    (numpy array of floats)          : Vector of eigenvalues after sorting
    (numpy array of floats)          : Matrix of eigenvectors after sorting
  """

  index = np.flip(np.argsort(evals))
  evals = evals[index]
  evectors = evectors[:, index]
  if evals.shape[0] == 2:
    if np.arccos(np.matmul(evectors[:, 0],
                           1 / np.sqrt(2) * np.array([1, 1]))) > np.pi / 2:
      evectors[:, 0] = -evectors[:, 0]
    if np.arccos(np.matmul(evectors[:, 1],
                           1 / np.sqrt(2)*np.array([-1, 1]))) > np.pi / 2:
      evectors[:, 1] = -evectors[:, 1]

  return evals, evectors

def pca(X):
  """
  Performs PCA on multivariate data. Eigenvalues are sorted in decreasing order

  Args:
     X (numpy array of floats) :   Data matrix each column corresponds to a
                                   different random variable

  Returns:
    (numpy array of floats)    : Data projected onto the new basis
    (numpy array of floats)    : Vector of eigenvalues
    (numpy array of floats)    : Corresponding matrix of eigenvectors

  """

  X = X - np.mean(X, 0)
  cov_matrix = get_sample_cov_matrix(X)
  evals, evectors = np.linalg.eigh(cov_matrix)
  evals, evectors = sort_evals_descending(evals, evectors)
  score = change_of_basis(X, evectors)

  return score, evectors, evals

def plot_eigenvalues(evals, limit=True):
  """
  Plots eigenvalues.

  Args:
     (numpy array of floats) : Vector of eigenvalues

  Returns:
    Nothing.

  """

  plt.figure()
  plt.plot(np.arange(1, len(evals) + 1), evals, 'o-k')
  plt.xlabel('Component')
  plt.ylabel('Eigenvalue')
  plt.title('Scree plot')
  if limit:
    plt.show()
```

---
# Section 1: Perform PCA on MNIST

The MNIST dataset consists of a 70,000 images of individual handwritten digits. Each image is a 28x28 pixel grayscale image. For convenience, each 28x28 pixel image is often unravelled into a single 784 (=28*28) element vector, so that the whole dataset is represented as a 70,000 x 784 matrix. Each row represents a different image, and each column represents a different pixel.

Enter the following cell to load the MNIST dataset and plot the first nine images.

```python
from sklearn.datasets import fetch_openml
mnist = fetch_openml(name='mnist_784')
X = mnist.data
plot_MNIST_sample(X)
```

The MNIST dataset has an extrinsic dimensionality of 784, much higher than the 2-dimensional examples used in the previous tutorials! To make sense of this data, we'll use dimensionality reduction. But first, we need to determine the intrinsic dimensionality $K$ of the data. One way to do this is to look for an "elbow" in the scree plot, to determine which eigenvalues are signficant.

## Exercise 1: Scree plot of MNIST

In this exercise you will examine the scree plot in the MNIST dataset.

**Steps:**
- Perform PCA on the dataset and examine the scree plot.
- When do the eigenvalues appear (by eye) to reach zero? (**Hint:** use `plt.xlim` to zoom into a section of the plot).

```python
help(pca)
help(plot_eigenvalues)
```

    Help on function pca in module __main__:

    pca(X)
        Performs PCA on multivariate data. Eigenvalues are sorted in decreasing order

        Args:
           X (numpy array of floats) :   Data matrix each column corresponds to a
                                         different random variable

        Returns:
          (numpy array of floats)    : Data projected onto the new basis
          (numpy array of floats)    : Vector of eigenvalues
          (numpy array of floats)    : Corresponding matrix of eigenvectors

    Help on function plot_eigenvalues in module __main__:

    plot_eigenvalues(evals, limit=True)
        Plots eigenvalues.

        Args:
           (numpy array of floats) : Vector of eigenvalues

        Returns:
          Nothing.

```python
#################################################
## TO DO for students: perform PCA and plot the eigenvalues
#################################################

# perform PCA
# score, evectors, evals = ...
# plot the eigenvalues
# plot_eigenvalues(evals, limit=False)
# plt.xlim(...)  # limit x-axis up to 100 for zooming
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_a876e927.py)

*Example output:*

---
# Section 2: Calculate the variance explained

The scree plot suggests that most of the eigenvalues are near zero, with fewer than 100 having large values. Another common way to determine the intrinsic dimensionality is by considering the variance explained. This can be examined with a cumulative plot of the fraction of the total variance explained by the top $K$ components, i.e.,

\begin{equation}
\text{var explained} = \frac{\sum_{i=1}^K \lambda_i}{\sum_{i=1}^N \lambda_i}
\end{equation}

The intrinsic dimensionality is often quantified by the $K$ necessary to explain a large proportion of the total variance of the data (often a defined threshold, e.g., 90%).

## Exercise 2: Plot the explained variance

In this exercise you will plot the explained variance.

**Steps:**
- Fill in the function below to calculate the fraction variance explained as a function of the number of principal componenets. **Hint:** use `np.cumsum`.
- Plot the variance explained using `plot_variance_explained`.

**Questions:**
- How many principal components are required to explain 90% of the variance?
- How does the intrinsic dimensionality of this dataset compare to its extrinsic dimensionality?

```python
help(plot_variance_explained)
```

    Help on function plot_variance_explained in module __main__:

    plot_variance_explained(variance_explained)
        Plots eigenvalues.

        Args:
          variance_explained (numpy array of floats) : Vector of variance explained
                                                       for each PC

        Returns:
          Nothing.

```python
def get_variance_explained(evals):
  """
  Calculates variance explained from the eigenvalues.

  Args:
    evals (numpy array of floats) : Vector of eigenvalues

  Returns:
    (numpy array of floats)       : Vector of variance explained

  """

  #################################################
  ## TO DO for students: calculate the explained variance using the equation
  ## from Section 2.
  # Comment once you've filled in the function
  raise NotImplementedError("Student excercise: calculate explaine variance!")
  #################################################

  # cumulatively sum the eigenvalues
  csum = ...
  # normalize by the sum of eigenvalues
  variance_explained = ...

  return variance_explained

#################################################
## TO DO for students: call the function and plot the variance explained
#################################################

# calculate the variance explained
variance_explained = ...

# Uncomment to plot the variance explained
# plot_variance_explained(variance_explained)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_0f5f51b9.py)

*Example output:*

---
# Section 3: Reconstruct data with different numbers of PCs

Now we have seen that the top 100 or so principal components of the data can explain most of the variance. We can use this fact to perform *dimensionality reduction*, i.e., by storing the data using only 100 components rather than the samples of all 784 pixels. Remarkably, we will be able to reconstruct much of the structure of the data using only the top 100 components. To see this, recall that to perform PCA we projected the data $\bf X$ onto the eigenvectors of the covariance matrix:
\begin{equation}
\bf S = X W
\end{equation}
Since $\bf W$ is an orthogonal matrix, ${\bf W}^{-1} = {\bf W}^T$. So by multiplying by ${\bf W}^T$ on each side we can rewrite this equation as
\begin{equation}
{\bf X = S W}^T.
\end{equation}
This now gives us a way to reconstruct the data matrix from the scores and loadings. To reconstruct the data from a low-dimensional approximation, we just have to truncate these matrices.  Let's call ${\bf S}_{1:K}$ and ${\bf W}_{1:K}$ as keeping only the first $K$ columns of this matrix. Then our reconstruction is:
\begin{equation}
{\bf \hat X = S}_{1:K} ({\bf W}_{1:K})^T.
\end{equation}

## Exercise 3: Data reconstruction

Fill in the function below to reconstruct the data using different numbers of principal components.

**Steps:**

* Fill in the following function to reconstruct the data based on the weights and scores. Don't forget to add the mean!
* Make sure your function works by reconstructing the data with all $K=784$ components. The two images should look identical.

```python
help(plot_MNIST_reconstruction)
```

    Help on function plot_MNIST_reconstruction in module __main__:

    plot_MNIST_reconstruction(X, X_reconstructed)
        Plots 9 images in the MNIST dataset side-by-side with the reconstructed
        images.

        Args:
          X (numpy array of floats)               : Data matrix each column
                                                    corresponds to a different
                                                    random variable
          X_reconstructed (numpy array of floats) : Data matrix each column
                                                    corresponds to a different
                                                    random variable

        Returns:
          Nothing.

```python
def reconstruct_data(score, evectors, X_mean, K):
  """
  Reconstruct the data based on the top K components.

  Args:
    score (numpy array of floats)    : Score matrix
    evectors (numpy array of floats) : Matrix of eigenvectors
    X_mean (numpy array of floats)   : Vector corresponding to data mean
    K (scalar)                       : Number of components to include

  Returns:
    (numpy array of floats)          : Matrix of reconstructed data

  """

  #################################################
  ## TO DO for students: Reconstruct the original data in X_reconstructed
  # Comment once you've filled in the function
  raise NotImplementedError("Student excercise: reconstructing data function!")
  #################################################

  # Reconstruct the data from the score and eigenvectors
  # Don't forget to add the mean!!
  X_reconstructed =  ...

  return X_reconstructed

K = 784

#################################################
## TO DO for students: Calculate the mean and call the function, then plot
## the original and the recostructed data
#################################################

# Reconstruct the data based on all components
X_mean = ...
X_reconstructed = ...

# Plot the data and reconstruction
# plot_MNIST_reconstruction(X, X_reconstructed)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_e3395916.py)

*Example output:*

## Interactive Demo: Reconstruct the data matrix using different numbers of PCs

Now run the code below and experiment with the slider to reconstruct the data matrix using different numbers of principal components.

**Steps**
* How many principal components are necessary to reconstruct the numbers (by eye)? How does this relate to the intrinsic dimensionality of the data?
* Do you see any information in the data with only a single principal component?

```python
# @title

# @markdown Make sure you execute this cell to enable the widget!

def refresh(K=100):
  X_reconstructed = reconstruct_data(score, evectors, X_mean, K)
  plot_MNIST_reconstruction(X, X_reconstructed)
  plt.title('Reconstructed, K={}'.format(K))

_ = widgets.interact(refresh, K=(1, 784, 10))
```

## Exercise 4: Visualization of the weights

Next, let's take a closer look at the first principal component by visualizing its corresponding weights.

**Steps:**

* Enter `plot_MNIST_weights` to visualize the weights of the first basis vector.
* What structure do you see? Which pixels have a strong positive weighting? Which have a strong negative weighting? What kinds of images would this basis vector differentiate?
* Try visualizing the second and third basis vectors. Do you see any structure? What about the 100th basis vector? 500th? 700th?

```python
help(plot_MNIST_weights)
```

    Help on function plot_MNIST_weights in module __main__:

    plot_MNIST_weights(weights)
        Visualize PCA basis vector weights for MNIST. Red = positive weights,
        blue = negative weights, white = zero weight.

        Args:
           weights (numpy array of floats) : PCA basis vector

        Returns:
           Nothing.

```python
#################################################
## TO DO for students: plot the weights calling the plot_MNIST_weights function
#################################################

# Plot the weights of the first principal component
# plot_MNIST_weights(...)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_f358e413.py)

*Example output:*

---
# Summary
* In this tutorial, we learned how to use PCA for dimensionality reduction by selecting the top principal components. This can be useful as the intrinsic dimensionality ($K$) is often less than the extrinsic dimensionality ($N$) in neural data. $K$ can be inferred by choosing the number of eigenvalues necessary to capture some fraction of the variance.
* We also learned how to reconstruct an approximation of the original data using the top $K$ principal components. In fact, an alternate formulation of PCA is to find the $K$ dimensional space that minimizes the reconstruction error.
* Noise tends to inflate the apparent intrinsic dimensionality, however the higher components reflect noise rather than new structure in the data. PCA can be used for denoising data by removing noisy higher components.
* In MNIST, the weights corresponding to the first principal component appear to discriminate between a 0 and 1. We will discuss the implications of this for data visualization in the following tutorial.

---
# Bonus: Examine denoising using PCA

In this lecture, we saw that PCA finds an optimal low-dimensional basis to minimize the reconstruction error. Because of this property, PCA can be useful for denoising corrupted samples of the data.

## Exercise 5: Add noise to the data
In this exercise you will add salt-and-pepper noise to the original data and see how that affects the eigenvalues.

**Steps:**
- Use the function `add_noise` to add noise to 20% of the pixels.
- Then, perform PCA and plot the variance explained. How many principal components are required to explain 90% of the variance? How does this compare to the original data?

```python
help(add_noise)
```

    Help on function add_noise in module __main__:

    add_noise(X, frac_noisy_pixels)
        Randomly corrupts a fraction of the pixels by setting them to random values.

        Args:
           X (numpy array of floats)  : Data matrix
           frac_noisy_pixels (scalar) : Fraction of noisy pixels

        Returns:
           (numpy array of floats)    : Data matrix + noise

```python
###################################################################
# Insert your code here to:
# Add noise to the data
# Plot noise-corrupted data
# Perform PCA on the noisy data
# Calculate and plot the variance explained
###################################################################
np.random.seed(2020)  # set random seed
X_noisy = ...
# score_noisy, evectors_noisy, evals_noisy = ...
# variance_explained_noisy = ...
# plot_MNIST_sample(X_noisy)
# plot_variance_explained(variance_explained_noisy)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_d4a41b8c.py)

*Example output:*

## Exercise 6: Denoising

Next, use PCA to perform denoising by projecting the noise-corrupted data onto the basis vectors found from the original dataset. By taking the top K components of this projection, we can reduce noise in dimensions orthogonal to the K-dimensional latent space.

**Steps:**
- Subtract the mean of the noise-corrupted data.
- Project the data onto the basis found with the original dataset (`evectors`, not `evectors_noisy`) and take the top $K$ components.
- Reconstruct the data as normal, using the top 50 components.
- Play around with the amount of noise and K to build intuition.

```python
###################################################################
# Insert your code here to:
# Subtract the mean of the noise-corrupted data
# Project onto the original basis vectors evectors
# Reconstruct the data using the top 50 components
# Plot the result
###################################################################

X_noisy_mean = ...
projX_noisy = ...
X_reconstructed = ...
# plot_MNIST_reconstruction(X_noisy, X_reconstructed)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial3_Solution_e3ee8262.py)

*Example output:*


# Facility Location

## Objective and Prerequisites

Facility location problems can be commonly found in many industries, including logistics and telecommunications. In this example, we’ll show you how to tackle a facility location problem that involves determining the number and location of warehouses that are needed to supply a group of supermarkets. We’ll demonstrate how to construct a mixed-integer programming (MIP) model of this problem, implement this model in the Gurobi Python API, and then use the Gurobi Optimizer to find an optimal solution.

This modeling example is at the beginner level, where we assume that you know Python and that you have some knowledge about building mathematical optimization models.

**Download the Repository**

You can download the repository containing this and other examples by clicking [here](https://github.com/Gurobi/modeling-examples/archive/master.zip).

**Gurobi License**

In order to run this Jupyter Notebook properly, you must have a Gurobi license. If you do not have one, you can request an [evaluation license](https://www.gurobi.com/downloads/request-an-evaluation-license/?utm_source=3PW&utm_medium=OT&utm_campaign=WW-MU-LOG-OR-O_LEA-PR_NO-Q3_FY20_WW_JPME_facility-location_COM_EVAL_GITHUB_&utm_term=facility-location-problem&utm_content=C_JPM) as a *commercial user*, or download a [free license](https://www.gurobi.com/academia/academic-program-and-licenses/?utm_source=3PW&utm_medium=OT&utm_campaign=WW-MU-LOG-OR-O_LEA-PR_NO-Q3_FY20_WW_JPME_facility-location_ACADEMIC_EVAL_GITHUB_&utm_term=facility-location-problem&utm_content=C_JPM) as an *academic user*.

## Motivation

The study of facility location problems - also known as "location analysis" [1] - is a branch of operations research and computational geometry concerned with the optimal placement of facilities to minimize transportation costs while considering factors like safety (e.g. by avoiding placing hazardous materials near housing) and the location of  competitors' facilities.

The Fermat-Weber problem, formulated in the 17'th century, was one of the first facility location problems ever devised.
The Fermat-Weber problem can be described as follows: Given three points in a plane, find a fourth point such that the sum of its distances to the three given points is minimal. This problem can be viewed as a variation of the facility location problem, where the assumption is made that the transportation costs per distance are the same for all destinations.

Facility location problems have applications in a wide variety of industries. For supply chain management and logistics, this problem  can be used to find the optimal location for stores, factories, warehouses, etc. Other applications range from public policy (e.g. positioning  police officers in a city), telecommunications (e.g. cell towers in a network), and even particle physics (e.g. separation distance between repulsive charges). Another application of the facility location problem is to determine the locations for natural gas transmission equipment. Finally, facility location problems can be applied to cluster analysis.

## Problem Description

A large supermarket chain in the UK needs to build warehouses for a set of supermarkets it is opening in Northern England. The locations of the supermarkets have been identified, but the locations of the warehouses have yet to be determined.

Several good candidate locations for the warehouses have been identified, but decisions must be made regarding
how many warehouses to open and at which candidate locations to build them.

Opening many warehouses would be advantageous as this would reduce the average distance a truck has to drive from the warehouse to the supermarket, and hence reduce the delivery cost. However, opening a warehouse has a fixed cost associated with it.

In this example, our goal is to find the optimal tradeoff between delivery costs and the costs of building new facilities.

## Solution Approach

Mathematical programming is a declarative approach where the modeler formulates a mathematical optimization model that captures the key aspects of a complex business problem. The Gurobi Optimizer solves such models using state-of-the-art mathematics and computer science.

A mathematical optimization model has five components, namely:

* Sets and indices.
* Parameters.
* Decision variables.
* Objective function(s).
* Constraints.

We now present a MIP formulation for the facility location problem.

## Model Formulation

### Sets and Indices

$i \in I$: Index and set of supermarket (or customer) locations.

$j \in J$: Index and set of candidate warehouse (or facility) locations.

### Parameters

$f_{j} \in \mathbb{R}^+$: Fixed cost associated with constructing facility $j \in J$.

$d_{i,j} \in \mathbb{R}^+$: Distance between facility $j \in J$ and customer $i \in I$.

$c_{i,j} \in \mathbb{R}^+$: Cost of shipping between candidate facility site $j \in J$ and customer location $i \in I$. We assume that this cost is proportional to the distance between the facility and the customer. That is, $c_{i,j} = \alpha \cdot d_{i,j}$, where $\alpha$ is the cost per mile of driving, adjusted to incorporate the average number of trips a delivery truck would be expected to make over a five year period.

### Decision Variables

$select_{j} \in \{0, 1 \}$: This variable is equal to 1 if we build a facility at candidate location $j \in J$; and 0 otherwise.

$0 \leq assign_{i,j} \leq 1$: This non-negative continuous variable determines the fraction of supply received by customer $i \in I$ from facility $j \in J$.

### Objective Function

- **Total costs**. We want to minimize the total cost to open and operate the facilities. This is the sum of the cost of opening facilities and the cost related to shipping between facilities and customers. This total cost measures the tradeoff between the cost of building a new facility and the total delivery cost over a five year period.

\begin{equation}
\text{Min} \quad Z = \sum_{j \in J} f_{j} \cdot select_{j} + \sum_{j \in J} \sum_{i \in I} c_{i,j} \cdot assign_{i,j}
\tag{0}
\end{equation}

### Constraints

- **Demand**. For each customer  $i \in I$ ensure that its demand is fulfilled. That is, the sum of the fraction received from each facility for each customer must be equal to 1:

\begin{equation}
\sum_{j \in J} assign_{i,j} = 1 \quad \forall i \in I
\tag{1}
\end{equation}

- **Shipping**. We need to ensure that we  only ship from facility $j \in J$,  if that facility has actually been built.

\begin{equation}
assign_{i,j} \leq select_{j} \quad \forall i \in I \quad \forall j \in J
\tag{2}
\end{equation}

## Python Implementation

This example considers two supermarkets and nine warehouse candidates. The coordinates of each supermarket are provided in the following table.

|  | Coordinates |
| --- | --- |
| Supermarket 1 | (0,1.5) |
| Supermarket 2 | (2.5,1.2) |

The following table shows the coordinates of the candidate warehouse sites and the fixed cost of building the warehouse in millions of GBP.

|  | coordinates | fixed cost |
| --- | --- |  --- |
| Warehouse 1 | (0,0) | 3 |
| Warehouse 2 | (0,1) | 2 |
| Warehouse 3 | (0,2) | 3 |
| Warehouse 4 | (1,0) | 1 |
| Warehouse 5 | (1,1) | 3 |
| Warehouse 6 | (1,2) | 3 |
| Warehouse 7 | (2,0) | 4 |
| Warehouse 8 | (2,1) | 3 |
| Warehouse 9 | (2,2) | 2 |

The cost per mile is one million GBP.

## Python Implementation

We now import the Gurobi Python Module and other Python libraries. Then, we initialize the data structures with the given data.

```python
from itertools import product
from math import sqrt

import gurobipy as gp
from gurobipy import GRB

# tested with Gurobi v9.1.0 and Python 3.7.0

# Parameters
customers = [(0,1.5), (2.5,1.2)]
facilities = [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2), (2,0), (2,1), (2,2)]
setup_cost = [3,2,3,1,3,3,4,3,2]
cost_per_mile = 1
```

### Preprocessing

We define a function that determines the Euclidean distance between each facility and customer sites. In addition, we compute key parameters required by the MIP model formulation of the facility location problem.

```python
# This function determines the Euclidean distance between a facility and customer sites.

def compute_distance(loc1, loc2):
    dx = loc1[0] - loc2[0]
    dy = loc1[1] - loc2[1]
    return sqrt(dx*dx + dy*dy)

# Compute key parameters of MIP model formulation

num_facilities = len(facilities)
num_customers = len(customers)
cartesian_prod = list(product(range(num_customers), range(num_facilities)))

# Compute shipping costs

shipping_cost = {(c,f): cost_per_mile*compute_distance(customers[c], facilities[f]) for c, f in cartesian_prod}
```

### Model Deployment

We now determine the MIP model for the facility location problem, by defining the decision variables, constraints, and objective function. Next, we start the optimization process and Gurobi finds the plan to build facilities that minimizes total costs.

```python
# MIP  model formulation

m = gp.Model('facility_location')

select = m.addVars(num_facilities, vtype=GRB.BINARY, name='Select')
assign = m.addVars(cartesian_prod, ub=1, vtype=GRB.CONTINUOUS, name='Assign')

m.addConstrs((assign[(c,f)] <= select[f] for c,f in cartesian_prod), name='Setup2ship')
m.addConstrs((gp.quicksum(assign[(c,f)] for f in range(num_facilities)) == 1 for c in range(num_customers)), name='Demand')

m.setObjective(select.prod(setup_cost)+assign.prod(shipping_cost), GRB.MINIMIZE)

m.optimize()
```

    Using license file c:\gurobi\gurobi.lic
    Gurobi Optimizer version 9.1.0 build v9.1.0rc0 (win64)
    Thread count: 4 physical cores, 8 logical processors, using up to 8 threads
    Optimize a model with 20 rows, 27 columns and 54 nonzeros
    Model fingerprint: 0x0939f503
    Variable types: 18 continuous, 9 integer (9 binary)
    Coefficient statistics:
      Matrix range     [1e+00, 1e+00]
      Objective range  [5e-01, 4e+00]
      Bounds range     [1e+00, 1e+00]
      RHS range        [1e+00, 1e+00]
    Presolve time: 0.00s
    Presolved: 20 rows, 27 columns, 54 nonzeros
    Variable types: 18 continuous, 9 integer (9 binary)

    Root relaxation: objective 4.723713e+00, 15 iterations, 0.00 seconds

        Nodes    |    Current Node    |     Objective Bounds      |     Work
     Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time

    *    0     0               0       4.7237129    4.72371  0.00%     -    0s

    Explored 0 nodes (15 simplex iterations) in 0.02 seconds
    Thread count was 8 (of 8 available processors)

    Solution count 1: 4.72371

    Optimal solution found (tolerance 1.00e-04)
    Best objective 4.723712908962e+00, best bound 4.723712908962e+00, gap 0.0000%

## Analysis

The result of the optimization model shows that the minimum total cost value is 4.72 million GBP. Let's see the solution that achieves that optimal result.

### Warehouse Build Plan

This plan determines at which site locations to build a warehouse.

```python
# display optimal values of decision variables

for facility in select.keys():
    if (abs(select[facility].x) > 1e-6):
        print(f"\n Build a warehouse at location {facility + 1}.")
```

     Build a warehouse at location 4.

### Shipment Plan

This plan determines the percentage of shipments to be sent from each facility built to each customer.

```python
# Shipments from facilities to customers.

for customer, facility in assign.keys():
    if (abs(assign[customer, facility].x) > 1e-6):
        print(f"\n Supermarket {customer + 1} receives {round(100*assign[customer, facility].x, 2)} % of its demand  from Warehouse {facility + 1} .")

```

     Supermarket 1 receives 100.0 % of its demand  from Warehouse 4 .

     Supermarket 2 receives 100.0 % of its demand  from Warehouse 4 .

##  Conclusion
In this example, we addressed a facility location problem where we want to build warehouses to supply a large number of supermarkets while minimizing the fixed total costs of building warehouses and the total variable shipping costs from warehouses to supermarkets. We learned how to formulate the problem as a MIP model. Also, we learned how to implement the MIP model formulation and solve it using the Gurobi Python API.

##  References
[1] Laporte, Gilbert, Stefan Nickel, and Saldanha da Gama, Francisco. Location Science. Springer, 2015.

Copyright © 2020 Gurobi Optimization, LLC


## Subsurface Data Analytics

### Bootstrap for Subsurface Data Analytics in Python

#### Michael Pyrcz, Associate Professor, University of Texas at Austin

##### [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)

### Exercise: Bootstrap for Subsurface Data Analytics in Python

Here's a simple workflow, demonstration of bootstrap for subsurface modeling workflows. This should help you get started with building subsurface models that integrate uncertainty in the sample statistics.

#### Bootstrap

Uncertainty in the sample statistics
* one source of uncertainty is the paucity of data.
* do 200 or even less wells provide a precise (and accurate estimate) of the mean? standard deviation? skew? P13?

Would it be useful to know the uncertainty in these statistics due to limited sampling?
* what is the impact of uncertainty in the mean porosity e.g. 20%+/-2%?

**Bootstrap** is a method to assess the uncertainty in a sample statistic by repeated random sampling with replacement.

Assumptions
* sufficient, representative sampling, identical, idependent samples

Limitations
1. assumes the samples are representative
2. assumes stationarity
3. only accounts for uncertainty due to too few samples, e.g. no uncertainty due to changes away from data
4. does not account for boundary of area of interest
5. assumes the samples are independent
6. does not account for other local information sources

The **Bootstrap Approach** (Efron, 1982)

Statistical resampling procedure to calculate uncertainty in a calculated statistic from the data itself.
* Does this work?  Prove it to yourself, for uncertainty in the mean solution is standard error:

\begin{equation}
\sigma^2_\overline{x} = \frac{\sigma^2_s}{n}
\end{equation}

Extremely powerful - could calculate uncertainty in any statistic!  e.g. P13, skew etc.
* Would not be possible access general uncertainty in any statistic without bootstrap.
* Advanced forms account for spatial information and sampling strategy (game theory and Journel’s spatial bootstrap (1993).

Steps:

1. assemble a sample set, must be representative, reasonable to assume independence between samples

2. optional: build a cumulative distribution function (CDF)
    * may account for declustering weights, tail extrapolation
    * could use analogous data to support

3. For $\ell = 1, \ldots, L$ realizations, do the following:

    * For $i = \alpha, \ldots, n$ data, do the following:

        * Draw a random sample with replacement from the sample set or Monte Carlo simulate from the CDF (if available).

6. Calculate a realization of the sammary statistic of interest from the $n$ samples, e.g. $m^\ell$, $\sigma^2_{\ell}$. Return to 3 for another realization.

7. Compile and summarize the $L$ realizations of the statistic of interest.

This is a very powerful method.  Let's try it out.

#### Objective

In the PGE 383: Stochastic Subsurface Modeling class I want to provide hands-on experience with building subsurface modeling workflows. Python provides an excellent vehicle to accomplish this. I have coded a package called GeostatsPy with GSLIB: Geostatistical Library (Deutsch and Journel, 1998) functionality that provides basic building blocks for building subsurface modeling workflows.

The objective is to remove the hurdles of subsurface modeling workflow construction by providing building blocks and sufficient examples. This is not a coding class per se, but we need the ability to 'script' workflows working with numerical methods.

#### Getting Started

Here's the steps to get setup in Python with the GeostatsPy package:

1. Install Anaconda 3 on your machine (https://www.anaconda.com/download/).
2. From Anaconda Navigator (within Anaconda3 group), go to the environment tab, click on base (root) green arrow and open a terminal.
3. In the terminal type: pip install geostatspy.
4. Open Jupyter and in the top block get started by copy and pasting the code block below from this Jupyter Notebook to start using the geostatspy functionality.

You will need to copy the data file to your working directory.  They are available here:

* Tabular data - sample_data_biased.csv at https://git.io/fh0CW

There are exampled below with these functions. You can go here to see a list of the available functions, https://git.io/fh4eX, other example workflows and source code.

```python
import geostatspy.GSLIB as GSLIB          # GSLIB utilies, visualization and wrapper
import geostatspy.geostats as geostats    # GSLIB methods convert to Python
```

We will also need some standard packages. These should have been installed with Anaconda 3.

```python
import numpy as np                        # ndarrys for gridded data
import pandas as pd                       # DataFrames for tabular data
import os                                 # set working directory, run executables
import matplotlib.pyplot as plt           # for plotting
import seaborn as sns                     # advanced statistical methods and plots
from scipy import stats                   # summary statistics
import math                               # trig etc.
import scipy.signal as signal             # kernel for moving window calculation
import random
```

#### Set the working directory

I always like to do this so I don't lose files and to simplify subsequent read and writes (avoid including the full address each time).

```python
os.chdir("c:/PGE383")                     # set the working directory
```

#### Loading Tabular Data

Here's the command to load our comma delimited data file in to a Pandas' DataFrame object.

```python
df = pd.read_csv('sample_data_biased.csv') # load our data table (wrong name!)
```

Let's drop some samples so that we increase the variations in bootstrap samples for our demonstration below.  We will resample 25% of the original data file.  I use a random number seed so that we all get the same sample to work with.  To get something different just drop the 'rand_state' parameter or change the value from '73073' to something else.

```python
df = df.sample(frac = 0.25, random_state = 73073) # extract 25% random samples to reduce the size of the dataset
print('Using ' + str(len(df)) + ' number of samples')
```

    Using 72 number of samples

Visualizing the DataFrame would be useful and we already learned about these methods in this demo (https://git.io/fNgRW).

We can preview the DataFrame by printing a slice or by utilizing the 'head' DataFrame member function (with a nice and clean format, see below). With the slice we could look at any subset of the data table and with the head command, add parameter 'n=13' to see the first 13 rows of the dataset.

```python
print(df.iloc[0:5,:])                   # display first 4 samples in the table as a preview
df.head(n=13)                           # we could also use this command for a table preview
```

           X    Y  Facies  Porosity         Perm
    150   20   69       0  0.104484     3.003225
    17   400  600       1  0.163054   317.883581
    68   640  529       1  0.120547    10.801778
    234  490  289       1  0.144525    16.251107
    214  470  899       1  0.211944  1302.937858

      X
      Y
      Facies
      Porosity
      Perm

      150
      20
      69
      0
      0.104484
      3.003225

      17
      400
      600
      1
      0.163054
      317.883581

      68
      640
      529
      1
      0.120547
      10.801778

      234
      490
      289
      1
      0.144525
      16.251107

      214
      470
      899
      1
      0.211944
      1302.937858

      153
      360
      279
      1
      0.134607
      9.776276

      5
      200
      800
      1
      0.154648
      106.491795

      170
      670
      889
      1
      0.119601
      17.052309

      252
      200
      359
      0
      0.077625
      0.540486

      131
      320
      99
      1
      0.141749
      13.154686

      133
      250
      569
      1
      0.120868
      22.277121

      245
      890
      489
      0
      0.103176
      2.846036

      159
      590
      69
      1
      0.114117
      13.251200

#### Summary Statistics for Tabular Data

The table includes X and Y coordinates (meters), Facies 1 and 0 (1 is sandstone and 0 interbedded sand and mudstone), Porosity (fraction), and permeability as Perm (mDarcy).

There are a lot of efficient methods to calculate summary statistics from tabular data in DataFrames. The describe command provides count, mean, minimum, maximum, and quartiles all in a nice data table. We use transpose just to flip the table so that features are on the rows and the statistics are on the columns.

```python
df.describe().transpose()
```

      count
      mean
      std
      min
      25%
      50%
      75%
      max

      X
      72.0
      448.611111
      239.825674
      0.000000
      295.000000
      415.000000
      640.000000
      990.000000

      Y
      72.0
      495.111111
      293.106026
      39.000000
      279.000000
      494.000000
      716.750000
      999.000000

      Facies
      72.0
      0.833333
      0.375293
      0.000000
      1.000000
      1.000000
      1.000000
      1.000000

      Porosity
      72.0
      0.135360
      0.039304
      0.074349
      0.106374
      0.124065
      0.147035
      0.223661

      Perm
      72.0
      242.573614
      553.551806
      0.093252
      2.979039
      13.509371
      51.052217
      2372.383732

#### Visualizing Tabular Data with Location Maps

It is natural to set the x and y coordinate and feature ranges manually. e.g. do you want your color bar to go from 0.05887 to 0.24230 exactly? Also, let's pick a color map for display. I heard that plasma is known to be friendly to the color blind as the color and intensity vary together (hope I got that right, it was an interesting Twitter conversation started by Matt Hall from Agile if I recall correctly). We will assume a study area of 0 to 1,000m in x and y and omit any data outside this area.

```python
xmin = 0.0; xmax = 1000.0               # range of x values
ymin = 0.0; ymax = 1000.0               # range of y values
pormin = 0.05; pormax = 0.25;           # range of porosity values
nx = 100; ny = 100; csize = 10.0
cmap = plt.cm.plasma                    # color map
```

Let's try out locmap. This is a reimplementation of GSLIB's locmap program that uses matplotlib. I hope you find it simpler than matplotlib, if you want to get more advanced and build custom plots lock at the source. If you improve it, send me the new code. Any help is appreciated. To see the parameters, just type the command name:

```python
GSLIB.locmap
```

    <function geostatspy.GSLIB.locmap(df, xcol, ycol, vcol, xmin, xmax, ymin, ymax, vmin, vmax, title, xlabel, ylabel, vlabel, cmap, fig_name)>

Now we can populate the plotting parameters and visualize the porosity data.

```python
plt.subplot(111)
GSLIB.locmap_st(df,'X','Y','Porosity',xmin,xmax,ymin,ymax,pormin,pormax,'Well Data - Porosity','X(m)','Y(m)','Porosity (fraction)',cmap)
plt.subplots_adjust(left=0.0, bottom=0.0, right=1.0, top=1.2, wspace=0.2, hspace=0.2)
plt.show()
```

#### Declustering

Let's calculate some declustering weights. There is a demonstration on declustering here https://git.io/fhgJl if you need more information.

```python
wts, cell_sizes, dmeans = geostats.declus(df,'X','Y','Porosity',iminmax = 1, noff= 10, ncell=100,cmin=10,cmax=2000)
df['Wts'] = wts                           # add weights to the sample data DataFrame
df.head()                                 # preview to check the sample data DataFrame

def weighted_avg_and_std(values, weights): # function to calculate weighted mean and st. dev., from Eric O Lebigot, stack overflow,
    average = np.average(values, weights=weights)
    variance = np.average((values-average)**2, weights=weights)
    return (average, math.sqrt(variance))

sample_avg, sample_stdev = weighted_avg_and_std(df['Porosity'],df['Wts'])
print('Declustered mean = ' + str(round(sample_avg,3)) + ' and declustered standard deviation = ' + str(round(sample_stdev,3)))
```

    There are 72 data with:
       mean of      0.13535995545833332
       min and max  0.074348923 and 0.22366070899999999
       standard dev 0.03902973923480811
    Declustered mean = 0.126 and declustered standard deviation = 0.032

##### A Couple of Bootstrap Realizations

We will attempt boostrap by-hand and manually loop over $L$ realizations and draw $n$ samples to calculate the summary statistics of interest, mean and variance. The choice function from the random package simplifies sampling with replacement from a set of samples with weights.

This command returns a ndarray with k samples with replacment from the 'Porosity' column of our DataFrame (df) accounting for the data weights in column 'Wts'.
```p
samples1 = random.choices(df['Porosity'].values, weights=df['Wts'].values, cum_weights=None, k=len(df))
```

It is instructive to look at a few of these realizations from the original declustered data set.

To understand this code below there are just a couple of Python concepts that you need to add to your Python arsenal.

1. declaring arrays - NumPy has a lot of great array (ndarray) functionality. There are build in functions to make a ndarray of any length (and dimension). This includes 'zeros', 'ones' and 'rand', so when we use this code:

 `samples = np.zeros((nreal,len(df))`

 we're making a 2D array of size number of realizations, nreal, x number of data in the DataFrame, df, pre-populated with zeros.  Note the len(df) command returns the number of samples in the DataFrame.

2. For Loops - when we are using the command below, we are instructing the computer to loop over all the indented code below the command for $ireal = 0,1,2,\ldots,nreal-1$ times. For each loop the $ireal$ variable increments, so we can use this to save each result to a different index in the arrays mean and stdev. Note, Python arrays index starting at 0 and stop at the length - 1.

 `for ireal in range(0,nreal):`

 we are running each bootstrap resampled realization, and storing each realization in a row of the 2D array (for a 2D array the first index is the row index and the second index is the column index).

3. Subplots - we can put multple plots together with subplots.  To do this we specify

 `plt.subplot(3,3,1)`

 before each plot.  Where the first value is the number of row, the second number is the number of columns and the third number is the image index from $0,...,nrow x ncol$.  The images sort from left to right and then from top to bottom.

```python
nreal = 200                               # number of bootstrap realizations
nhist = 8                                 # number of bootstrap histograms to view (max 8)
samples = np.zeros((nreal,len(df))); real_label = []

for ireal in range(0,nreal):
    samples[ireal] = random.choices(df['Porosity'].values, weights=df['Wts'].values, cum_weights=None, k=len(df))

plt.subplot(3,3,1)
GSLIB.hist_st(df['Porosity'],pormin,pormax,False,False,20,df['Wts'],'Porosity (fraction)','Declustered Porosity')

for ireal in range(0,nhist):
    plt.subplot(3,3,ireal+2)
    GSLIB.hist_st(samples[ireal],pormin,pormax,False,False,20,None,'Porosity (fraction)','Bootstrap Porosity Realization ' + str(ireal+1))

plt.subplots_adjust(left=0.0, bottom=0.0, right=3.0, top=3.2, wspace=0.2, hspace=0.2)
plt.show()
```

#### Distribution Shape

The distribution seems to be bimodel (two peaks).  Instead of looking at a few histograms of bootstrap realizations, let's look at many realizations and see if there is always, mostly or just sometimes two modes.

We will just look over many

```python
for ireal in range(0,nreal):              # loop over the realizations and plot with seaborn package
    # Draw the density plot
    sns.distplot(samples[ireal], color = 'black', hist = False, kde = True,kde_kws = {"alpha": 0.02, 'linewidth': 5})
plt.title('Bootstrap Porosity Distribution - Uncertainty in the Distribution Shape')
plt.xlabel('Porosity (fraction)')
plt.ylabel('Density')

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.2, wspace=0.2, hspace=0.2)
plt.show()
```

Note that the bootstrap distributions vary quite a bit from the original, but the bimodal shape is consistent; therefore the bimodal distribution form is **not due to random effect / sparse sampling**! We are likely dealing with two distinct processes and / or spatial trends.

#### Summarizations Over Bootstrap Realizations

Let's make a loop to conduct $L$ resamples and calculate the average and standard deviation for each  ($m^\ell$, $\sigma^2_{\ell}$,  for $\ell = 0,\dots,L-1$). We then summarization over these $L$ realizations.

I did not find any built-in, concise functions to accomplish this, i.e. with a single line of code, so we are going to do it by hand.

To understand this code there are just a couple of Python concepts that you need to add to your Python arsenal.

1. declaring arrays - NumPy has a lot of great array (ndarray) functionality. There are build in functions to make a ndarray of any length (and dimension). This includes 'zeros', 'ones' and 'rand', so when we use this code:

 `mean = np.zeros(L); stdev = np.zeros(L)`

 we're making arrays of length $L$ pre-populated with zeros.

2. For Loops - when we are using the command below, we are instructing the computer to loop over all the indented code below the command for $l = 0,1,2,\ldots,L-1$ times. For each loop the $l$ variable increments, so we can use this to save each result to a different index in the arrays mean and stdev. Note, Python arrays index starting at 0 and stop at the length - 1.

 `for l in range(0, L):`

 we are running each bootstrap resampled realization, calculating the average and standard deviation and storing them in the arrays that we already declared.

```python
L = 1000                                   # set the number of realizations for uncertainty calculation
mean = np.zeros(L); stdev = np.zeros(L)    # declare arrays to hold the realizations of the statistics
for l in range(0, L):                      # loop over realizations
    samples = random.choices(df['Porosity'].values, weights=df['Wts'].values, cum_weights=None, k=len(df))
    mean[l] = np.average(samples)
    stdev[l] = np.std(samples)

plt.subplot(121)
GSLIB.hist_st(mean,0.11,0.15,False,False,50,None,'Average Porosity (fraction)','Bootstrap Uncertainty in Porosity Average')

plt.subplot(122)
GSLIB.hist_st(stdev,0.015,0.045,False,False,50,None,'Standard Deviation Porosity (fraction)','Bootstrap Uncertainty in Porosity Standard Deviation')

plt.subplots_adjust(left=0.0, bottom=0.0, right=2.0, top=1.2, wspace=0.2, hspace=0.2)
plt.show()

print('Summary Statistics for Bootstrap Porosity Average:')
print(stats.describe(mean))
print('\nUncertainty in Porosity Average: P10 ' + str(round(np.percentile(mean,10)*100,1)) + '%, P50 ' + str(round(np.percentile(mean,50)*100,1)) + '%, P90 ' + str(round(np.percentile(mean,90)*100,1)) +"%")

print('\nSummary Statistics for Bootstrap Porosity Standard Deviation:')
print(stats.describe(stdev));
print('\nUncertainty in Porosity Standard Deviation: P10 ' + str(round(np.percentile(stdev,10),3)*100) + '%, P50 ' + str(round(np.percentile(stdev,50),3)*100) + '%, P90 ' + str(round(np.percentile(stdev,90)*100,1)) + "%")

```

#### Comments

This was a basic demonstration of bootstrap. Much more could be done, you could replace the statistics, average and standard deviation with any other statistics, for example P90, kurtosis, P13 etc. I have other demonstrations on the basics of working with DataFrames, ndarrays, univariate statistics, plotting data, declustering, data transformations, trend modeling and many other workflows available at https://github.com/GeostatsGuy/PythonNumericalDemos and https://github.com/GeostatsGuy/GeostatsPy.

I hope this was helpful,

*Michael*

Michael Pyrcz, Ph.D., P.Eng. Associate Professor The Hildebrand Department of Petroleum and Geosystems Engineering, Bureau of Economic Geology, The Jackson School of Geosciences, The University of Texas at Austin

#### More Resources Available at: [Twitter](https://twitter.com/geostatsguy) | [GitHub](https://github.com/GeostatsGuy) | [Website](http://michaelpyrcz.com) | [GoogleScholar](https://scholar.google.com/citations?user=QVZ20eQAAAAJ&hl=en&oi=ao) | [Book](https://www.amazon.com/Geostatistical-Reservoir-Modeling-Michael-Pyrcz/dp/0199731446) | [YouTube](https://www.youtube.com/channel/UCLqEr-xV-ceHdXXXrTId5ig)  | [LinkedIn](https://www.linkedin.com/in/michael-pyrcz-61a648a1)