```python
# Mount Google Drive
from google.colab import drive # import drive from google colab

ROOT = "/content/drive"     # default location for the drive
print(ROOT)                 # print content of ROOT (Optional)

drive.mount(ROOT,force_remount=True)
```

    /content/drive
    Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

    Enter your authorization code:
    ··········
    Mounted at /content/drive

# Neuromatch Academy: Week 1, Day 5, Tutorial 2
# Dimensionality Reduction: Principal component analysis

__Content creators:__ Alex Cayco Gajic, John Murray

__Content reviewers:__ Roozbeh Farhoudi, Matt Krause, Spiros Chavlis, Richard Gao, Michael Waskom

---
# Tutorial Objectives

In this notebook we'll learn how to perform PCA by projecting the data onto the eigenvectors of its covariance matrix.

Overview:
- Calculate the eigenvectors of the sample covariance matrix.
- Perform PCA by projecting data onto the eigenvectors of the covariance matrix.
- Plot and explore the eigenvalues.

To quickly refresh your knowledge of eigenvalues and eigenvectors, you can watch this [short video](https://www.youtube.com/watch?v=kwA3qM0rm7c) (4 minutes) for a geometrical explanation. For a deeper understanding, this [in-depth video](https://www.youtube.com/watch?v=PFDu9oVAE-g&list=PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab&index=14) (17 minutes) provides an excellent basis and is beautifully illustrated.

```python
# @title Video 1: PCA
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="-f6T9--oM0E", width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=-f6T9--oM0E

---
# Setup
Run these cells to get the tutorial started.

```python
# Imports
import numpy as np
import matplotlib.pyplot as plt
```

```python
# @title Figure Settings
import ipywidgets as widgets  # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
# @title Helper functions

def plot_eigenvalues(evals):
  """
  Plots eigenvalues.

  Args:
      (numpy array of floats) : Vector of eigenvalues

  Returns:
    Nothing.

  """
  plt.figure(figsize=(4, 4))
  plt.plot(np.arange(1, len(evals) + 1), evals, 'o-k')
  plt.xlabel('Component')
  plt.ylabel('Eigenvalue')
  plt.title('Scree plot')
  plt.xticks(np.arange(1, len(evals) + 1))
  plt.ylim([0, 2.5])

def sort_evals_descending(evals, evectors):
  """
  Sorts eigenvalues and eigenvectors in decreasing order. Also aligns first two
  eigenvectors to be in first two quadrants (if 2D).

  Args:
    evals (numpy array of floats)    : Vector of eigenvalues
    evectors (numpy array of floats) : Corresponding matrix of eigenvectors
                                        each column corresponds to a different
                                        eigenvalue

  Returns:
    (numpy array of floats)          : Vector of eigenvalues after sorting
    (numpy array of floats)          : Matrix of eigenvectors after sorting
  """

  index = np.flip(np.argsort(evals))
  evals = evals[index]
  evectors = evectors[:, index]
  if evals.shape[0] == 2:
    if np.arccos(np.matmul(evectors[:, 0],
                           1 / np.sqrt(2) * np.array([1, 1]))) > np.pi / 2:
      evectors[:, 0] = -evectors[:, 0]
    if np.arccos(np.matmul(evectors[:, 1],
                           1 / np.sqrt(2) * np.array([-1, 1]))) > np.pi / 2:
      evectors[:, 1] = -evectors[:, 1]
  return evals, evectors

def plot_data(X):
  """
  Plots bivariate data. Includes a plot of each random variable, and a scatter
  scatter plot of their joint activity. The title indicates the sample
  correlation calculated from the data.

  Args:
    X (numpy array of floats) : Data matrix each column corresponds to a
                                different random variable

  Returns:
    Nothing.
  """

  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(X[:, 0], color='k')
  plt.ylabel('Neuron 1')
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(X[:, 1], color='k')
  plt.xlabel('Sample Number (sorted)')
  plt.ylabel('Neuron 2')
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(X[:, 0], X[:, 1], '.', markerfacecolor=[.5, .5, .5],
           markeredgewidth=0)
  ax3.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(X[:, 0], X[:, 1])[0, 1]))
  plt.show()

def get_data(cov_matrix):
  """
  Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian

  Note that samples are sorted in ascending order for the first random
  variable.

  Args:
    var_1 (scalar)                     : variance of the first random variable
    var_2 (scalar)                     : variance of the second random variable
    cov_matrix (numpy array of floats) : desired covariance matrix

  Returns:
    (numpy array of floats)            : samples from the bivariate Gaussian,
                                          with each column corresponding to a
                                          different random variable
  """

  mean = np.array([0, 0])
  X = np.random.multivariate_normal(mean, cov_matrix, size=1000)
  indices_for_sorting = np.argsort(X[:, 0])
  X = X[indices_for_sorting, :]
  return X

def calculate_cov_matrix(var_1, var_2, corr_coef):
  """
  Calculates the covariance matrix based on the variances and
  correlation coefficient.

  Args:
    var_1 (scalar)         :  variance of the first random variable
    var_2 (scalar)         :  variance of the second random variable
    corr_coef (scalar)     :  correlation coefficient

  Returns:
    (numpy array of floats) : covariance matrix
  """
  cov = corr_coef * np.sqrt(var_1 * var_2)
  cov_matrix = np.array([[var_1, cov], [cov, var_2]])
  return cov_matrix

def define_orthonormal_basis(u):
  """
  Calculates an orthonormal basis given an arbitrary vector u.

  Args:
    u (numpy array of floats) : arbitrary 2D vector used for new basis

  Returns:
    (numpy array of floats)   : new orthonormal basis columns correspond to
                                basis vectors
  """

  u = u / np.sqrt(u[0] ** 2 + u[1] ** 2)
  w = np.array([-u[1], u[0]])
  W = np.column_stack((u, w))
  return W

def plot_data_new_basis(Y):
  """
  Plots bivariate data after transformation to new bases. Similar to plot_data
  but with colors corresponding to projections onto basis 1 (red) and
  basis 2 (blue).
  The title indicates the sample correlation calculated from the data.

  Note that samples are re-sorted in ascending order for the first random
  variable.

  Args:
    Y (numpy array of floats) : Data matrix in new basis each column
                                corresponds to a different random variable

  Returns:
    Nothing.
  """

  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(Y[:, 0], 'r')
  plt.ylabel('Projection \n basis vector 1')
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(Y[:, 1], 'b')
  plt.xlabel('Projection \n basis vector 1')
  plt.ylabel('Projection \n basis vector 2')
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(Y[:, 0], Y[:, 1], '.', color=[.5, .5, .5])
  ax3.axis('equal')
  plt.xlabel('Projection basis vector 1')
  plt.ylabel('Projection basis vector 2')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(Y[:, 0], Y[:, 1])[0, 1]))
  plt.show()

def change_of_basis(X, W):
  """
  Projects data onto a new basis.

  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors

  Returns:
    (numpy array of floats)   : Data matrix expressed in new basis
  """

  Y = np.matmul(X, W)
  return Y

def plot_basis_vectors(X, W):
  """
  Plots bivariate data as well as new basis vectors.

  Args:
    X (numpy array of floats) : Data matrix each column corresponds to a
                                different random variable
    W (numpy array of floats) : Square matrix representing new orthonormal
                                basis each column represents a basis vector

  Returns:
    Nothing.
  """

  plt.figure(figsize=[4, 4])
  plt.plot(X[:, 0], X[:, 1], '.', color=[.5, .5, .5], label='Data')
  plt.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
           label='Basis vector 1')
  plt.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
           label='Basis vector 2')
  plt.legend()
  plt.show()

```

---
# Section 1: Calculate the eigenvectors of the the sample covariance matrix

As we saw in the lecture, PCA represents data in a new orthonormal basis defined by the eigenvectors of the covariance matrix. Remember that in the previous tutorial, we generated bivariate normal data with a specified covariance matrix $\bf \Sigma$, whose $(i,j)$th element is:
\begin{equation}
\Sigma_{ij} = E[ x_i x_j ] - E[ x_i] E[ x_j ] .
\end{equation}
However, in real life we don't have access to this ground-truth covariance matrix. To get around this, we can use the sample covariance matrix, $\bf\hat\Sigma$, which is calculated directly from the data. The $(i,j)$th element of the sample covariance matrix is:
\begin{equation}
 \hat \Sigma_{ij} =  \frac{1}{N_\text{samples}}{\bf x}_i^T {\bf x}_j - \bar {\bf x}_i \bar{\bf x}_j ,
\end{equation}
where ${\bf x}_i = [ x_i(1), x_i(2), \dots,x_i(N_\text{samples})]^T$ is a column vector representing all measurements of neuron $i$, and  $\bar {\bf x}_i$ is the mean of neuron $i$ across samples:
\begin{equation}
\bar {\bf x}_i = \frac{1}{N_\text{samples}} \sum_{k=1}^{N_\text{samples}} x_i(k).
\end{equation}
If we assume that the data has already been mean-subtracted, then we can write the sample covariance matrix in a much simpler matrix form:
\begin{align}
{\bf \hat \Sigma}
&= \frac{1}{N_\text{samples}} {\bf X}^T {\bf X}.
\end{align}

## Exercise 1: Calculation of the covariance matrix

Before calculating the eigenvectors, you must first calculate the sample covariance matrix.

**Steps**
* Complete the function `get_sample_cov_matrix` by first subtracting the sample mean of the data, then calculate $\bf \hat \Sigma$ using the equation above.
* Use `get_data` to generate bivariate normal data, and calculate the sample covariance matrix with your finished `get_sample_cov_matrix`. Compare this estimate to the true covariate matrix using `calculate_cov_matrix`.

```python
help(get_data)
help(calculate_cov_matrix)
```

    Help on function get_data in module __main__:

    get_data(cov_matrix)
        Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian

        Note that samples are sorted in ascending order for the first random
        variable.

        Args:
          var_1 (scalar)                     : variance of the first random variable
          var_2 (scalar)                     : variance of the second random variable
          cov_matrix (numpy array of floats) : desired covariance matrix

        Returns:
          (numpy array of floats)            : samples from the bivariate Gaussian,
                                                with each column corresponding to a
                                                different random variable

    Help on function calculate_cov_matrix in module __main__:

    calculate_cov_matrix(var_1, var_2, corr_coef)
        Calculates the covariance matrix based on the variances and
        correlation coefficient.

        Args:
          var_1 (scalar)         :  variance of the first random variable
          var_2 (scalar)         :  variance of the second random variable
          corr_coef (scalar)     :  correlation coefficient

        Returns:
          (numpy array of floats) : covariance matrix

```python
def get_sample_cov_matrix(X):
  """
  Returns the sample covariance matrix of data X

  Args:
    X (numpy array of floats) : Data matrix each column corresponds to a
                                different random variable

  Returns:
    (numpy array of floats)   : Covariance matrix
  """

  #################################################
  ## TODO for students: calculate the covariance matrix
  # Fill out function and remove
  #raise NotImplementedError("Student excercise: calculate the covariance matrix!")
  #################################################

  # Subtract the mean of X
  X = X-np.mean(X)
  # Calculate the covariance matrix (hint: use np.matmul)
  cov_matrix = np.transpose(X)@X/X.shape[0]

  return cov_matrix

##########################################################
## TODO for students: generate bivariate Gaussian data
# with variances of 1 and a correlation coefficient of 0.8
# compare the true and sample covariance matrices
##########################################################
variance_1 = 1
variance_2 = 1
corr_coef = 0.8

np.random.seed(2020)  # set random seed
# Uncomment below code to test your function
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)
print(cov_matrix)

X = get_data(cov_matrix)
sample_cov_matrix = get_sample_cov_matrix(X)
print(sample_cov_matrix)
```

    [[1.  0.8]
     [0.8 1. ]]
    [[0.99316457 0.82346444]
     [0.82346444 1.01282542]]

SAMPLE OUTPUT
```
[[1.  0.8]
 [0.8 1. ]]
[[0.99315313 0.82347589]
 [0.82347589 1.01281397]]
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial2_Solution_a98b14b7.py)

## Exercise 2: Eigenvectors of the Covariance matrix

Next you will calculate the eigenvectors of the covariance matrix. Plot them along with the data to check that they align with the geometry of the data.

**Steps:**
* Calculate the eigenvalues and eigenvectors of the sample covariance matrix. (**Hint:** use `np.linalg.eigh`, which finds the eigenvalues of a symmetric matrix).
* Use the provided code to sort the eigenvalues in descending order.
* Plot the eigenvectors on a scatter plot of the data, using the function `plot_basis_vectors`.

```python
help(sort_evals_descending)
help(plot_basis_vectors)
```

    Help on function sort_evals_descending in module __main__:

    sort_evals_descending(evals, evectors)
        Sorts eigenvalues and eigenvectors in decreasing order. Also aligns first two
        eigenvectors to be in first two quadrants (if 2D).

        Args:
          evals (numpy array of floats)    : Vector of eigenvalues
          evectors (numpy array of floats) : Corresponding matrix of eigenvectors
                                              each column corresponds to a different
                                              eigenvalue

        Returns:
          (numpy array of floats)          : Vector of eigenvalues after sorting
          (numpy array of floats)          : Matrix of eigenvectors after sorting

    Help on function plot_basis_vectors in module __main__:

    plot_basis_vectors(X, W)
        Plots bivariate data as well as new basis vectors.

        Args:
          X (numpy array of floats) : Data matrix each column corresponds to a
                                      different random variable
          W (numpy array of floats) : Square matrix representing new orthonormal
                                      basis each column represents a basis vector

        Returns:
          Nothing.

```python
#################################################
## TO DO for students: Calculate and sort the eigenvalues in descending order
#################################################
# Calculate the eigenvalues and eigenvectors
evals, evectors = np.linalg.eigh(sample_cov_matrix)
# Sort the eigenvalues in descending order
evals, evectors = sort_evals_descending(evals,evectors)

plot_basis_vectors(X, evectors)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial2_Solution_ca961dca.py)

*Example output:*

---
# Section 2: Perform PCA by projecting data onto the eigenvectors

To perform PCA, we will project the data onto the eigenvectors of the covariance matrix, i.e.:
\begin{equation}
\bf S = X W
\end{equation}
where $\bf S$ is an $N_\text{samples} \times N$ matrix representing the projected data (also called *scores*), and $W$ is an $N\times N$ orthogonal matrix, each of whose columns represents the eigenvectors of the covariance matrix (also called *weights* or *loadings*).

## Exercise 3: PCA implementation

You will now perform PCA on the data using the intuition you have developed so far. Fill in the function below to carry out the steps to perform PCA by projecting the data onto the eigenvectors of its covariance matrix.

**Steps:**
* First subtract the mean.
* Then calculate the sample covariance matrix.
* Then find the eigenvalues and eigenvectors and sort them in descending order.
* Finally project the mean-centered data onto the eigenvectors.

```python
help(change_of_basis)
help(plot_data_new_basis)
```

    Help on function change_of_basis in module __main__:

    change_of_basis(X, W)
        Projects data onto a new basis.

        Args:
          X (numpy array of floats) : Data matrix each column corresponding to a
                                      different random variable
          W (numpy array of floats) : new orthonormal basis columns correspond to
                                      basis vectors

        Returns:
          (numpy array of floats)   : Data matrix expressed in new basis

    Help on function plot_data_new_basis in module __main__:

    plot_data_new_basis(Y)
        Plots bivariate data after transformation to new bases. Similar to plot_data
        but with colors corresponding to projections onto basis 1 (red) and
        basis 2 (blue).
        The title indicates the sample correlation calculated from the data.

        Note that samples are re-sorted in ascending order for the first random
        variable.

        Args:
          Y (numpy array of floats) : Data matrix in new basis each column
                                      corresponds to a different random variable

        Returns:
          Nothing.

```python
def pca(X):
  """
  Sorts eigenvalues and eigenvectors in decreasing order.

  Args:
    X (numpy array of floats): Data matrix each column corresponds to a
                               different random variable

  Returns:
    (numpy array of floats)  : Data projected onto the new basis
    (numpy array of floats)  : Vector of eigenvalues
    (numpy array of floats)  : Corresponding matrix of eigenvectors

  """

  #################################################
  ## TODO for students: calculate the covariance matrix
  # Fill out function and remove
  #raise NotImplementedError("Student excercise: sort eigenvalues/eigenvectors!")
  #################################################

  # Subtract the mean of X
  X = X-np.mean(X)
  # Calculate the sample covariance matrix
  cov_matrix = get_sample_cov_matrix(X)
  # Calculate the eigenvalues and eigenvectors
  evals, evectors = np.linalg.eigh(cov_matrix)
  # Sort the eigenvalues in descending order
  evals, evectors = sort_evals_descending(evals,evectors)
  # Project the data onto the new eigenvector basis
  score = change_of_basis(X,evectors)

  return score, evectors, evals

#################################################
## TODO for students: Call the function to calculate the eigenvectors/eigenvalues
#################################################

# Perform PCA on the data matrix X
score, evectors, evals = pca(X)
#Plot the data projected into the new basis
plot_data_new_basis(score)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial2_Solution_d65c4e04.py)

*Example output:*

## Plot and explore the eigenvalues

Finally, we will examine the eigenvalues of the covariance matrix. Remember that each eigenvalue describes the variance of the data projected onto its corresponding eigenvector. This is an important concept because it allows us to rank the PCA basis vectors based on how much variance each one can capture. First run the code below to plot the eigenvalues (sometimes called the "scree plot"). Which eigenvalue is larger?

```python
plot_eigenvalues(evals)
```

## Interactive Demo: Exploration of the correlation coefficient

Run the following cell and use the slider to change the correlation coefficient in the data. You should see the scree plot and the plot of basis vectors update.

**Questions:**
* What happens to the eigenvalues as you change the correlation coefficient?
* Can you find a value for which both eigenvalues are equal?
* Can you find a value for which only one eigenvalue is nonzero?

```python
# @title

# @markdown Make sure you execute this cell to enable the widget!

def refresh(corr_coef=.8):
  cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)
  X = get_data(cov_matrix)
  score, evectors, evals = pca(X)
  plot_eigenvalues(evals)
  plot_basis_vectors(X, evectors)

_ = widgets.interact(refresh, corr_coef=(-1, 1, .1))
```

    interactive(children=(FloatSlider(value=0.8, description='corr_coef', max=1.0, min=-1.0), Output()), _dom_clas…

---
# Summary
- In this tutorial, we learned that goal of PCA is to find an orthonormal basis capturing the directions of maximum variance of the data. More precisely, the $i$th basis vector is the direction that maximizes the projected variance, while being orthogonal to all previous basis vectors. Mathematically, these basis vectors are the eigenvectors of the covariance matrix (also called *loadings*).
- PCA also has the useful property that the projected data (*scores*) are uncorrelated.
- The projected variance along each basis vector is given by its corresponding eigenvalue. This is important because it allows us rank the "importance" of each basis vector in terms of how much of the data variability it explains. An eigenvalue of zero means there is no variation along that direction so it can be dropped without losing any information about the original data.
- In the next tutorial, we will use this property to reduce the dimensionality of high-dimensional data.

---
# Bonus: Mathematical basis of PCA properties

```python
# @title Video 2: Properties of PCA
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="p56UrMRt6-U", width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=p56UrMRt6-U


# Financial Economics HW_02

**11510691 程远星$\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\newcommand{\using}[1]{\stackrel{\mathrm{#1}}{=}}
\newcommand{\ffrac}{\displaystyle \frac}
\newcommand{\space}{\text{ }}
\newcommand{\bspace}{\;\;\;\;}
\newcommand{\QQQ}{\boxed{?\:}}
\newcommand{\void}{\left.\right.}
\newcommand{\CB}[1]{\left\{ #1 \right\}}
\newcommand{\SB}[1]{\left[ #1 \right]}
\newcommand{\P}[1]{\left( #1 \right)}
\newcommand{\dd}{\mathrm{d}}
\newcommand{\Tran}[1]{{#1}^{\mathrm{T}}}
\newcommand{\d}[1]{\displaystyle{#1}}
\newcommand{\EE}[2][\,\!]{\mathbb{E}_{#1}\left[#2\right]}
\newcommand{\Var}[2][\,\!]{\mathrm{Var}_{#1}\left[#2\right]}
\newcommand{\Cov}[2][\,\!]{\mathrm{Cov}_{#1}\left(#2\right)}
\newcommand{\Corr}[2][\,\!]{\mathrm{Corr}_{#1}\left(#2\right)}
\newcommand{\I}[1]{\mathrm{I}\left( #1 \right)}
\newcommand{\N}[1]{\mathrm{N} \left( #1 \right)}
\newcommand{\ow}{\text{otherwise}}\void^\dagger$**

## Question 4.1

$\P{a}$

$\bspace$Consider the last four column vectors in $X$, which represent four tradable securities in the market. Their payoff matrix is $\begin{bmatrix}
1 & 0 & 0 & 0 \\
2 & 1 & 0 & 0\\
3 & 2 & 1 & 0\\
4 & 3 & 2 & 1
\end{bmatrix}$ which is obviously equivalent to the Identity matrix, the payoff for the four state-$\omega$ contingent securities. That is to say the market is complete.

$\P{b}$

$\bspace$Given the price vector $Z$, let the state price vector be $\phi=\SB{\phi_1;\phi_2;\phi_3;\phi_4}$ we have the equation:

$$\Tran\phi X = S \Rightarrow \phi = \begin{bmatrix}
0.25 \\
0.25 \\
0.25 \\
0.25
\end{bmatrix}$$

$\P{c}$

$\bspace$There's redundant vector. It could be the one with first column vector of $X$ as its payoff vector. Since that

$$\begin{bmatrix}
1 \\
1 \\
1 \\
1
\end{bmatrix} = \begin{bmatrix}
1 \\
2 \\
3 \\
4
\end{bmatrix} - \begin{bmatrix}
0 \\
1 \\
2 \\
3
\end{bmatrix}$$

$\P{d}$

$\bspace$We can take the last four. Let the portfolio be $\Theta$, then

$$\begin{bmatrix}
1 & 0 & 0 & 0 \\
2 & 1 & 0 & 0\\
3 & 2 & 1 & 0\\
4 & 3 & 2 & 1
\end{bmatrix}\Theta = I_4 \Rightarrow \Theta = \begin{bmatrix}
1 & 0 & 0 & 0 \\
-2 & 1 & 0 & 0\\
1 & -2 & 1 & 0\\
0 & 1 & -2 & 1
\end{bmatrix}$$

$\P{e}$

$\bspace$Let the holding vector be $\theta = \SB{\theta_1;\theta_2;\theta_3;\theta_4;\theta_5}$, then the position for the state-$\omega$ contingent portfolio are

$$X\theta = \begin{bmatrix}
\theta_1 + \theta_2 \\
\theta_1 + 2\theta_2 + \theta_3\\
\theta_1 + 3\theta_2 + 2\theta_3 + \theta_4 \\
\theta_1 + 4\theta_2 + 3\theta_3 + 2\theta_4 + \theta_5
\end{bmatrix}$$

$\P{f}$

$\bspace$The number of securities with independent payoff is $4$, same with the number of state in period $1$.

## Question 4.5

$\P{a}$

$\bspace$When $\phi = \SB{2;-1;2}$, we have

$$\Tran\phi X = \Tran{\begin{bmatrix}
2 \\
-1 \\
2
\end{bmatrix}} \cdot \begin{bmatrix}
1 & 0 & 0.5\\
1 & 1 & 1 \\
0 & 1 & 0.5
\end{bmatrix} = \begin{bmatrix}
1 \\
1 \\
1
\end{bmatrix} = \begin{bmatrix}
S_a \\
S_b \\
S_c
\end{bmatrix} \equiv S$$

$\P{b}$

$\bspace$It's a requirement for No-arbitrage. For that if so, then there'll be a positive payoff at period $1$ at state $\omega$ while $\phi_\omega <0$.

$\P{c}$

$\bspace$It's not a counterexample since the only ONE state price vector should still be with all positive entries. This market, at this time is not even complete.

$\P{d}$

$\bspace$When $\phi = \SB{0.5;0.5;0.5}$, we have

$$\Tran\phi X = \Tran{\begin{bmatrix}
0.5\\
0.5 \\
0.5
\end{bmatrix}} \cdot \begin{bmatrix}
1 & 0 & 0.5\\
1 & 1 & 1 \\
0 & 1 & 0.5
\end{bmatrix} = \begin{bmatrix}
1 \\
1 \\
1
\end{bmatrix} =S$$

$\P{e}$

$\bspace$Cuz this market is not even complete!

## Question 4.6

$\P{a}$

$$\ffrac{F_0} {1+r_F} = S_0 \Rightarrow F_0 = \P{1+r_F} S_0$$

$\P{b}$

$$\ffrac{F_0 + y} {1+r_F} = S_0 \Rightarrow F_0 = \P{1+r_F} S_0 - y$$

## Question 4.8

$\P{a}$

$\bspace$Still we let $\phi = \SB{\phi_1;\phi_2;\phi_3;\phi_4}$ be the state price vector so that

$$\Tran\phi \begin{bmatrix}
1 & 1 & 0 & 0 & 0 \\
1 & 2 & 1 & 0 & 0\\
1 & 3 & 2 & 1 & 0\\
1 & 4 & 3 & 2 & 1
\end{bmatrix} = S = \begin{bmatrix}
0.8 \\
2 \\
1.2 \\
0.6 \\
0.2
\end{bmatrix} \Rightarrow \phi = \begin{bmatrix}
k \\
0.6-2k \\
k \\
0.2
\end{bmatrix}, \text{where }k=0.2$$

$\bspace$The interest rate is:

$$r_F = \ffrac{1-S_1} {S_1} = \ffrac{1-\P{k+0.6-2k+k+0.2}} {k+0.6-2k+k+0.2} = 0.25$$

$\P{b}$

$\bspace$The equivalent $Q$ measure is $Q = \SB{\ffrac{k}{0.8};\ffrac{0.6-2k}{0.8};\ffrac{k}{0.8};0.25}$ where $k=0.2$.

$\P{c}$

$\bspace$Except for the forth one, the others can be verified of the property of martingale.

$$\ffrac{\Tran QX_1}{1+0.25} = 0.8\times\P{\ffrac{k}{0.8} + \ffrac{0.6-2k}{0.8} + \ffrac{k}{0.8} + 0.25} = 0.8$$

$$\ffrac{\Tran QX_2}{1+0.25} = 0.8\times\P{\ffrac{k}{0.8} +2\times \ffrac{0.6-2k}{0.8} +3\times \ffrac{k}{0.8} + 4\times0.25} = 2$$

$$\ffrac{\Tran QX_3}{1+0.25} = 0.8\times\P{1\times \ffrac{0.6-2k}{0.8} +2\times \ffrac{k}{0.8} + 3\times0.25} = 1.2$$

$$\ffrac{\Tran QX_4}{1+0.25} = 0.8\times\P{1\times \ffrac{k}{0.8} + 2\times0.25} = 0.6,\text{where }k=0.2$$

$$\ffrac{\Tran QX_5}{1+0.25} = 0.8\times0.25 = 0.2$$

## Question 5.1

$\P{a}$

$\bspace$To eliminate the chance to arbitrage, we have $d < 1+r_F < u$ while since now there's extra payoff, the relaiton should change to

$$\delta + d < 1 + r_F < \delta + u$$

$\P{b}$

$\bspace$Let the state price vector be $\phi = \SB{\phi_u;\phi_d}$. Then we have the equation:

$$\begin{cases}
S = \phi_u \cdot u S + \phi_d \cdot dS + \ffrac{\delta S} {1+r_F} \\
B = \ffrac{1} {1+r_F} = \phi_u + \phi_d
\end{cases} \Rightarrow \begin{cases}
\phi_u = \ffrac{\P{1+r_F} - \P{d+\delta}} {1+r_F}\\
\phi_d = \ffrac{\P{u+\delta} - \P{1+r_F}} {1+r_F}
\end{cases}$$

$\bspace$Hence the price of that option is $c = \phi_u \cdot [uS − K]_+ + \phi_d \cdot[dS − K]_+$

$\P{c}$

$\bspace C = \max\P{c,\SB{S-K}_+}$. Obviously with $\delta$ getting larger, the agent will tend to early exercise more cause $S$ is getting larger as well.

$\P{d}$

$\bspace$The price of the European put option is $p = \phi_u \cdot [K-uS]_+ + \phi_d \cdot[K-dS]_+$. And if it's an American put, we have $P = \max\P{p,\SB{K-S}_+}$. At this time the increasing $\delta$ will decrease the intention of the agent to early exercise.

## Question 5.5

$\P{a}$

$\bspace$Still we assume the state price vector, and obtain the equation set:

$$\begin{cases}
S = \phi_u \cdot u S + \phi_d \cdot dS  \\
B = \ffrac{1} {1+r_F} = \phi_u + \phi_d
\end{cases} \Rightarrow \begin{cases}
\phi_u = \ffrac{\P{1+r_F} - d} {1+r_F} = \ffrac{15} {22}\\
\phi_d = \ffrac{u- \P{1+r_F}} {1+r_F} = \ffrac{5} {22}
\end{cases}$$

$\bspace$So that the put option price is $p = \ffrac{15} {22}\cdot0 + \ffrac{5} {22}\times 1 = \ffrac{5} {22}$

$\P{b}$

$\bspace$Actually nothing changes here so the price remains unchanged.

## Question 6.1

$\bspace$Consider two consumption plan $c$ and $c'$ and suppose we have $c \succsim c'$. If in state $\omega^\dagger$ both two have the consumption path from $c_0$ to $c_{1\omega^\dagger}$, consumption plan $c$ must provide a higher utility for some paths to states other than $\omega^*$:

$$U\P{c} \geq U\P{c'} \Rightarrow u_0\P{c_0} + \pi_{\omega^\dagger}u_1\P{c_{x}} + \sum_{\omega \neq \omega^\dagger} \pi_{\omega}u_1\P{c_{1\omega}} \geq u_0\P{c_0} + \pi_{\omega^\dagger}u_1\P{c_{x}} + \sum_{\omega \neq \omega^\dagger} \pi_{\omega}u_1\P{c'_{1\omega}} \\
\Downarrow \\
\sum_{\omega \neq \omega^\dagger} \pi_{\omega}u_1\P{c_{1\omega}} \geq \sum_{\omega \neq \omega^\dagger} \pi_{\omega}u_1\P{c'_{1\omega}}
$$

$\bspace$So now even the comsumption path is changed to $c_{1\omega^\ddagger}$, we still have

$$U\P{c} = u_0\P{c_0} + \pi_{\omega^*}u_1\P{c_{y}} + \sum_{\omega \neq \omega^\ddagger} \pi_{\omega}u_1\P{c_{1\omega}} \geq u_0\P{c_0} + \pi_{\omega^\ddagger}u_1\P{c_{y}} + \sum_{\omega \neq \omega^\ddagger} \pi_{\omega}u_1\P{c'_{1\omega}} = U\P{c'}$$

## Question 6.4

$\bspace$By the definition of weak convexity, we have $\forall a,b, \in C = \mathbb{R}_+^{1+\Omega}$, $\alpha \in \SB{0,1}$ if $a \succsim b$, then $\alpha a + \P{1-\alpha}b \succsim b$, so that $U\P{\alpha a + \P{1-\alpha}b} \geq U\P{b}$. As assumed, we can write

$$u_0\P{\alpha a_0 + \P{1-\alpha}b_0} + \sum_{\omega\in\Omega}\pi_\omega u_1\P{\alpha a_{1\omega} + \P{1-\alpha}b_{1\omega}} \geq u_0\P{b_0} + \sum_{\omega\in\Omega}\pi_\omega u_1\P{b_{1\omega}}$$

$\bspace$And this inequality holds for all possible positive $\SB{a_{0};a_{11};\cdots;a_{1\omega}}$ and $\SB{b_{0};b_{11};\cdots;b_{1\omega}}$ so to obtain the quasiconcavity of $u_0$ and $u_1$, we plug in the consumption plan with only one positive entry while all others are $0$, then since $u_0\P{0} = 0$, $u_1\P{0} = 0$

$$u_0\P{\alpha a_0 + \P{1-\alpha}b_0} \geq u_0\P{b_0} \\[0.6em]
u_1\P{\alpha a_{1\omega} + \P{1-\alpha}b_{1\omega}} \geq u_1\P{b_{1\omega}}
$$

$\bspace\newcommand{\Exp}{\mathrm{E}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\ZZ}{\mathbb{Z}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\AcA}{\mathcal{A}}
\newcommand{\FcF}{\mathcal{F}}
\newcommand{\AsA}{\mathscr{A}}
\newcommand{\FsF}{\mathscr{F}}$And since $a \succsim b$, we have $u_0\P{b_0} = \min\CB{u_0\P{a_0}, u_0\P{b_0}}$ and $u_1\P{b_{1\omega}} = \min\CB{u_1\P{a_{1\omega}}, u_1\P{b_{1\omega}}}$. Combine these we can conclude that the weak convexity of $U$ promises the quasiconcavity of $u_0\P{\cdot}$ and $u_1\P{\cdot}$.

## Question 7.1

$\P{1}$

$$R_1\P{w} = -w \ffrac{u_1''\P{w}} {u_1'\P{w}} = -w\ffrac{\lambda^2u''\P{\lambda w}}{\lambda u'\P{\lambda w}} = R\P{\lambda w} \geq R\P{w} = R_2\P{w}$$

$\P{2}$

$\bspace$To find its maximum we first take its derivative $w.r.t.$ $a$ and let it be $0$:

$$\ffrac{\partial \Exp\SB{u_1\P{w+a\tilde r}}} {\partial a} = \Exp\SB{u'\P{\lambda w + a\lambda\tilde r}\cdot\lambda\tilde r} = 0 \iff \Exp\SB{u'\P{\lambda w + a\lambda\tilde r}\cdot\tilde r} = 0$$

$\bspace$Then find its derivative $w.r.t.$ $\lambda$, still, it should be $0$:

$$\Exp\SB{u''\P{\lambda w + \lambda a \tilde r}\tilde r\P{w+a\tilde r + \lambda a' \tilde r}} = 0$$

$\bspace$Let $\tilde w = \lambda w + \lambda a \tilde r$ we will then have

$$a' = -\ffrac{\Exp\SB{u''\P{\tilde w} \tilde w \tilde r}} {\lambda^2 \Exp\SB{u''\P{\tilde w}\tilde r^2}} = \ffrac{\Exp\SB{u'\P{\tilde w} R\P{\tilde w} \tilde r}} {\lambda^2 \Exp\SB{u''\P{\tilde w}\tilde r^2}}$$

$\bspace$The numerator is $\Exp\SB{u'\P{\tilde w} R\P{\tilde w} \tilde r} \geq R\P{w} \Exp\SB{u'\P{\tilde w} \tilde r} = 0$. And the denominator is, of course, negative, since $u''\P{\cdot}\leq 0$. Thus $a' \leq 0$ and consequently, we have $a_1\P{w} = a\P{\lambda w} \leq a\P{w} = a_2\P{w}$.

$\P{3}$

$\bspace$First we define $\lambda = w_2/w_1 \geq 1$, then

$$a\P{w_2} = a\P{\lambda w_1} \leq a\P{w_1} \leq \ffrac{w_2} {w_1}a\P{w_1} \Rightarrow \hat a\P{w_1}\geq\hat a\P{w_2}$$

## Question 7.2

$\bspace$It's a constant absolute risk aversion so we just assume that the utility function to be $u=-e^{-aw}$ with $a>0$. And let $-\pi \equiv \xi$ so that $u\P{w+\xi}=-e^{-a\P{w+\xi}}$.

$\P{1}$

$\bspace$Since it's a fair game we have $P\CB{\tilde g = +b} = P\CB{\tilde g = -b} = 0.5$.

$$\begin{align}
\Exp\SB{u\P{w+\tilde g}} &= \ffrac{1} {2}u\P{w+b} +\ffrac{1} {2}u\P{w-b} = -\ffrac{1} {2}\P{e^{-a\P{w+b}} + e^{-a\P{w-b}}} = u\P{w+\xi}\\
\Rightarrow \xi &= -\ffrac{1} {a}\log\P{\ffrac{1} {2}e^{-ab} + \ffrac{1}{2}e^{ab}}
\end{align}$$

$\P{2}$

$$\begin{align}
\Exp\SB{u\P{w+\tilde g}} &= \int_{-c}^{c} \ffrac{1} {2c}\P{-e^{-a\P{w+x}}}\dd{x} = \left.\ffrac{1} {2ac}e^{-a\P{w+x}}\right|_{-c}^{c} = \ffrac{1}{2ac}\P{e^{-a\P{w+c}}-e^{-a\P{w-c}}} = u\P{w+\xi}\\
\Rightarrow \xi &= -\ffrac{1} {a}\log\P{\ffrac{e^{ac}-e^{-ac}} {2ac}}
\end{align}$$

$\P{3}$

$$\begin{align}
\Exp\SB{u\P{w+\tilde g}} &= \int_{-\infty}^{\infty} -e^{-a\P{w+x}} \ffrac{1} {\sqrt{2\pi}\sigma} \exp\CB{-\ffrac{x^2} {2\sigma^2}}\dd{x} = -\exp\CB{-aw + \ffrac{a^2\sigma^2} {2}} = u\P{w+\xi}\\
\Rightarrow \xi &= -\ffrac{a\sigma^2} {2}
\end{align}$$

$\P{4}$

$\bspace$NONE of the above fair games show that the certainty equivalence is dependent on $w$. I think that's because they're fair games. And the extra $w$ has no effect on the absolute risk aversion.

***


+ This notebook is part of lecture 15 *Projections onto subspaces* in the OCW MIT course 18.06 by Prof Gilbert Strang [1]
+ Created by me, Dr Juan H Klopper
    + Head of Acute Care Surgery
    + Groote Schuur Hospital
    + University Cape Town
    + Email me with your thoughts, comments, suggestions and corrections

Linear Algebra OCW MIT18.06 IPython notebook [2] study notes by Dr Juan H Klopper is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.

+ [1] OCW MIT 18.06
+ [2] Fernando Pérez, Brian E. Granger, IPython: A System for Interactive Scientific Computing, Computing in Science and Engineering, vol. 9, no. 3, pp. 21-29, May/June 2007, doi:10.1109/MCSE.2007.53. URL: http://ipython.org

```python
from IPython.core.display import HTML, Image
css_file = 'style.css'
HTML(open(css_file, 'r').read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Philosopher:400,700,400italic,700italic' rel='stylesheet' type='text/css'>

```python
from sympy import init_printing, Matrix, symbols
from IPython.display import Image
from warnings import filterwarnings
```

```python
init_printing(use_latex = 'mathjax')
filterwarnings('ignore')
```

# Projections onto subspaces

## Geometry in the plane

* Projection of a vector onto another (in the plane)
* Consider the orthogonal projection of **b** onto **a**

```python
Image(filename = 'Orthogonal projection in the plane.png')
```

* Note that **p** falls on a line, which is a subspace of the plane ℝ2
* Remember from the previous lecture that orthogonal subspaces have A**x** = **0**
* Note that **p** is some scalar multiple of **a**
* With **a** perpendicular to **e** and **e** = **b** - x**a**
* Thus we have **a**T(**b** - x**a**) = 0 and x**a**T**a** = **a**T**b**
* Since **a**T**a** is a number we can simplify
$$ x=\frac { { \underline { a }  }^{ T }\underline { b }  }{ { \underline { a }  }^{ T }\underline { a }  }  $$

* We also have **p** = **a**x
$$ \underline { p } =\underline { a } x=\underline { a } \frac { { \underline { a }  }^{ T }\underline { b }  }{ { \underline { a }  }^{ T }\underline { a }  }  $$

* This equation is helpful
    * Doubling (or any other scalar multiple of) **b** doubles (or scalar multiplies) **p**
    * Doubling (or scalar multiple of) **a** has no effect

* Eventually we are looking for proj**p** = P**b**, where P is the projection matrix
$$ \underline { p } =P\underline { b } \\ P=\frac { 1 }{ { \underline { a }  }^{ T }\underline { a }  } \underline { a } { \underline { a }  }^{ T } $$

* Properties of the projection matrix P
    * The columnspace of P (C(P)) is the line which contains **a**
    * The rank is 1, rank(P) = 1
    * P is symmetrix, i.e. PT = P
    * Applying the projection matrix a second time (i.e. P2) nothing changes, thus P2 = P

## Why project?

(projecting onto more than a one-dimensional line)

* Because A**x** = **b** may not have a solution
    * **b** may not be in the columnspace
    * May have more equations than unknowns
* Solve for the closest vector in the columnspace
    * This is done by solving for **p** instead, where **p** is the projection of **b** onto the columnsapce of A
$$ A\hat { x } =\underline { p }  $$

* Now we have to get **b** orthogonally project (as **p**) onto the column(sub)space
* This is done by calculating two bases vectors for the plane that contains **p**, i.e. **a**1 and **a**2

* Going way back to the graph up top we note that **e** is perpendicular to the plane
* So, we have:
$$ A\hat { x } =\underline { p } $$
* We know that both **a**1 and **a**2 is perpendicular to **e**, so:
$$ { a }_{ 1 }^{ T }\underline { e } =0;\quad { a }_{ 2 }^{ T }\underline { e } =0\\ \because \quad \underline { e } =\underline { b } -\underline { p } \\ \because \quad \underline { p } =A\hat { x } \\ { a }_{ 1 }^{ T }\left( \underline { b } -A\hat { x }  \right) =0;\quad { a }_{ 2 }^{ T }\left( \underline { b } -A\hat { x }  \right) =0 $$

* We know that from ...
$$ \begin{bmatrix} { a }_{ 1 }^{ T } \\ { a }_{ 2 }^{ T } \end{bmatrix}\left( \underline { b } -A\hat { x }  \right) =\begin{bmatrix} 0 \\ 0 \end{bmatrix}\\ { A }^{ T }\left( \underline { b } -A\hat { x }  \right) =0 $$
* ... **e** must be in the nullspace of AT
* Which is right because from the previous lecture the nullspace of AT is orthogonal to the columnspace of A

* Simplifying the last equations we have
$$ {A}^{T}{A} \hat{x} = {A}^{T}{b} $$

* Just look back at the plane example in ℝ2 example we started with
* Simplifying things back to a column vector **a** instead of a matrix subspace A in this last equation does give us what we had in ℝ2

* Solving this we have
$$ \hat { x } ={ \left( { A }^{ T }A \right)  }^{ -1 }{ A }^{ T }\underline { b }  $$

* Which leaves us with
$$ \underline { p } =A\hat { x } \\ \underline { p } =A{ \left( { A }^{ T }A \right)  }^{ -1 }{ A }^{ T }\underline { b }  $$

* Making the projection matrix P
$$ P=A{ \left( { A }^{ T }A \right)  }^{ -1 }{ A }^{ T } $$

* Just note that for a square invertible matrix A, P is the identity matrix
* Most of the time A is not square (and thus invertible) so we have to leave the equation as it is
* Also, note that PT = P and P2 = P

## Applications

### Least squares

* Given a set of data points in two dimensions, i.e. with variables (*t*,*b*)
* We need to fit them onto the best line
* So, as an example consider the points (1,1), (2,2), (3,2)

* A best line in this instance means a straight line in the form
$$ {b}={C}+{D}{t} $$
* Using the three points above we get three equations
$$ {C}+{D}=1 \\ {C}+{2D} = 2 \\ {C}+{3D}=2 $$

* If the line goes through all points, we would give a solution
* Instead we have the following
$$ \begin{bmatrix} 1 & 1 \\ 1 & 2 \\ 1 & 3 \end{bmatrix}\begin{bmatrix} C \\ D \end{bmatrix}=\begin{bmatrix} 1 \\ 2 \\ 2 \end{bmatrix} $$
* Three equation, two unknowns, no solution, **so** solve ...
$$ { A }^{ T }A\hat { x } ={ A }^{ T }b $$
* ... which for the solution is
$$ \hat { x } ={ \left( { A }^{ T }A \right)  }^{ -1 }{ A }^{ T }b $$

```python
A = Matrix([[1, 1], [1, 2], [1, 3]])
A
```

$$\left[\begin{matrix}1 & 1\\1 & 2\\1 & 3\end{matrix}\right]$$

```python
b = Matrix([1, 2, 2])
b
```

$$\left[\begin{matrix}1\\2\\2\end{matrix}\right]$$

```python
(A.transpose() * A).inv() * A.transpose() * b
```

$$\left[\begin{matrix}\frac{2}{3}\\\frac{1}{2}\end{matrix}\right]$$

* Thus, the solution is:
$$ b=\frac { 2 }{ 3 } +\frac { 1 }{ 2 } t $$

```python

```


# Hemoglobin Model Construction

Based on Chapter 13 of Systems Biology: Simulation of Dynamic Network States

To construct a module of hemoglobin, first we import **MASSpy** and other essential packages. Constants used throughout the notebook are also defined.

```python
from os import path

import matplotlib.pyplot as plt

from sympy import Equality, Symbol, solveset, sympify, pprint

from cobra import DictList

from mass import (
    MassConfiguration, MassMetabolite, MassModel,
    MassReaction, Simulation, UnitDefinition)
from mass.test import create_test_model
from mass.io import json, sbml
from mass.util import strip_time, qcqa_model

mass_config = MassConfiguration()

mass_config.irreversible_Keq = float("inf")
```

## Model Construction

The first step of creating a model of hemoglobin is to define the `MassModel`.

```python
hemoglobin = MassModel("Hemoglobin")
```

### Metabolites

The next step is to define all of the metabolites using the `MassMetabolite` object. Some considerations for this step include the following:

1. It is important to use a clear and consistent format for identifiers and names when defining the `MassMetabolite` objects for various reasons, some of which include improvements to model clarity and utility, assurance of unique identifiers (required to add metabolites to the model), and consistency when collaborating and communicating with others.

2. In order to ensure our model is physiologically accurate, it is important to provide the `formula` argument with a string representing the chemical formula for each metabolite, and the `charge` argument with an integer representing the metabolite's ionic charge (Note that neutrally charged metabolites are provided with 0). These attributes can always be set later if necessary using the `formula` and `charge` attribute set methods. To include the Hemoglobin macromolecule in the formula, brackets are used (e.g., [HB]).

3. To indicate that the cytosol is the cellular compartment in which the reactions occur, the string "c" is provided to the `compartment` argument.

This model will be created using identifiers and names found in the [BiGG Database](http://bigg.ucsd.edu/).

In this model, there are 13 metabolites inside the cytosol compartment. Note that for metabolites without BiGG identifiers are given ones that are similar to BiGG style.

```python
hb_c = MassMetabolite(
    "hb_c",
    name="Hemoglobin",
    formula="[HB]",
    charge=0,
    compartment="c",
    fixed=False)

hb_1o2_c = MassMetabolite(
    "hb_1o2_c",
    name="Oxyhemoglobin (1)",
    formula="[HB]-O2",
    charge=0,
    compartment="c",
    fixed=False)

hb_2o2_c = MassMetabolite(
    "hb_2o2_c",
    name="Oxyhemoglobin (2)",
    formula="[HB]-O4",
    charge=0,
    compartment="c",
    fixed=False)

hb_3o2_c = MassMetabolite(
    "hb_3o2_c",
    name="Oxyhemoglobin (3)",
    formula="[HB]-O6",
    charge=0,
    compartment="c",
    fixed=False)

hb_4o2_c = MassMetabolite(
    "hb_4o2_c",
    name="Oxyhemoglobin (4)",
    formula="[HB]-O8",
    charge=0,
    compartment="c",
    fixed=False)

dhb_c = MassMetabolite(
    "dhb_c",
    name="Deoxyhemoglobin",
    formula="[HB]-C3H3O10P2",
    charge=-5,
    compartment="c",
    fixed=False)

_23dpg_c = MassMetabolite(
    "_23dpg_c",
    name="2,3-Disphospho-D-glycerate",
    formula="C3H3O10P2",
    charge=-5,
    compartment="c",
    fixed=False)

_13dpg_c = MassMetabolite(
    "_13dpg_c",
    name="3-Phospho-D-glyceroyl phosphate",
    formula="C3H4O10P2",
    charge=-4,
    compartment="c",
    fixed=False)

_3pg_c = MassMetabolite(
    "_3pg_c",
    name="3-Phospho-D-glycerate",
    formula="C3H4O7P",
    charge=-3,
    compartment="c",
    fixed=False)

o2_c = MassMetabolite(
    "o2_c",
    name="Oxygen",
    formula="O2",
    charge=0,
    compartment="c",
    fixed=False)

h_c = MassMetabolite(
    "h_c",
    name="H+",
    formula="H",
    charge=1,
    compartment="c",
    fixed=False)

pi_c = MassMetabolite(
    "pi_c",
    name="Phosphate",
    formula="HPO4",
    charge=-2,
    compartment="c",
    fixed=False)

h2o_c = MassMetabolite(
    "h2o_c",
    name="H2O",
    formula="H2O",
    charge=0,
    compartment="c",
    fixed=False)
```

### Reactions

Once all of the `MassMetabolite` objects for each metabolite, the next step is to define all of the reactions that occur and their stoichiometry.

1. As with the metabolites, it is also important to use a clear and consistent format for identifiers and names when defining when defining the `MassReaction` objects.

2. To make this model useful for integration with other models, it is important to provide a string to the `subsystem` argument. By providing the subsystem, the reactions can be easily obtained even when integrated with a significantly larger model through the `subsystem` attribute.

3. After the creation of each `MassReaction` object, the metabolites are added to the reaction using a dictionary where keys are the `MassMetabolite` objects and values are the stoichiometric coefficients (reactants have negative coefficients, products have positive ones).

This model will be created using identifiers and names found in the [BiGG Database](http://bigg.ucsd.edu/).

In this model, there are 7 reactions occuring inside the cytosol compartment.

```python
DPGase = MassReaction(
    "DPGase",
    name="Diphosphoglycerate phosphatase",
    subsystem=hemoglobin.id,
    reversible=False)
DPGase.add_metabolites({
    h2o_c: -1,
    _23dpg_c: -1,
    _3pg_c: 1,
    pi_c: 1})

DPGM = MassReaction(
    "DPGM",
    name="Diphosphoglyceromutase",
    subsystem=hemoglobin.id,
    reversible=True)
DPGM.add_metabolites({
    _13dpg_c: -1,
    _23dpg_c: 1,
    h_c: 1})

HBDPG = MassReaction(
    "HBDPG",
    name="Hemoglobin-23dpg binding",
    subsystem=hemoglobin.id,
    reversible=True)
HBDPG.add_metabolites({
    hb_c: -1,
    _23dpg_c: -1,
    dhb_c: 1})

HBO1 = MassReaction(
    "HBO1",
    name="Oxygen Loading (1)",
    subsystem=hemoglobin.id,
    reversible=True)
HBO1.add_metabolites({
    hb_c: -1,
    o2_c: -1,
    hb_1o2_c: 1})

HBO2 = MassReaction(
    "HBO2",
    name="Oxygen Loading (2)",
    subsystem=hemoglobin.id,
    reversible=True)
HBO2.add_metabolites({
    hb_1o2_c: -1,
    o2_c: -1,
    hb_2o2_c: 1})

HBO3 = MassReaction(
    "HBO3",
    name="Oxygen Loading (3)",
    subsystem=hemoglobin.id,
    reversible=True)
HBO3.add_metabolites({
    hb_2o2_c: -1,
    o2_c: -1,
    hb_3o2_c: 1})

HBO4 = MassReaction(
    "HBO4",
    name="Oxygen Loading (4)",
    subsystem=hemoglobin.id,
    reversible=True)
HBO4.add_metabolites({
    hb_3o2_c: -1,
    o2_c: -1,
    hb_4o2_c: 1})
```

After generating the reactions, all reactions are added to the model through the `MassModel.add_reactions` class method. Adding the `MassReaction` objects will also add their associated `MassMetabolite` objects if they have not already been added to the model.

```python
hemoglobin.add_reactions([
    DPGase, DPGM, HBDPG, HBO1, HBO2, HBO3, HBO4])

for reaction in hemoglobin.reactions:
    print(reaction)
```

    DPGase: _23dpg_c + h2o_c --> _3pg_c + pi_c
    DPGM: _13dpg_c <=> _23dpg_c + h_c
    HBDPG: _23dpg_c + hb_c <=> dhb_c
    HBO1: hb_c + o2_c <=> hb_1o2_c
    HBO2: hb_1o2_c + o2_c <=> hb_2o2_c
    HBO3: hb_2o2_c + o2_c <=> hb_3o2_c
    HBO4: hb_3o2_c + o2_c <=> hb_4o2_c

### Boundary reactions

After generating the reactions, the next step is to add the boundary reactions and boundary conditions (the concentrations of the boundary 'metabolites' of the system). This can easily be done using the `MassModel.add_boundary` method. With the generation of the boundary reactions, the system becomes an open system, allowing for the flow of mass through the biochemical pathways of the model. Once added, the model will be able to return the boundary conditions as a dictionary through the `MassModel.boundary_conditions` attribute.

All boundary reactions are originally created with the metabolite as the reactant. However, there are times where it would be preferable to represent the metabolite as the product. For these situtations, the `MassReaction.reverse_stoichiometry` method can be used with its `inplace` argument to create a new `MassReaction` or simply reverse the stoichiometry for the current `MassReaction.`

In this model, there is 1 boundary reaction that must be defined.

```python
SK_o2_c = hemoglobin.add_boundary(
    metabolite=o2_c, boundary_type="sink", subsystem="Pseudoreaction",
    boundary_condition=0.0200788)

print("Boundary Reactions and Values\n-----------------------------")
for reaction in hemoglobin.boundary:
    boundary_met = reaction.boundary_metabolite
    bc_value = hemoglobin.boundary_conditions.get(boundary_met)
    print("{0}\n{1}: {2}\n".format(
        reaction, boundary_met, bc_value))
```

    Boundary Reactions and Values
    -----------------------------
    SK_o2_c: o2_c <=>
    o2_b: 0.0200788

### Ordering of internal species and reactions

Sometimes, it is also desirable to reorder the metabolite and reaction objects inside the model to follow the physiology. To reorder the internal objects, one can use `cobra.DictList` containers and the `DictList.get_by_any` method with the list of object identifiers in the desirable order. To ensure all objects are still present and not forgotten in the model, a small QA check is also performed.

```python
new_metabolite_order = [
    "_23dpg_c", "hb_c", "hb_1o2_c", "hb_2o2_c",
    "hb_3o2_c", "hb_4o2_c", "dhb_c", "_13dpg_c",
    "_3pg_c",  "o2_c", "pi_c", "h_c", "h2o_c"]

if len(hemoglobin.metabolites) == len(new_metabolite_order):
    hemoglobin.metabolites = DictList(
        hemoglobin.metabolites.get_by_any(new_metabolite_order))

new_reaction_order = [
    "DPGM", "DPGase", "HBO1", "HBO2",
    "HBO3", "HBO4", "HBDPG", "SK_o2_c"]

if len(hemoglobin.reactions) == len(new_reaction_order):
    hemoglobin.reactions = DictList(
        hemoglobin.reactions.get_by_any(new_reaction_order))

hemoglobin.update_S(array_type="DataFrame", dtype=int)
```

      DPGM
      DPGase
      HBO1
      HBO2
      HBO3
      HBO4
      HBDPG
      SK_o2_c

      _23dpg_c
      1
      -1
      0
      0
      0
      0
      -1
      0

      hb_c
      0
      0
      -1
      0
      0
      0
      -1
      0

      hb_1o2_c
      0
      0
      1
      -1
      0
      0
      0
      0

      hb_2o2_c
      0
      0
      0
      1
      -1
      0
      0
      0

      hb_3o2_c
      0
      0
      0
      0
      1
      -1
      0
      0

      hb_4o2_c
      0
      0
      0
      0
      0
      1
      0
      0

      dhb_c
      0
      0
      0
      0
      0
      0
      1
      0

      _13dpg_c
      -1
      0
      0
      0
      0
      0
      0
      0

      _3pg_c
      0
      1
      0
      0
      0
      0
      0
      0

      o2_c
      0
      0
      -1
      -1
      -1
      -1
      0
      -1

      pi_c
      0
      1
      0
      0
      0
      0
      0
      0

      h_c
      1
      0
      0
      0
      0
      0
      0
      0

      h2o_c
      0
      -1
      0
      0
      0
      0
      0
      0

### Computing the steady state concentrations.

The binding of the two ligands, oxygen and DPG23, to hemoglobin is a rapid process. Since hemoglobin is confined to the RBC, we can use equilibrium assumptions for the binding reactions.

1. The binding of oxygen is at equilibrium for each form of oxygenated hemoglobin.
2. The binding of DPG23 to hemoglobin is also at equilibrium
3. The total mass of hemoglobin is a constant

These six equations have six unknowns (the six forms of Hb) and need to be solved simultaneously as a function of the oxygen and DPG23 concentrations. The equilibrium relationships can be combined with the $\text{Hb}_{\mathrm{tot}}$ mass balance, and this equation is solved for  $\text{Hb}_{\mathrm{0}}$ for given oxygen and 23DPG concentrations. Then the steady state concentrations for all other forms of hemoglobin can be computed from the equilibrium relationships.

To do this, the **SymPy** package is utilized. The metabolites and equilibrium constants are defined as `sympy.Symbol` objects, and then the equilibrium expressions are converted into `sympy.Equality` objects for symbolic calculations.

```python
metabolites = {metabolite.id: Symbol(metabolite.id)
               for metabolite in hemoglobin.metabolites}

concentration_equations = {}
# Iterate through reactions assumed to be at equilibrium
for reaction in [HBO1, HBO2, HBO3, HBO4, HBDPG]:
    equilibrium_expression = Equality(
        Symbol(reaction.Keq_str),
        strip_time(reaction.get_mass_action_ratio()))
    # Find the hemoglobin form being made as a product (bound to most oxygen)
    hb_product = [
        Symbol(metabolite.id) for metabolite in reaction.products
        if metabolite.id not in ["_23dpg_c", "hb_c", "o2_c"]].pop()
    # Solve equation for the desired form hemoglobin
    equation = solveset(equilibrium_expression, hb_product)
    equation = next(iter(equation))
    # Update equilibrium expression dict with the equation
    # for the bound form of hemoglobin. These equations will
    # be dependent on hb_c, o2_c, and _23dpg_c.
    concentration_equations.update({
        hb_product: equation.subs(concentration_equations)})
# Specify an equation for the total amount of hemoglobin
HB_total_symbol = Symbol("HB-Total")
HB_total = Equality(
    HB_total_symbol,
    sympify("+".join([
        metabolite.id for metabolite in hemoglobin.metabolites
        if "hb" in metabolite.id]), locals=metabolites))
HB_total = HB_total.subs(concentration_equations)
pprint(HB_total)
```

    HB-Total = Keq_HBDPG⋅_23dpg_c⋅hb_c + Keq_HBO1⋅Keq_HBO2⋅Keq_HBO3⋅Keq_HBO4⋅hb_c⋅

         4                                        3
    o_2_c  + Keq_HBO1⋅Keq_HBO2⋅Keq_HBO3⋅hb_c⋅o_2_c  + Keq_HBO1⋅Keq_HBO2⋅hb_c⋅o_2_c

    2
      + Keq_HBO1⋅hb_c⋅o_2_c + hb_c

At this point, the numerical values for the equilibrium constant and the total concetration of hemoglobin are specified. The total amount of hemoglobin is a constant, at circa 7.3 mM. These values are substituted into the current equations.

```python
numerical_values = {HB_total_symbol: 7.3}

DPGM.Keq = 2.3*1e6
HBO1.Keq = 41.8352
HBO2.Keq = 73.2115
HBO3.Keq = 177.799
HBO4.Keq = 1289.92
HBDPG.Keq = 1/4
SK_o2_c.Keq = 1

numerical_values.update({
    Symbol(reaction.Keq_str): reaction.Keq
    for reaction in hemoglobin.reactions})

concentration_equations.update({
    hb_form: equation.subs(numerical_values)
    for hb_form, equation in concentration_equations.items()})
HB_total = HB_total.subs(numerical_values)
pprint(HB_total)
```

                                                         4
    7.3 = 0.25⋅_23dpg_c⋅hb_c + 702446487.27335⋅hb_c⋅o_2_c  + 544565.932207695⋅hb_c

          3                          2
    ⋅o_2_c  + 3062.8177448⋅hb_c⋅o_2_c  + 41.8352⋅hb_c⋅o_2_c + hb_c

To find the steady state, we have to specify the numerical values of the variables that characterize the network environment. The flux through the Rapoport-Luebering shunt is typically about 0.44 mM/hr (Schrader 1993).  The steady state concentration of 23DPG is typically about 3.1 mM (Mehta 2005). The concentration of oxygen that we chose to solve for the steady state is 70 mmHg, that is mid way between 100 mmHg in the lung, and 40 mmHg in tissue. Using these numbers, the computed steady state concentrations are obtained, as:

```python
# Define known concentrations
concentrations = {
    metabolites["_23dpg_c"]: 3.1,
    metabolites["o2_c"]: 70*2.8684*1e-4}
# Convert the solution into a numerical value
hb_conc =  next(iter(solveset(
    HB_total.subs(concentrations),
    Symbol("hb_c"))))
concentrations.update({metabolites["hb_c"]: hb_conc})
# Solve for the rest of the hemoglobin concentrations
for hb_form, equation in concentration_equations.items():
    equation = equation.subs(concentrations)
    concentrations.update({hb_form: equation})
```

Once the steady state concentrations have been determined, the hemoglobin module can be updated. The remaining concentrations are obtained from the glycolysis module.

```python
glycolysis = create_test_model("SB2_Glycolysis.json")

for metabolite_symbol, value_symbol in concentrations.items():
    metabolite = hemoglobin.metabolites.get_by_id(str(metabolite_symbol))
    metabolite.ic = float(value_symbol)

for met in hemoglobin.metabolites:
    if met.ic is None:
        met.ic = glycolysis.metabolites.get_by_id(str(met)).ic

for metabolite, concentration in hemoglobin.initial_conditions.items():
    print("{0}: {1:.6f}".format(metabolite, concentration))
```

    _23dpg_c: 3.100000
    hb_c: 0.059625
    hb_1o2_c: 0.050085
    hb_2o2_c: 0.073625
    hb_3o2_c: 0.262842
    hb_4o2_c: 6.807613
    dhb_c: 0.046210
    _13dpg_c: 0.000243
    _3pg_c: 0.077300
    o2_c: 0.020079
    pi_c: 2.500000
    h_c: 0.000090
    h2o_c: 1.000000

With the steady state concentrations and steady state flux values, the PERCs can be calculated. For this module, the PERCs for the binding of hemoglobin to oxygen will be set manually to better reflect the physiology.

__Note:__ Reactions at equilibrium have a steady state flux of 0.

```python
DPGM.v = 0.441
DPGase.v = 0.441
HBO1.v = 0
HBO2.v = 0
HBO3.v = 0
HBO4.v = 0
HBDPG.v = 0
SK_o2_c.v = 0

hemoglobin.calculate_PERCs(update_reactions=True)

HBO1.kf = 506935
HBO2.kf = 511077
HBO3.kf = 509243
HBO4.kf = 501595
HBDPG.kf =519613
SK_o2_c.kf = 509726
```

## QC/QA Model

Before simulating the model, it is important to ensure that the model is elementally balanced, and that the model can simulate. Therefore, the `qcqa_model` function from `mass.util.qcqa` is used to provide a report on the model quality and indicate whether simulation is possible and if not, what parameters and/or initial conditions are missing.

```python
qcqa_model(hemoglobin,  parameters=True, concentrations=True,
           fluxes=True, superfluous=True, elemental=True)
```

    ╒══════════════════════════════════════════╕
    │ MODEL ID: Hemoglobin                     │
    │ SIMULATABLE: True                        │
    │ PARAMETERS NUMERICALY CONSISTENT: True   │
    ╞══════════════════════════════════════════╡
    ╘══════════════════════════════════════════╛

From the results of the QC/QA test, it can be seen that the model can be simulated and is numerically consistent.

## Steady State and Model Validation

In order to determine whether the module can be successfully integrated into a model, another model can be loaded, merged with the module, and simulated. To validate this module, it will be merged with a glycolysis model.

To find the steady state of the model and perform simulations, the model must first be loaded into a `Simulation`. In order to load a model into a `Simulation`, the model must be simulatable, meaning there are no missing numerical values that would prevent the integration of the ODEs that comprise the model. The `verbose` argument can be used while loading a model to produce a message indicating the successful loading of a model, or why a model could not load.

Once loaded into a `Simulation`, the `find_steady_state` method can be used with the `update_values` argument in order to update the initial conditions and fluxes of the model to a steady state (if necessary). The model can be simulated using the `simulate` method by passing the model to simulate, and a tuple containing the start time and the end time. The number of time points can also be included, but is optional.

After a successful simulation, two `MassSolution` objects are returned. The first `MassSolution` contains the concentration results of the simulation, and the second contains the flux results of the simulation.

To visually validate the steady state of the model, concentration and flux solutions can be plotted using the `plot_time_profile` function from `mass.visualization`. Alternatively, the `MassSolution.view_time_profile` property can be used to quickly generate a time profile for the results.

```python
glyc_hb = glycolysis.merge(hemoglobin, inplace=False)

# Setup simulation object, ensure model is at steady state
sim = Simulation(glyc_hb, verbose=True)
sim.find_steady_state(glyc_hb, strategy="simulate", update_values=True)
# Simulate from 0 to 1000 with 10001 points in the output
conc_sol, flux_sol = sim.simulate(glyc_hb, time=(0, 1e3, 1e4 + 1))
# Quickly render and display time profiles
conc_sol.view_time_profile()
```

### Storing information and references
#### Compartment
Because the character "c" represents the cytosol compartment, it is recommended to define and set the compartment in the `MassModel.compartments` attribute.

```python
hemoglobin.compartments = {"c": "Cytosol"}
print(hemoglobin.compartments)
```

    {'c': 'Cytosol'}

#### Units
All of the units for the numerical values used in this model are "Millimoles" for amount and "Liters" for volume (giving a concentration unit of 'Millimolar'), and "Hours" for time. In order to ensure that future users understand the numerical values for model, it is important to define the `MassModel.units` attribute.

The `MassModel.units` is a `cobra.DictList` that contains only `UnitDefinition` objects from the `mass.core.unit` submodule. Each `UnitDefinition` is created from `Unit` objects representing the base units that comprise the `UnitDefinition`. These `Units` are stored in the `list_of_units` attribute. Pre-built units can be viewed using the `print_defined_unit_values` function from the `mass.core.unit` submodule. Alternatively, custom units can also be created using the `UnitDefinition.create_unit` method. For more information about units, please see the module docstring for `mass.core.unit` submodule.

__Note:__ It is important to note that this attribute will NOT track units, but instead acts as a reference for the user and others so that they can perform necessary unit conversions.

```python
# Using pre-build units to define UnitDefinitions
concentration = UnitDefinition("mM", name="Millimolar",
                               list_of_units=["millimole", "per_litre"])
time = UnitDefinition("hr", name="hour", list_of_units=["hour"])

# Add units to model
hemoglobin.add_units([concentration, time])
print(hemoglobin.units)
```

    [<UnitDefinition Millimolar "mM" at 0x7f9d473a5d90>, <UnitDefinition hour "hr" at 0x7f9d474a9f50>]

## Export

After validation, the model is ready to be saved. The model can either be exported as a ".json" file or as an ".sbml" (".xml") file using their repsective submodules in `mass.io`.

To export the model, only the path to the directory and the model object itself need to be specified.

### Export using SBML

```python
sbml.write_sbml_model(mass_model=hemoglobin, filename="SB2_" + hemoglobin.id + ".xml")
```

### Export using JSON

```python
json.save_json_model(mass_model=hemoglobin, filename="SB2_" + hemoglobin.id + ".json")
```


# Metabolite concentrations follow an order relation in _Escherichia coli_

  Several studies have shown that certain metabolites maintain an order relation in their concentrations under different steady states of \emph{Escherichia coli} \cite{Bennett2008,Bennett2009a}. This order relation may be the result of diverse constraints operating at steady state, such as stoichiometric, thermodynamic and growth constraints. In the following, we will derive a theoretical explanation for this ordering of concentrations.

  We describe the concentration dynamics of the biochemical network with the general system,

  \begin{equation}
    \label{eq:1}
    \frac{dx_i}{dt} = \sum_j n_{ij} v_j(x),
  \end{equation}

  \noindent where, $n_{ij}$ represents the stoichiometric coefficient of metabolite $i$ in reaction $j$, with $n_{ij} < 0$ if it is a substrate of the reaction, $n_{ij} > 0$ if it is a product, $x$ the metabolite concentrations and $v(x)$ the metabolic fluxes, which are a function of the metabolite concentration --- the exact form provided by the selected kinetic law. All reactions in \ref{eq:1} are reversible, with the exception of the biomass production (pseudo)reaction, and the forward and backward direction are represented as two different reactions. Thus, the net flux of a reaction $v_j = v_j^{for} - v_j^{back}$. We assume that cells are growing at steady state, thus the flux through the biomass reaction, $v_{bio} > \gamma v^{max}_{bio}$, with $\gamma \in [0, 1]$, a fraction of the theoretical maximum.

  Thus far, we have determined that certain reactions are irreversible in an escenario where cells grow at steady state. Additionally, the second law of thermodynamics imposes that flux of free energy $g_j = \Delta^{\circ} G_j v_j < 0$ for a reaction to have non-zero flux \cite{Kondepudi2014a}. In our case, we have already determined the direction of the reaction with the linear programs in \ref{eq:2}, hence we only need the reaction Gibbs free energy

  \begin{equation}
    \label{eq:2}
    \Delta^{\circ} G_j < 0,
  \end{equation}

  \noindent where,

  \begin{equation}
    \label{eq:3}
    \Delta^{\circ} G_j = \Delta^{\circ} G_{r(j)} + RT \sum_i n_{ij} \log{x_i}
  \end{equation}

  \noindent in which $\Delta^{\circ} G_{r(j)} = \sum_i n_{ij}\Delta^{\circ} G_{f(i)}$. Further, the energies of formation $\Delta^{\circ} G_{f(i)}$ of the metabolites participating in the reaction can be estimated with the component contribution method \cite{Noor2013}.

```python
from gurobipy import GRB
from importlib import reload
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
import parameters as par
import data
import model
import output
reload(par)

import pickle

def saveToPickleFile(python_object, path_to_file='object.pkl'):
    """
    Save python object to pickle file
    """
    out_file = open(path_to_file,'wb')
    pickle.dump(python_object, out_file)
    out_file.close()

def readFromPickleFile(path_to_file='object.pkl'):
    """
    Load python object from pickle file.
    Returns python object.
    """
    in_file = open(path_to_file,'rb')
    python_object = pickle.load(in_file)
    return python_object

GEM = data.prepareGEM(path_to_GEM=f'{par.work_directory}/{par.model}',
                      carbon_source=par.carbon_source,
                      uptake_rate=par.uptake_rate,
                      loopless=False,
                      biomass_reaction_id=par.biomass_rxn_id)

dG0_data = data.getFreeEnergyData(GEM,
                                  work_directory=par.work_directory,
                                  pH_i=par.pH_i,
                                  Ionic_strength=par.Ionic_strength,
                                  dG0_uncertainty_threshold=par.alpha)

GEM
```

    Downloading package metadata...
    Fragments already downloaded
    Downloading package metadata...
    Fragments already downloaded
    Using license file C:\Users\tinta\gurobi.lic
    Academic license - for non-commercial use only
    Read LP format model from file C:\Users\tinta\AppData\Local\Temp\tmpitt8bst0.lp
    Reading time = 0.05 seconds
    : 1147 rows, 4276 columns, 17168 nonzeros

        Name
        iML1515

        Memory address
        0x0274bd05c848

        Number of metabolites
        1147

        Number of reactions
        2138

        Number of groups
        0

        Objective expression
        1.0*BIOMASS_Ec_iML1515_core_75p37M - 1.0*BIOMASS_Ec_iML1515_core_75p37M_reverse_35685

        Compartments
        cytosol, extracellular space, periplasm

## 2 iML1515 model and gibbs data with some examples

```python
# Let's look at the Cytidine kinase
rxn_id = 'CYTDK2'
rxn = GEM.reactions.get_by_id('CYTDK2')
rxn
```

        Reaction identifier	CYTDK2

        Name	Cytidine kinase (GTP)

        Memory address
        0x022b10302f08

        Stoichiometry

cytd_c + gtp_c --> cmp_c + gdp_c + h_c

Cytidine + GTP C10H12N5O14P3 --> CMP C9H12N3O8P + GDP C10H12N5O11P2 + H+

        GPR	b2066

        Lower bound	0.0

        Upper bound	1000.0

```python
# Its estimated Gibbs free energy of reaction is
dG0, error = dG0_data[rxn_id]["dG0"], dG0_data[rxn_id]["error"]
print(f'Reaction {rxn.name} with dG_0 = {dG0:.2f} +/- {error:.2f} kJ/mmol')
```

    Reaction Cytidine kinase (GTP) with dG_0 = -11.50 +/- 3.10 kJ/mmol

## 3 Constructing the MILP in Gurobi

  Now, the logarithm is a monotonically increasing function, hence $\log{x_p} > \log{x_q} \implies x_p > x_q$. Furthermore, we can establish if $\log{x_p} > \log{x_q}$ with the following convex optimization problem ($\mathrm{OP}_1$):

\begin{align}
    \begin{aligned}
      \label{eq:4}
      &v_{bio}^* = \max_{\substack{ \log{x} \in \rm I\!R^m, \\
                            v \in \rm I\!R^n_{\geq 0}, \\
                            \Delta^{\circ} G_{r} \in \rm I\!R^k, \\
                            y \in \{0, 1\}^k }} \; v_{bio}
      \\
      &\mathrm{s.t.}
      \\
      &1.\;Sv = 0
      \\
      &2.\;\Delta^{\circ} G_{r(j)} + RT \sum_i n_{ij} \log{x_i} - (1 - y_j^{(+, -)})M< 0 \; \forall j \in \mathrm{R_{\Delta^{\circ} G_{r}}}
      \\
      &3.\;\Delta^{\circ} G_{r}^{\dagger} - \epsilon \leq \Delta^{\circ} G_{r} \leq \Delta^{\circ} G_{r}^{\dagger} + \epsilon
      \\
      &4.\;\sum_i x_i \leq \; 300 \,\mathrm{mM}
      \\
      &5.\;\log{x}_{min} \leq \log{x} \leq \log{x}_{max}
      \\
      &6.\;v_{min} \leq v \leq v_{max}
      \\
      &7.\;v^{(+, -)}_j \leq y_j^{(+, -)} v_{max(j)} \; \forall j \in \mathrm{R_{\Delta^{\circ} G_{r}}}
      \\
      &8.\;y^+ + y^- \leq 1
    \end{aligned}
\end{align}

Here $\mathrm{OP}_2$

\begin{align}
\begin{aligned}
  \label{eq:5}
  &z = \min_{\substack{ \log{x} \in \rm I\!R^m, \\
                        v \in \rm I\!R^n_{\geq 0}, \\
                        \Delta^{\circ} G_{r} \in \rm I\!R^k, \\
                        y \in \{0, 1\}^k }} \; \log{x_p} - \log{x_q}
  \\
  &\mathrm{s.t.}
  \\
  &(1-8. \; \mathrm{OP}_1)
  \\
  &9.\;v_{bio} \geq \gamma v^*_{bio}
\end{aligned}
\end{align}

__NOTE__
What if Kms are already optimized to guarantee that metabolite concentrations that are thermodynamically feasible are already enzyme saturating? i.e., the system (or at least part of the system) has linear kinetics which are concentration independent.

```python
# Build gurobi TFBA model
reload(par)
m, variables = model.buildMILPModel(GEM, dG0_data)

# Fix internal pH (change to mM)
logx_h_c = variables['logx']['logx_h_c']
logx_h_c.lb = np.log(10**(-(par.pH_i + par.delta_pH) + 3))
logx_h_c.ub = np.log(10**(-(par.pH_i - par.delta_pH) + 3))

# Upper bound to internal h2o (mM)
logx_h2o_c = variables['logx']['logx_h2o_e']
logx_h2o_c.ub = np.log(1e2) # mM

# Upper bound to internal o2 (mM)
logx_o2_c = variables['logx']['logx_o2_c']
logx_o2_c.ub = np.log(par.maxo2) # mM

# Fixed external glucose concentration
logx_glu_e = variables['logx']['logx_glu__L_e']
logx_glu_e.lb = np.log(20) # mM 22.2
logx_glu_e.ub = np.log(23) # mM

# Find biomass maximum under thermodynamic constraints and constraint biomass reaction
m.setObjective(variables['v']['v_' + par.biomass_rxn_id], GRB.MAXIMIZE)
m.update()
m.setParam('OutputFlag', False)
m.optimize()

try:
    print(f'Maximum growth rate: {m.objval:.2f} h^-1')
    max_biomass_flux = m.getVarByName('v_' + par.biomass_rxn_id).x
    m.getVarByName('v_' + par.biomass_rxn_id).lb = par.beta * max_biomass_flux
except:
    print('Model is infeasible!')
```

    Maximum growth rate: 0.88 h^-1

```python
# # Some sanity checks of the solution
# conc_sum = 0
# for var in m.getVars():
#     if 'x_' in var.varName[:2]:
# #         conc_sum += var.x
# total_conc_internal_mets = conc_sum - m.getVarByName('x_glu__L_e').x
# print(f'Total concentration of internal metabolites: {total_conc_internal_mets:.2f} mM')
# print(f'Total sum of metabolite concentrations: {conc_sum:.2f} mM')
```

## 4 Finding candidate ordered pairs

\begin{align}
\begin{aligned}
  \label{eq:10}
  &x^* = \mathrm{argmin}_{\substack{ \log{x} \in \rm I\!R^m, \\
                        v \in \rm I\!R^n_{\geq 0}, \\
                        \Delta^{\circ} G_{r} \in \rm I\!R^k, \\
                        y \in \{0, 1\}^k }} \; \sum_{i=1}^m {(\epsilon^+_i + \epsilon^-_i)}
  \\
  &\mathrm{s.t.}
  \\
  &(1-8. \; \mathrm{OP}_1)
  \\
  &(9. \; \mathrm{OP}_2)
  \\
  &10.\;\epsilon^+ - \epsilon^- = \log{x_{rand}} - \log{x}
  \\
  &11\;\epsilon^+, \epsilon^- \geq 0
\end{aligned}
\end{align}

```python
# Find candidate ordered metabolite pairs
import datetime
reload(par)
m.update()
start = datetime.datetime.now()
X_sample, candidatePairs, sampling_model = model.findCandidatePairs(m, n_samples=par.n_samples)
end = datetime.datetime.now()
print (f'Ellapsed time: {end - start}')
print(f'There are {len(candidatePairs)} candidate pairs')
```

    Ellapsed time: 0:03:30.377159
    There are 3647 candidate pairs

### 4.2 Restricting search to central metabolism

We can also look for pairs that do not involved constrained metabolites, like h_c, o_c and glu__L_c. I'm sure this will reduce the number of pairs to evaluate considerably.

```python
# Remove candidate pairs involving constrained metabolites
constrained_mets = ['h_c', 'o2_c', 'h2o_e', 'glu__L_e', 'h2o_c',
                    'h_e', 'co2_c', 'co2_e', 'o2_e', 'h2o_p', 'o2_p',
                    'co2_p', 'h_p', 'ppi_e', 'ppi_c', 'ppi_p',
                    'pi_c', 'pi_e', 'pi_p']
filtered_pairs = [pair for pair in candidatePairs
                  if (pair[0] not in constrained_mets
                      and pair[1] not in constrained_mets)]

print(f'There are {len(filtered_pairs)} filtered candidate pairs')
```

    There are 287 filtered candidate pairs

```python
systems_df = pd.read_excel(f'{par.work_directory}/iML1515_subsystems.xlsx')
central_metabolism = ['Carbohydrate metabolism'
                      ,'Amino acid metabolism', 'Nucleotide metabolism',
                       'Lipid metabolism']
central_metabolites = data.findMetabolitesInPathways(GEM, systems_df,
                                                     central_metabolism)

centralCandidatePairs = []
for pair in filtered_pairs:
    met_i, met_j = pair
    if met_i in central_metabolites and met_j in central_metabolites:
        centralCandidatePairs.append(pair)

print(f'There are {len(centralCandidatePairs)} filtered and central candidate pairs')
```

    There are 163 filtered and central candidate pairs

```python
saveToPickleFile(X_sample, 'X_sample.pkl')
```

## 4.3 Evaluating candidate ordered pairs

```python
# Evaluate metabolite orders from candidate pairs
import datetime
start = datetime.datetime.now()
ordered_pairs = model.findConcentrationOrderedPairs(sampling_model, filtered_pairs)
end = datetime.datetime.now()
print (f'Ellapsed time: {end - start}')
```

    Evaluating concentration-ordered pairs...
    Ellapsed time: 0:06:29.622865

```python
print(f'There are {len(ordered_pairs)} true ordered pairs')
ordered_pairs
```

    There are 9 true ordered pairs

    [['dhap_c', 'g3p_c', 1.0021431070183684],
     ['3pg_c', '2pg_c', 1.001611461402214],
     ['5caiz_c', '5aizc_c', 1.012661974412411],
     ['mlthf_c', '5mthf_c', 98.64911545693387],
     ['mlthf_c', 'nad_c', 98.64911545693307],
     ['g6p_c', 'f6p_c', 1.0009747501109063],
     ['gam6p_c', 'gam1p_c', 1.0035629977225256],
     ['nadh_c', '5mthf_c', 98.64911545693369],
     ['nadh_c', 'nad_c', 98.64911545693307]]

```python
# Retrieve ordered pairs from pickle files
# import os
# import parameters as par
# directory=par.work_directory+'/'+par.directory+'/ordered_pairs'
# files = os.listdir(directory)
# ordered_pairs = []
# for file in files:
#     pairs = readFromPickleFile(directory+'/'+file)
#     ordered_pairs += pairs
```

```python
# ordered_pairs
```

## 4.4 Evaluating concentration order DAG

```python
reload(par)
reload(output)
A_plus, total_mets = output.buildDAGadjacencyMatrix(GEM, ordered_pairs)
reduced_met_orders = output.removeEdgesToProton(A_plus, total_mets)
graphData = output.buildGraphJSONData(GEM, reduced_met_orders)
output.writeOutputFiles(graphData)
```

    Writing files...

```python
from gurobipy import GRB
from importlib import reload
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
import parameters as par
import data
import model
import output
reload(par)

GEM = data.prepareGEM(path_to_GEM=f'{par.work_directory}/{par.model}',
                      carbon_source=par.carbon_source,
                      uptake_rate=par.uptake_rate,
                      loopless=False,
                      biomass_reaction_id=par.biomass_rxn_id)

# dG0_data = data.getFreeEnergyData(GEM,
#                                   work_directory=par.work_directory,
#                                   pH_i=par.pH_i,
#                                   Ionic_strength=par.Ionic_strength,
#                                   dG0_uncertainty_threshold=par.alpha)

GEM
```

    Downloading package metadata...
    Fragments already downloaded
    Downloading package metadata...
    Fragments already downloaded
    Using license file C:\Users\tinta\gurobi.lic
    Academic license - for non-commercial use only
    Read LP format model from file C:\Users\tinta\AppData\Local\Temp\tmp9qizxd21.lp
    Reading time = 0.03 seconds
    : 1147 rows, 4276 columns, 17168 nonzeros

        Name
        iML1515

        Memory address
        0x0219cfa7f3c8

        Number of metabolites
        1147

        Number of reactions
        2138

        Number of groups
        0

        Objective expression
        1.0*BIOMASS_Ec_iML1515_core_75p37M - 1.0*BIOMASS_Ec_iML1515_core_75p37M_reverse_35685

        Compartments
        cytosol, extracellular space, periplasm


**
Instituto Nacional de Pesquisa Espacial (INPE) - 2020
**

**
Disciplina: SER-347
**
***

## Projeto 9
***

# Detecção de nuvens e sombra de nuvens:

## Informação espectral e metadados
***
#### Alunos
* Aline Casassola
* Felipe Rafael de Sá Menezes Lucena
* Grazieli Rodigheri

### Importação das bibliotecas usadas

```python
# Importa a gdal, ogr e osr
from osgeo import gdal, ogr, osr

# Importa *
from gdalconst import *

# Uso de exceções
gdal.UseExceptions()

# Importa o matplotlib
import matplotlib.pyplot as plt

# Importa o Numpy
import numpy as np

# Importa os widgets
import ipywidgets as widgets

#Importa shape e mapping
from shapely.geometry import shape, mapping

import fiona
import math
import xml.etree.ElementTree as ET
import os
```

### Função que abre e retorna a imagem NIR do CBERS para calcular a máscara de nuvens:

```python
# Função para abrir dataset:
def abrir_dataset (nome_arquivo):
    print ("Abrindo o arquivo: " + nome_arquivo)

    # Tenta abrir a imagem
    dataset = None
    try:
        dataset = gdal.Open(nome_arquivo, GA_ReadOnly)
        print("Arquivo aberto com sucesso!")
    except:
        print("Erro na abertura do arquivo!")

    return dataset
```

### Seleção da imagem pelo usuário

```python
cenas = []
for i in os.listdir("../imagens/"):
    if '.' not in i:
        cenas.append("../imagens/"+i)

# Cria um dropdown com a lista de imagens
cena_folder = widgets.Dropdown(options = cenas)

print ("Selecione a pasta da imagem")
cena_folder
```

    Selecione a pasta da imagem

    Dropdown(options=('../imagens/CBERS_4_MUX_20200605_165_117_L4', '../imagens/CBERS_4_MUX_20200606_156_109_L2'),…

```python
bandas = []
values = []
for i in os.listdir(str(cena_folder.value)):
    if '.' not in i:
        values.append(i)

for i in values:
    bandas.append(('Banda ' + i[-1], cena_folder.value + '/' + i +"/"+ i[-14:] +".tif"))

# Cria um dropdown com a lista de imagens
bandas_folder = widgets.ToggleButtons(
    options=bandas,
    description='Selecione:',
    disabled=False,
    button_style=''
)

bandas_folder
```

    ToggleButtons(description='Selecione:', options=(('Banda 5', '../imagens/CBERS_4_MUX_20200606_156_109_L2/20060…

### Abertura do dataset e conversão da banda NIR para array

```python
# Abre a imagem NIR:
raster_NIR = abrir_dataset(bandas_folder.value)

# Obtém a banda única do dataset
banda_NIR = raster_NIR.GetRasterBand(1)

# Transforma a banda em array
array_banda_NIR = banda_NIR.ReadAsArray()
```

    Abrindo o arquivo: ../imagens/CBERS_4_MUX_20200606_156_109_L2/200606_BAND8/200606_BAND8.tif
    Arquivo aberto com sucesso!

## Máscara de nuvens

### Seleção do limiar pelo usuário

```python
limiar = widgets.IntSlider( min=0, max=255, value=120, step=1)

print("Selecione um limiar (DN)")
limiar
```

    Selecione um limiar (DN)

    IntSlider(value=120, max=255)

### Geração da máscara de nuvens

```python
# Cria uma cópia da banda NIR para trabalhar
mascara_nuvens = np.copy(array_banda_NIR)

# Cria a máscara em função do limiar
mascara_nuvens[mascara_nuvens < limiar.value] = False
mascara_nuvens[mascara_nuvens >= limiar.value] = True
```

### Mínimos e máximos dos pixels

```python
print("Valores banda NIR:")
print("Min: ", array_banda_NIR.min())
print("Max: ", array_banda_NIR.max())
print("")

print("Valores máscara de Nuvens:")
print("Min: ", mascara_nuvens.min())
print("Max: ", mascara_nuvens.max())
print("")
```

    Valores banda NIR:
    Min:  0
    Max:  222

    Valores máscara de Nuvens:
    Min:  0
    Max:  1

### Apresenta as imagens

```python
# Cria a figura
plt.figure(figsize = (16, 8))

# Mostra a imagem NIR
plt.subplot(121)
plt.title("Banda Selecionada")
plt.imshow(array_banda_NIR, cmap='gray');

# Mostra a máscara de nuvens
plt.subplot(122)
plt.title("Máscara de Nuvens")
plt.imshow(mascara_nuvens, cmap='gray');
```

## Estatísticas da máscara de nuvens

### Estatísticas da imagem inteira

```python
num_total_pixels = mascara_nuvens.size
print ("Número total de pixels:", num_total_pixels)

num_pixels_nuvem = len(mascara_nuvens[mascara_nuvens == 1])
print ("Número total de pixels com nuvens:", num_pixels_nuvem)

per_pixels_nuvem = (num_pixels_nuvem/num_total_pixels)*100
print ("Percentual de pixels com nuvens:", (round(per_pixels_nuvem, 3)))
```

    Número total de pixels: 51050025
    Número total de pixels com nuvens: 1182349
    Percentual de pixels com nuvens: 2.316

### Classificação da imagem de acordo com percentual de nuvens

```python
# Limiares de classificação definidos como constantes
limiar_ceu_coberto = 65
limiar_ceu_claro = 35

print ("Classificação da imagem de acordo com o percentual de nuvens: ")

if per_pixels_nuvem > limiar_ceu_coberto:
    print("Céu coberto")
elif per_pixels_nuvem < limiar_ceu_claro:
    print("Céu claro")
else:
    print("Céu parcialmente coberto")
```

    Classificação da imagem de acordo com o percentual de nuvens:
    Céu coberto

### Estatísticas por quadrantes

```python
# Obtém a metade da linha e da coluna:
meio_linhas = int(np.size(mascara_nuvens,0)/2)
meio_colunas = int(np.size(mascara_nuvens,1)/2)

# Obtém o array de cada quadrante:
Q1 = mascara_nuvens[:meio_linhas, :meio_colunas]
Q2 = mascara_nuvens[:meio_linhas, meio_colunas:]
Q3 = mascara_nuvens[meio_linhas:, :meio_colunas]
Q4 = mascara_nuvens[meio_linhas:, meio_colunas:]

# Quadrante 1:
total_pixels_Q1 = Q1.size
pixels_nuvem_Q1 = len(Q1[Q1 == 1])
per_nuvem_Q1 = (pixels_nuvem_Q1/total_pixels_Q1)*100
print ("Percentual de pixels com nuvens no Q1: ", (round(per_nuvem_Q1, 2)), "%", sep='')

# Quadrante 2:
total_pixels_Q2 = Q2.size
pixels_nuvem_Q2 = len(Q2[Q2 == 1])
per_nuvem_Q2 = (pixels_nuvem_Q2/total_pixels_Q2)*100
print ("Percentual de pixels com nuvens no Q2: ", (round(per_nuvem_Q2, 2)), "%", sep='')

# Quadrante 3:
total_pixels_Q3 = Q3.size
pixels_nuvem_Q3 = len(Q3[Q3 == 1])
per_nuvem_Q3 = (pixels_nuvem_Q3/total_pixels_Q3)*100
print ("Percentual de pixels com nuvens no Q3: ", (round(per_nuvem_Q3, 2)), "%", sep='')

# Quadrante 4:
total_pixels_Q4 = Q4.size
pixels_nuvem_Q4 = len(Q4[Q4 == 1])
per_nuvem_Q4 = (pixels_nuvem_Q4/total_pixels_Q4)*100
print ("Percentual de pixels com nuvens no Q4: ", (round(per_nuvem_Q4, 2)), "%", sep='')
```

    Percentual de pixels com nuvens no Q1: 3.48%
    Percentual de pixels com nuvens no Q2: 1.27%
    Percentual de pixels com nuvens no Q3: 1.72%
    Percentual de pixels com nuvens no Q4: 2.8%

## Salvando a máscara como arquivo tif

```python
def save_mask(matriz_de_pixels, nome_do_arquivo, dataset_de_referencia):
    # Obtém as informações de tamanho do raster de referência
    linhas = dataset_de_referencia.RasterYSize
    colunas = dataset_de_referencia.RasterXSize
    bandas = 1

    # Carrega o driver para salvar tif
    driver = gdal.GetDriverByName('GTiff')

    # Obtém o tipo de dado do dataset de referência
    data_type = dataset_de_referencia.GetRasterBand(1).DataType

    # Criar novo dataset
    dataset_output = driver.Create(nome_do_arquivo, colunas, linhas, bandas, data_type)

    # Copiar informações espaciais da banda já existente
    dataset_output.SetGeoTransform(dataset_de_referencia.GetGeoTransform())

    # Copiar informações de projeção
    dataset_output.SetProjection(dataset_de_referencia.GetProjectionRef())

    # Escrever dados da matriz NumPy na banda
    dataset_output.GetRasterBand(1).WriteArray(matriz_de_pixels)

    # Salvar valores
    dataset_output.FlushCache()

    # Fechar dataset
    dataset_output = None
```

```python
# Nome do arquivo de saída
nome_arquivo_mascara_destino = '../produtos/imagens/mascara_nuvens_' + bandas_folder.value[-16:]

# Chama a função para salvar a máscara de nuvens
save_mask(mascara_nuvens,nome_arquivo_mascara_destino,raster_NIR)
```

## Máscara de sombras
### Abrindo a máscara de nuvens

```python
# Abre a mascara de nuvem gerada anteriormente
mascara_nuvens = abrir_dataset (nome_arquivo_mascara_destino)

# Obtém a banda única do dataset
banda_mascara = mascara_nuvens.GetRasterBand(1)
```

    Abrindo o arquivo: ../produtos/imagens/mascara_nuvens_200606_BAND8.tif
    Arquivo aberto com sucesso!

### Gerando a máscara de sombras inicial (vetorização da máscara de nuvens)

```python
srs = osr.SpatialReference()
srs.ImportFromWkt(mascara_nuvens.GetProjectionRef())

shp_layername_cloud = '../produtos/shp/mascara_nuvens_' + bandas_folder.value[-16:-4]
driver = ogr.GetDriverByName("ESRI Shapefile")
shp_datasource = driver.CreateDataSource(shp_layername_cloud + '.shp')
shp_layer = shp_datasource.CreateLayer(shp_layername_cloud, srs=srs)

new_field = ogr.FieldDefn('DN', ogr.OFTReal)
shp_layer.CreateField(new_field)

gdal.Polygonize(banda_mascara, banda_mascara, shp_layer, 0, [], callback=None)

shp_datasource.Destroy()
shp_layername_cloud = shp_layername_cloud + ".shp"
```

### Deslocamento da máscara de nuvens a partir de um delta_x e delta_y

### Delta a partir dos metadados

```python
tree = ET.parse(bandas_folder.value[:-3]+'xml')
root = tree.getroot()

sun_incidence = root.find("{http://www.gisplan.com.br/xmlsat}sunIncidenceAngle")
degree_sun_incidence = int(sun_incidence.find('{http://www.gisplan.com.br/xmlsat}degree').text)
minute_sun_incidence = int(sun_incidence.find('{http://www.gisplan.com.br/xmlsat}minute').text)
second_sun_incidence = int(sun_incidence.find('{http://www.gisplan.com.br/xmlsat}second').text)

sun_incidence = degree_sun_incidence + minute_sun_incidence/60 + second_sun_incidence/3600

image = root.find("{http://www.gisplan.com.br/xmlsat}image")
sun_position = image.find('{http://www.gisplan.com.br/xmlsat}sunPosition')
sun_azim = float(sun_position.find('{http://www.gisplan.com.br/xmlsat}sunAzimuth').text)
sun_zenith = float(sun_position.find('{http://www.gisplan.com.br/xmlsat}elevation').text)

sun_azim = math.radians(sun_azim)
sun_zenith = math.radians(sun_zenith)
view_azim = math.radians(0)
view_zenith = math.radians(0)

sen_sun_azi = math.sin(sun_azim)
cos_sun_azi = math.cos(sun_azim)
sen_view_azi = math.sin(view_azim)
cos_view_azi = math.cos(view_azim)
tan_sun_zenith = math.tan(sun_zenith)
tan_view_zenith = math.tan(view_zenith)

shadow_direction = math.pi + math.atan(sen_sun_azi*tan_sun_zenith-sen_view_azi*tan_view_zenith/
                                       (cos_sun_azi*tan_sun_zenith-cos_view_azi*tan_view_zenith))

h = 1500 #Altura das nuvens

shadow_distance = h * math.tan(math.radians(sun_incidence))

# shadow_distance2 = h * math.sqrt(
#     ( (sen_sun_azi*tan_sun_zenith) - (sen_view_azi*tan_view_zenith) )**2 +
#     ( (cos_sun_azi*tan_sun_zenith) - (cos_view_azi*tan_view_zenith) )**2
#     )

delta = (shadow_distance*math.sin(shadow_direction), shadow_distance*math.cos(shadow_direction))
```

\begin{equation} {\phi _s} = \pi + \arctan \left({\frac{{\sin {\phi _s}\tan {\theta _s} - \sin {\phi _v}\tan {\theta _v}}}{{\cos {\phi _s}\tan {\theta _s} - \cos {\phi _v}\tan {\theta _v}}}} \right) \end{equation}

B. Zhong et al. (2017)

\begin{equation} {d _s} = h * \tan(\Theta _s) \end{equation}

### Deslocamento da máscara

```python
# Essas três funções realizam o deslocamento das feições da camada. (Elas são recursivas entre si)
def movePoint_Coords(coords, delta):     # "delta" é uma tupla (delta_x, delta_y)
    return tuple(c + d for c, d in zip(coords, delta))
def moveLine_Coords(coords, delta):
    return list(movePoint_Coords(pt_coords, delta) for pt_coords in coords)
def movePolygon_Coords(coords, delta):
    return list(moveLine_Coords(ring_coords, delta) for ring_coords in coords)

with fiona.open(shp_layername_cloud, "r") as shadow_cloud:

    with fiona.open("../produtos/shp/mascara_sombra_deslocada.shp",
                    "w",
                    driver=shadow_cloud.driver,
                    schema=shadow_cloud.schema,
                    crs=shadow_cloud.crs) as moved_shadow:

        # Deslocamento de todas as feições da camada de máscara de nuvens vetorizada
        for feature in shadow_cloud:
            try:
                feature['geometry']['coordinates'] = movePolygon_Coords(feature['geometry']['coordinates'], delta)
                moved_shadow.write(feature)
            except:
                print("Error processing record %s:", feature)
```

### Subtração da mascara deslocada pela mascara de nuvens

Observação: a intersecção entre a nuvem e a sombra deve ser removida, visto que não é somente sombra. Com a subtração, na máscara, permanece o que é exclusivamente sombra.

```python
with fiona.open(shp_layername_cloud, "r") as shadow_cloud:
    with fiona.open("../produtos/shp/mascara_sombra_deslocada.shp", "r") as moved_shadow:

        shp_layername_shadow = "../produtos/shp/mascara_sombra_" + bandas_folder.value[-16:-3] +"shp"

        with fiona.open(shp_layername_shadow,
                    "w",
                    driver=shadow_cloud.driver,
                    schema=shadow_cloud.schema,
                    crs=shadow_cloud.crs) as dif:

            for i, j in zip(moved_shadow,shadow_cloud):

                # Corrigir erros topológico que impedem a subtração dos poligonos
                polygons_source = shape(j['geometry'])
                polygons_shift = shape(i['geometry'])

                polygons_source_correct = polygons_source.buffer(0)
                polygons_shift_correct = polygons_shift.buffer(0)

                j['geometry'] = mapping(polygons_source_correct)
                i['geometry'] = mapping(polygons_shift_correct)
                #################################################################

                try:
                    difference = shape(i['geometry']).difference(shape(j['geometry']))
                    i['geometry'] = mapping(difference)
                    dif.write(i)
                except:
                    print("Error processing record %s:", i)

# Apagar arquivo shapefile auxiliar
driver.DeleteDataSource('../produtos/shp/mascara_sombra_deslocada.shp');
```

# Transformar o shp das sombras em máscara

```python
# Abrir a máscara de nuvens para setar a de sombras com a mesma configuração (limites, tamanho do pixel...)
mascara_nuvens = abrir_dataset (nome_arquivo_mascara_destino)
```

    Abrindo o arquivo: ../produtos/imagens/mascara_nuvens_200606_BAND8.tif
    Arquivo aberto com sucesso!

```python
linhas = mascara_nuvens.RasterXSize
colunas = mascara_nuvens.RasterYSize

# Criação da camada vetorial
shp_mask = ogr.Open(shp_layername_shadow)
layer = shp_mask.GetLayer()
output = '../produtos/imagens/mascara_sombras_' + bandas_folder.value[-16:]
driver = gdal.GetDriverByName('GTiff')
dataset_output = driver.Create(output, linhas, colunas, 1, gdal.GDT_Byte)

# Copiar informações espaciais da mascara de nuvens
dataset_output.SetGeoTransform(mascara_nuvens.GetGeoTransform())

# Copiar informações de projeção da mascara de nuvens
dataset_output.SetProjection(mascara_nuvens.GetProjectionRef())

# Escrever dados na banda
band = dataset_output.GetRasterBand(1)

# Salvar valores
band.FlushCache()

# Ralizar a transformação de poligonos para raster
gdal.RasterizeLayer(dataset_output, [1], layer)

# Fechar dataset
dataset_output = None
```

# Resultado