```python
from sympy import symbols, pprint
from sympy import diff
from sympy.solvers import solve
import numpy as np
from scipy import optimize
import string
import random
from autodp.transformer_zoo import Composition
from functools import lru_cache

# data subject
class Entity():
    def __init__(self, name="", id=None):
        self.name = name
        self.id = id

scalar_name2obj = {}
```

```python
def individual_RDP_gaussian(params, alpha):
    """
    :param params:
        'sigma' --- is the normalized noise level: std divided by global L2 sensitivity
    :param alpha: The order of the Renyi Divergence
    :return: Evaluation of the RDP's epsilon
    """
    sigma = params['sigma']
    value = params['value']
    L = params['L']
    assert(sigma > 0)
    assert(alpha >= 0)

    return (alpha * (L ** 2) * (value**2)) / (2 * (sigma**2))

import math

from autodp.autodp_core import Mechanism
from autodp import rdp_bank, dp_bank, fdp_bank, utils
from autodp import transformer_zoo

from scipy.optimize import minimize_scalar

# Example of a specific mechanism that inherits the Mechanism class
class iDPGaussianMechanism(Mechanism):
    def __init__(self, sigma, value, L, entity, name='Gaussian',
                 RDP_off=False, approxDP_off=False, fdp_off=True,
                 use_basic_RDP_to_approxDP_conversion=False,
                 use_fDP_based_RDP_to_approxDP_conversion=False):
        # the sigma parameter is the std of the noise divide by the l2 sensitivity
        Mechanism.__init__(self)

        self.name = name # When composing
        self.params = {'sigma': sigma, 'value':value, 'L':L} # This will be useful for the Calibrator
        self.entity = entity
        # TODO: should a generic unspecified mechanism have a name and a param dictionary?

        self.delta0 = 0
        if not RDP_off:
            new_rdp = lambda x: individual_RDP_gaussian({'sigma': sigma,
                                                        'value': value,
                                                        'L':L}, x)
            if use_fDP_based_RDP_to_approxDP_conversion:
                # This setting is slightly more complex, which involves converting RDP to fDP,
                # then to eps-delta-DP via the duality
                self.propagate_updates(new_rdp, 'RDP', fDP_based_conversion=True)
            elif use_basic_RDP_to_approxDP_conversion:
                self.propagate_updates(new_rdp, 'RDP', BBGHS_conversion=False)
            else:
                # This is the default setting with fast computation of RDP to approx-DP
                self.propagate_updates(new_rdp, 'RDP')

        if not approxDP_off: # Direct implementation of approxDP
            new_approxdp = lambda x: dp_bank.get_eps_ana_gaussian(sigma, x)
            self.propagate_updates(new_approxdp,'approxDP_func')

        if not fdp_off: # Direct implementation of fDP
            fun1 = lambda x: fdp_bank.log_one_minus_fdp_gaussian({'sigma': sigma}, x)
            fun2 = lambda x: fdp_bank.log_neg_fdp_grad_gaussian({'sigma': sigma}, x)
            self.propagate_updates([fun1,fun2],'fDP_and_grad_log')
            # overwrite the fdp computation with the direct computation
            self.fdp = lambda x: fdp_bank.fDP_gaussian({'sigma': sigma}, x)

        # the fDP of gaussian mechanism is equivalent to analytical calibration of approxdp,
        # so it should have been automatically handled numerically above

        # Discussion:  Sometimes delta as a function of eps has a closed-form solution
        # while eps as a function of delta does not
        # Shall we represent delta as a function of eps instead?
```

```python
class AdversarialAccountant():

    def __init__(self, max_budget=10, delta=1e-6):
        self.entity2ledger = {}
        self.max_budget = max_budget
        self.delta = delta

    def append(self, entity2mechanisms):
        for key, ms in entity2mechanisms.items():
            if key not in self.entity2ledger.keys():
                self.entity2ledger[key] = list()
            for m in ms:
                self.entity2ledger[key].append(m)

    def get_eps_for_entity(self, entity_name):
        # compose them with the transformation: compose.
        compose = Composition()
        mechanisms = self.entity2ledger[entity_name]
        composed_mech = compose(mechanisms, [1]*len(mechanisms))

        # Query for eps given delta
        return Scalar(value=composed_mech.get_approxDP(self.delta),
                      min_val=0,
                      max_val=self.max_budget,
                      ent=Entity(name=entity_name))

    def has_budget(self, entity_name):
        return self.get_eps_for_entity(entity_name)._value < self.max_budget

    @property
    def entities(self):
        return self.entity2ledger.keys()

    @property
    def overbudgeted_entities(self):
        entities = set()

        for ent in self.entities:
            if not self.has_budget(ent):
                entities.add(ent)

        return entities

    def print_ledger(self, delta=1e-6):
        for entity, mechanisms in self.entity2ledger.items():
            print(entity + "\t" + str(self.get_eps_for_entity(entity)._value))
```

```python
def run(func, **kwargs):
    return func.subs(kwargs)

def run_specific(f, **kwargs):
    """pass in kwargs to run in fixed polynomial because this is what
    optimize.brute expects"""
    return run(f, **kwargs)

@lru_cache(maxsize=None)
def search(run_specific_args, rranges, full_output, finish, disp):
    return optimize.shgo(run_specific_args, rranges)

class Scalar():

    def __init__(self, value, min_val=None, max_val=None, poly=None, ent=None, name=None):

        if name is None:
            lower_upper_alphabet = string.ascii_letters
            name = ''.join([random.choice(lower_upper_alphabet) for i in range(5)])

        self.name = name
        self._value = value
        self._min_val = min_val
        self._max_val = max_val
        self.enabled = True

        if poly is not None:
            # if this Scalar is being formed as a function of other Scalar objects
            self._poly = poly

        elif ent is not None:
            # if you're creating a Scalar for the first time (no parents)
            self.scalar_name = self.name + "_" + ent.name
            self._poly = symbols(self.scalar_name)
            scalar_name2obj[self.scalar_name] = self
        else:
            raise Exception("Poly or ent must be not None")

    @property
    def poly(self):
        return self._poly

    @property
    def value(self):

        if(self._value is not None):
            return self._value

        sy_names = self.poly.free_symbols
        sym = list()
        for sy_name in sy_names:
            sym.append(scalar_name2obj[str(sy_name)])

        run_specific_args, index2symbol, symbol2index = self.create_run_specific_args(f=self.poly)

        inputs = list()

        for sym in index2symbol:
            inputs.append(scalar_name2obj[sym]._value)
        return run_specific_args(inputs)

    @property
    def min_val(self):
        return self._min_val

    @property
    def max_val(self):
        return self._max_val

    def __mul__(self, other):

        result_poly = self.poly * other.poly

        result = Scalar(value=None, poly=result_poly)

        return result

    def __add__(self, other):
        result_poly = self.poly + other.poly

        result = Scalar(value=None, poly=result_poly)

        return result

    def __sub__(self, other):
        result_poly = self.poly - other.poly

        result = Scalar(value=None, poly=result_poly)

        return result

    def __str__(self):
        return str(self.poly) + "=" + str(self.value)

    def __repr__(self):
        return str(self)

    @property
    def sens(self):
        if self.min_val is not None and self.max_val is not None:
            return self.max_val - self.min_val

    def neg_deriv(self, name):
        obj = scalar_name2obj[name]
        return -diff(self.poly, obj.poly)

    def create_run_specific_args(self, f):

        free_symbols_list = list(self.poly.free_symbols)
        index2symbol = list(map(lambda x:str(x), free_symbols_list))

        symbol2index = {}
        for i,sym in enumerate(index2symbol):
            symbol2index[sym] = i

        def _run_specific_args(tuple_of_args, *params):
            kwargs = {}
            for sym,i in symbol2index.items():
                kwargs[sym] = tuple_of_args[i]

            return run_specific(f=f, **kwargs)

        return _run_specific_args, index2symbol, symbol2index

    def get_mechanism(self,
            symbol_name = 'b',
            sigma = 0.1):

        # Step 1: get derivative we want to maximize
        z = self.neg_deriv(symbol_name)

        # Step 2: prepare metadata for optimize.brute() function
        sy_names = z.free_symbols
        sym = list()
        for sy_name in sy_names:
            sym.append(scalar_name2obj[str(sy_name)])

        run_specific_args, index2symbol, symbol2index = self.create_run_specific_args(f=z)

        rranges = list()
        for i,sym in enumerate(index2symbol):
            obj = scalar_name2obj[sym]
            rranges.append(tuple([obj.min_val, obj.max_val]))

        # Step 3: maximize the derivative over a bounded range of <entity_name>
        resbrute = search(run_specific_args, tuple(rranges), full_output=False, finish=None, disp=True)
        resbrute = resbrute.x

        if isinstance(resbrute, np.float64):
            L = resbrute
        else:
            L = resbrute[symbol2index[symbol_name]]
        input_obj = scalar_name2obj[symbol_name]

        # Step 4: create the gaussian mechanism object
        gm1 = iDPGaussianMechanism(sigma=sigma, value=input_obj._value, L=L, entity=symbol_name.split("_")[1], name='gm_'+symbol_name)

        return gm1

    def get_all_entity_mechanisms(self,sigma=0.1):
        sy_names = self.poly.free_symbols
        entity2mechanisms = {}
        for sy_name in sy_names:
            mechanism = self.get_mechanism(symbol_name=str(sy_name), sigma=sigma)

            split_name = str(sy_name).split("_")
            entity_name = split_name[1]
            var_name = split_name[0]

            if entity_name not in entity2mechanisms.keys():
                entity2mechanisms[entity_name] = list()

            entity2mechanisms[entity_name].append(mechanism)

        return entity2mechanisms

    @property
    def entities(self):
        entities = set()
        sy_names = self.poly.free_symbols
        for sy_name in sy_names:
            entities.add(str(sy_name).split("_")[1])
        return entities

    def publish(self, acc, sigma=1.5):
        acc_original = acc

        assert sigma > 0

        acc_temp = deepcopy(acc_original)

        # get mechanisms for new publish event
        ms = self.get_all_entity_mechanisms(sigma=sigma)
        acc_temp.append(ms)

        overbudgeted_entities = acc_temp.overbudgeted_entities

        sample = random.gauss(0,sigma)

        while len(overbudgeted_entities) > 0:

            for sy_name in self.poly.free_symbols:
                entity_name = str(sy_name).split("_")[1]
                if(entity_name in overbudgeted_entities):
                    sym = scalar_name2obj[str(sy_name)]
                    self._poly = self.poly.subs(sym.poly, 0)

            acc_temp = deepcopy(acc_original)

            # get mechanisms for new publish event
            ms = self.get_all_entity_mechanisms(sigma=sigma)
            acc_temp.append(ms)

            overbudgeted_entities = acc_temp.overbudgeted_entities

        output = self.value + sample

        acc_original.entity2ledger = deepcopy(acc_temp.entity2ledger)

        return output

```

```python
from copy import deepcopy
```

```python
bob = Scalar(value=1, min_val=-2, max_val=2, ent=Entity(name="Bob"))
bobby = Scalar(value=1, min_val=-2, max_val=2, ent=Entity(name="Bob"))
alice = Scalar(value=1, min_val=-1, max_val=1, ent=Entity(name="Alice"))
charlie = Scalar(value=2, min_val=-2, max_val=2, ent=Entity(name="Charlie"))
david = Scalar(value=2, min_val=-2, max_val=2, ent=Entity(name="David"))

acc = AdversarialAccountant(max_budget=70)

# PhiScalar()
bob2 = bob + bob
bobby2 = bobby + bobby

alice2 = alice + alice
charlie2 = charlie + charlie

(alice2 * bobby2) + (alice2 * bob2)

def __add__(self, other):
    if self.gama==False and other.gamma== False and self.entity == other.entity
        return Scalar(entity=self.entity,
                      value=self.value + other.value,
                      min_val=self.min_val + other.min_value,
                      max_val=self.max_val + other.max_val,
                      symbol_name=entity_name + "_" + random_hash)
    else:

        self.gama = True
        self.poly = symbol(self.symbol_name)

        other.gama = True
        other.poly = symbol(other.symbol_name)

# GammaScalar()
out = bob2**2 + alice2*0.5 + bobby2**3

#
```

```python
%%timeit -n1 -r1

public_out = out.publish(acc=acc, sigma=0.5)
acc.print_ledger()
```

    Bob	27.811968501910776
    Alice	11.688596249354894
    45.5 ms ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)

```python

```

```python

```

```python
result = optimize.shgo(eggholder, bounds)
```

```python
result.x, result.fun
```

    (array([439.48099854, 453.9774378 ]), -935.3379515605761)

```python

```


# Fracciones parciales

Laboratorio de cálculo diferencial e integral 2.

Prof. Armando Benjamin Cruz Hinojosa (Gory).

```python
from sympy import *
```

```python
x = symbols('x', real=True)

lin = x + 1
cuad = x**2 + 1
```

```python
expand(cuad**2)
```

$\displaystyle x^{4} + 2 x^{2} + 1$

```python
a1,a2,a3,b1,b2,b3,c1,c2,c3 = symbols('a1 a2 a3 b1 b2 b3 c1 c2 c3', real=True)
```

```python
suma_expandida = expand(
    (a1*x + b1)*cuad**5*lin**6 +
    (a2*x + b2)*cuad**4*lin**6 +
    (a3*x + b3)*cuad**3*lin**6
)
```

```python
suma_expandida
```

$\displaystyle a_{1} x^{17} + 6 a_{1} x^{16} + 20 a_{1} x^{15} + 50 a_{1} x^{14} + 100 a_{1} x^{13} + 166 a_{1} x^{12} + 236 a_{1} x^{11} + 290 a_{1} x^{10} + 310 a_{1} x^{9} + 290 a_{1} x^{8} + 236 a_{1} x^{7} + 166 a_{1} x^{6} + 100 a_{1} x^{5} + 50 a_{1} x^{4} + 20 a_{1} x^{3} + 6 a_{1} x^{2} + a_{1} x + a_{2} x^{15} + 6 a_{2} x^{14} + 19 a_{2} x^{13} + 44 a_{2} x^{12} + 81 a_{2} x^{11} + 122 a_{2} x^{10} + 155 a_{2} x^{9} + 168 a_{2} x^{8} + 155 a_{2} x^{7} + 122 a_{2} x^{6} + 81 a_{2} x^{5} + 44 a_{2} x^{4} + 19 a_{2} x^{3} + 6 a_{2} x^{2} + a_{2} x + a_{3} x^{13} + 6 a_{3} x^{12} + 18 a_{3} x^{11} + 38 a_{3} x^{10} + 63 a_{3} x^{9} + 84 a_{3} x^{8} + 92 a_{3} x^{7} + 84 a_{3} x^{6} + 63 a_{3} x^{5} + 38 a_{3} x^{4} + 18 a_{3} x^{3} + 6 a_{3} x^{2} + a_{3} x + b_{1} x^{16} + 6 b_{1} x^{15} + 20 b_{1} x^{14} + 50 b_{1} x^{13} + 100 b_{1} x^{12} + 166 b_{1} x^{11} + 236 b_{1} x^{10} + 290 b_{1} x^{9} + 310 b_{1} x^{8} + 290 b_{1} x^{7} + 236 b_{1} x^{6} + 166 b_{1} x^{5} + 100 b_{1} x^{4} + 50 b_{1} x^{3} + 20 b_{1} x^{2} + 6 b_{1} x + b_{1} + b_{2} x^{14} + 6 b_{2} x^{13} + 19 b_{2} x^{12} + 44 b_{2} x^{11} + 81 b_{2} x^{10} + 122 b_{2} x^{9} + 155 b_{2} x^{8} + 168 b_{2} x^{7} + 155 b_{2} x^{6} + 122 b_{2} x^{5} + 81 b_{2} x^{4} + 44 b_{2} x^{3} + 19 b_{2} x^{2} + 6 b_{2} x + b_{2} + b_{3} x^{12} + 6 b_{3} x^{11} + 18 b_{3} x^{10} + 38 b_{3} x^{9} + 63 b_{3} x^{8} + 84 b_{3} x^{7} + 92 b_{3} x^{6} + 84 b_{3} x^{5} + 63 b_{3} x^{4} + 38 b_{3} x^{3} + 18 b_{3} x^{2} + 6 b_{3} x + b_{3}$

```python

```

```python

```

```python
r = factor(g, extension=list(filter(lambda x: x.is_real, roots(g).keys())))
r
```

$\displaystyle \left(x - \sqrt[3]{2}\right) \left(x^{2} + \sqrt[3]{2} x + 2^{\frac{2}{3}}\right)$

```python
r = factor(g, extension=list(filter(lambda x: x.is_real, roots(g).keys())))
r
```

$\displaystyle x^{3} - 2$

```python
r = apart(f/g, extension=list(filter(lambda x: x.is_real, roots(g).keys())))
r
```

$\displaystyle - \frac{- 4 x + \sqrt[3]{2} x + 2 \cdot 2^{\frac{2}{3}} x - 4 - 2 \cdot \sqrt[3]{2} + 2 \cdot 2^{\frac{2}{3}}}{6 \left(x^{2} + \sqrt[3]{2} x + 2^{\frac{2}{3}}\right)} + \frac{\sqrt[3]{2} + 2 + 2 \cdot 2^{\frac{2}{3}}}{6 \left(x - \sqrt[3]{2}\right)}$

```python
roots(g).keys()
```

    dict_keys([2**(1/3), -2**(1/3)/2 - 2**(1/3)*sqrt(3)*I/2, -2**(1/3)/2 + 2**(1/3)*sqrt(3)*I/2])

```python

```


```
# Install packages first
!pip install qml
!pip install ase
```

# Lecture 2: Introduction to Machine Learning in Python, May 2020
By Dr. Anders Christensen `anders.christensen @ unibas.ch`

---

## Part 1: Non-linear regression?
Last time we saw *linear least squares regression*:

```
import matplotlib.pyplot as plt
# plt.rcParams['figure.figsize'] = [8, 6]
import numpy as np

# X-values
x = np.arange(0,20.0, 0.2)

# Y-values: Y = 1.2*X + random noise
y = 1.2 * x + np.random.normal(scale=2.0, size=len(x))

plt.scatter(x,y)
plt.plot(x, x*1.2, color="g")

plt.show()
```

Approximate a target function as a weighted sum of the features:
\begin{equation}
y(\mathbf{x}) = x_1 \alpha_1 + x_2 \alpha_2 + \dots + x_n \alpha_n
\end{equation}
In matrix notation:

\begin{equation}
\mathbf{y} = \mathbf{X} \mathbf{\alpha}
\end{equation}

Minimze the error:
\begin{equation}
\mathbf{\hat{\alpha}} = \text{arg min} || \mathbf{y}^\text{ref} - \mathbf{X}\mathbf{\alpha}||^2
\end{equation}

## What a about linear regression for $\sin \left(x\right)$?

Same example as above, just with $y(x) = \sin \left(x\right)$:

```
import numpy as np
import matplotlib.pyplot as plt

x = np.arange(0,6.6, 0.6)
y = np.sin(x) #+ (np.random.random(size=len(x)) - 0.5) * 0.5

print(x.shape)
print(y.shape)

xplot = np.arange(0,6.6, 0.01)

plt.scatter(x,y, label="Training")
plt.plot(xplot, np.sin(xplot), color="g", label="sin(x)")

plt.legend()
plt.show()
```

Just like yesterday, we can use `numpy.linalg.lstsq()` to solve the optimal coefficients by minimizing the error:

\begin{equation}
\mathbf{\hat{\alpha}} = \text{arg min} || \mathbf{y}^\text{ref} - \mathbf{X}\mathbf{\alpha}||^2
\end{equation}
Closed-form solution:
\begin{equation}
\mathbf{\hat{\alpha}} = \left(\mathbf{X}^\top\mathbf{X} \right)^{-1}\mathbf{X}^\top\mathbf{y}^\text{ref}
\end{equation}
However, Numpy uses a singluar-value decomposition.

```
x_reshape = x.reshape(len(x),1)
alpha, sing, rank, vecs = np.linalg.lstsq(x_reshape, y, rcond=None)

print(alpha)
```

Making new predictions:
*   Test set of new $x$-values

```
x_test = np.arange(0.0, 6.6, 0.6)
print(x_test)
print(x_test.shape)
```

Predictions are made with the same equation:
\begin{equation}
\mathbf{y} = \mathbf{X} \mathbf{\alpha}
\end{equation}

```
x_test = x_test.reshape(len(x_test),1)
y_test = np.dot(x_test, alpha)

print(y_test)

```

Making a plot of the predictions:

```
plt.plot(xplot, np.sin(xplot), color="g", label="sin(x)")
plt.scatter(x, y, label="Training")
plt.scatter(x_test, y_test, color="r", label="Test")

plt.legend()
plt.show()
```

## Part 1½: Kernel ridge regression
The exercise today is about coding kernel ridge regession!

The idea is to describe the target function in terms of basis functions ("kernel functions") placed on the training points.

Plot of basis functions for the $\sin\left(x\right)$ curve:

```
import qml.kernels
import qml.math

sigma = 0.3
K = qml.kernels.gaussian_kernel(x_reshape, x_reshape, sigma)
K[np.diag_indices_from(K)] += 1e-10
alphas = qml.math.cho_solve(K, y)

xgauss = np.arange(0.0,6.0, 0.01)
fit_curve = np.zeros(len(xgauss))

for alp, xval in zip(alphas, x):
  ygauss = np.exp(-(xgauss-xval)**2/(2*sigma**2)) * alp
  fit_curve += ygauss
  # plt.plot(xgauss, ygauss, color="C0")

plt.scatter(x,y, label="Training")
plt.plot(xgauss, np.sin(xgauss), color="g", label="sin(x)")
plt.plot(xgauss, fit_curve, color="C3", label="KRR")

# plt.ylim([-1.5,1.5])
plt.legend()
plt.show()
```

$y\left(x\right)$ is now a sum of weighted basis functions:
\begin{equation}
\hat{y}\left(\tilde{\mathbf{x}}\right) = \sum_i \kappa\left(\mathbf{x}_i, \tilde{\mathbf{x}} \right) \alpha_i
\end{equation}

As in linear regression, $\mathbf{\alpha}_i$ are our regression weights.

In matrix form:
\begin{equation}
\mathbf{y} = \mathbf{K}\mathbf{\alpha}
\end{equation}

---

For example, Gaussian kernel:
\begin{equation}
\kappa\left(\mathbf{x}, \tilde{\mathbf{x}} \right) = \exp \left(-\frac{\|\mathbf{x} - \tilde{\mathbf{x}} \|^2}{2\sigma^2} \right)
\end{equation}
Alternative definition: (used in the exercises today)
\begin{equation}
\kappa\left(\mathbf{x}, \tilde{\mathbf{x}} \right) = \exp \left(-\gamma \|\mathbf{x} - \tilde{\mathbf{x}} \|^2 \right)
\end{equation}
The Gaussian kernel is always number between 0 and 1:
*   0 if $x$ and $\tilde{x}$ are infinitely apart
*   1 if $x$ and $\tilde{x}$ are the same

---

Fit the best $\alpha$ coefficients:

\begin{equation}
\mathbf{\hat{\alpha}} = \text{arg min} || \mathbf{y}^\text{ref} - \mathbf{K}^\mathrm{train}\mathbf{\alpha}||^2
\end{equation}
Where $\mathbf{y}^\mathrm{ref}$ are the training lables and $\mathbf{K}^\mathrm{train}$ is the pairwise kernel matrix for all the points in the training set, defined as:
\begin{equation}
\mathbf{K}^\mathrm{train}_{ij} = \kappa\left(\mathbf{x}_i, \mathbf{x}_j \right)
\end{equation}

Closed-form solution for regression coefficients:

\begin{equation}
\alpha = \left(\mathbf{K}^\mathrm{train} + \mathbf{I}\lambda\right)^{-1}\mathbf{y}^\text{ref}
\end{equation}

$\lambda$ is a small number to be added to the digonal for numerical reasons (regularization and numerical stability).

First, let's define the kernel function

```
def kernel(xi, xj):

  sigma = 0.3
  k = np.exp(-np.linalg.norm(xi - xj)**2 / (2 * sigma**2))

  return k
```

Next, let's calculate the kernel matrix:

```
K = np.zeros((len(x),len(x)))

for i in range(len(x)):
  for j in range(len(x)):
    K[i,j] = kernel(x[i], x[j])

np.set_printoptions(linewidth=666)
print(K)
```

With the Kernel matrix above, we can now find the regression coefficients:
\begin{equation}
\alpha = \left(\mathbf{K}^\mathrm{train} + \mathbf{I}\lambda\right)^{-1}\mathbf{y}^\text{ref}
\end{equation}

```
alphas = np.matmul(np.linalg.inv(K + np.eye(11)*1e-10), y)
print(alphas)
print(len(alphas))
```

Another way to solve the equation is via a so-called Cholesky decomposition which is more numerically stable:

```
import qml.math

# Add to diagonal
K[np.diag_indices_from(K)] += 1e-10

# Solve
alphas = qml.math.cho_solve(K, y)

print(alphas)

```

### Predictions:
Remembering again:

\begin{equation}
\hat{y}\left(\tilde{\mathbf{x}}\right) = \sum_i \kappa\left(\mathbf{x}_i, \tilde{\mathbf{x}} \right) \alpha_i
\end{equation}
In matrix form:
\begin{equation}
\mathbf{y} = \mathbf{K}\mathbf{\alpha}
\end{equation}

```
# Test points
x_test = np.random.random(size=(20,1)) * 6
# x_test = np.arange(-5, 11.5, 0.5)

# Zero kernel
K_test = np.zeros((len(x_test),len(x)))

# Calculate pair-wise Gaussian kernel function
for i in range(len(x_test)):
  for j in range(len(x)):
    K_test[i,j] = kernel(x_test[i],  x[j])

# Make prediction
y_test = np.dot(K_test, alphas)

# Plot everything
plt.scatter(x,y, label="Training")
plt.plot(xplot, np.sin(xplot), color="g", label="True curve")
plt.scatter(x_test, y_test, color="r", label="Test")
plt.grid(True)
plt.legend()
plt.show()

```

## Note on Hyperparameters!

## Everything repeated, but with Scikit-learn:

Everything we've just coded is of course all available in Scikit-learn:

https://scikit-learn.org/stable/modules/generated/sklearn.kernel_ridge.KernelRidge.html

```
from sklearn.kernel_ridge import KernelRidge

# Gamma instead of sigma!
gam = 1 / (2.0 * sigma**2)

# Make the KRR object
krr = KernelRidge(alpha=1e-10, kernel="rbf", gamma=gam)
# krr = KernelRidge(alpha=1e-10, kernel="linear")

# Fit the machine
krr.fit(x_reshape, y)

# Prediction
y_scikit = krr.predict(x_test)

# Plot everything
plt.scatter(x,y, label="Training")
plt.plot(xgauss, np.sin(xgauss), color="g", label="True curve")
plt.scatter(x_test, y_scikit, color="r", label="Scikit-learn")

plt.legend()
plt.grid(True)
plt.show()
```

```

```

What about hyper parameters?

## Part 2: Molecules in and Machine Learning?

### Molecule data files

```
# Download Coordinate file for ethanol
!wget -O ethanol.xyz https://raw.githubusercontent.com/andersx/ml-intro/master/ethanol.xyz
```

Molecules are often stored in the ".xyz" format. For example, a conformation for ethanol is stored in the above file:

```
9

C         -0.69924       -0.01497        0.32426
C         -1.90198       -0.93192        0.56363
O          0.50350       -0.72657        0.14397
H          0.30059       -1.64320       -0.17552
H         -2.08482       -1.56542       -0.33061
H         -2.80809       -0.31867        0.75417
H         -1.71524       -1.58372        1.44281
H         -0.89079        0.62463       -0.56368
H         -0.58049        0.63584        1.21559
```

Such files can for example be visualized with ASE (Atomic Simulation Environment).
https://wiki.fysik.dtu.dk/ase/

```
!pip install ase
```

```
import ase.io
import ase.visualize

ethanol = ase.io.read("ethanol.xyz")
ase.visualize.view(ethanol, viewer="x3d")
```

# Making representations for molecules?

We need to calculate the pair wise kernels between molecules:
\begin{equation}
\kappa\left(\mathbf{x}, \tilde{\mathbf{x}} \right) = \exp \left(-\frac{\|\mathbf{x} - \tilde{\mathbf{x}} \|^2}{2\sigma^2} \right)
\end{equation}

Also sometimes called the *similarity kernel*: 1 if the two molecules are identical, 0 if they are completely different.

Problem: how do we get an input representation? Which features are good?

Anything could in principle be used:
*   Numbers of atoms
*   Physical observables
*   Name string (e.g. SMILES)
*   Coordinates
*   Nuclear charges
*   Bonding information
*   Etc ...

Some desireable properties:
*   Rotational, translations invariance?
*   Permutational invariance
*   Uniqueness (injectivity)

This kind of problem in machine learning is called "*feature engineering*".

There are *many* representations available for molecules. Two examples are follow, which will also be used in the exercies for today.

```
!pip install qml
```

### The Coulomb Matrix:
Proposed by Rupp et al. (2012) Phys Rev Lett https://doi.org/10.1103/PhysRevLett.108.058301

\begin{equation}
x_{ij}^\text{CM} =
  \begin{cases}
    0.5Z_i^{2.4} & \text{for } i=j \\
    \frac{Z_i Z_j}{R_{ij}} & \text{for } i \neq j
  \end{cases}
\end{equation}

The above takes care of translational and permutational invariance.

Furthermore, the rows and columns are sorted to ensure permutational invariance.

Example for ethane:

```
73.51669472
34.06515641 36.8581052
19.58838249 23.5104435  36.8581052
 8.06700497  3.03799493  2.46941255  0.5
 3.91313164  5.40535191  2.78865961  0.35565997  0.5
 3.87121997  5.40076757  2.76284181  0.38595636  0.55366084  0.5
 2.9519494   2.75461693  5.40414993  0.38673507  0.39949749  0.32304269  0.5
 2.89651793  2.75224092  5.4003281   0.41811064  0.32445499  0.39915951  0.55199418  0.5
 2.35852226  2.76046537  5.40252013  0.28533493  0.40534695  0.39832789  0.5530945   0.55433763  0.5
```

Example of how to generate with QML:

```
import qml
mol = qml.Compound(xyz="ethanol.xyz")
mol.generate_coulomb_matrix(size=12)

np.set_printoptions(linewidth=100)
print(mol.representation)
```

### Bag-of-Bonds:
Proposed by Hansen et al. (2015) J Phys Chem Lett https://doi.org/10.1021/acs.jpclett.5b00831

Same content as the Coulomb Matrix, but items are grouped differently, so only identical terms will be compared.

More accurate for molecules? (Exercise of today)

```
mol.generate_bob(asize={"O":2, "C": 3, "H": 8})
np.set_printoptions(linewidth=100)
print(mol.representation)
```

## Exercises set #2: The QM7 dataset.
Rupp et al. (2012) Phys Rev Lett https://doi.org/10.1103/PhysRevLett.108.058301

The QM7 dataset contains XYZ structures for 7101 small molecules, with up to 7 atoms of type CNO, saturated with H.

Some of them look like this:

Attempt to map all organic molecules with up to 7 CNO-atoms.

For each molecule you will be given the atomization energy calculated using a QM method (PBE0/def2-TZVP).

*   Instead of the raw 7101 XYZ files, you will be given the Coulomb Matrix features and Bag-of-Bonds features for each molecule, along with the atomization energies.

## How to determine which machine learning method is better than the other?

For a given training/test split, one could calculate, for example, the mean-absolute-error (MAE) or root-mean-squared-error (RMSE) of the predictions, compared to the true values.

\begin{equation}
\text{MAE} =\frac{1}{N} \sum_{i=1}^{N}|y_i - \tilde{y}_i|
\end{equation}

\begin{equation}
\text{RMSE} = \sqrt{\frac{1}{N}\sum_{i=1}^{N}\left(y_i - \tilde{y}_i\right)^2}
\end{equation}

```
y_true = np.array([0.00, 1.32, 2.64, 3.96])
y_pred = np.array([0.30, 1.14, 3.18, 2.70])
```

```
diff = y_true-y_pred
print(diff)
```

```
# Mean-absolute-error
mae = np.mean(np.abs(diff))
print(mae)

# Root-mean-squared-error
rmse = np.sqrt(np.mean(diff**2))
print(rmse)
```

## Learning Curves:

What is learning? Able to make better prediction with more training data.

The error decays according to a power law with the training set size:
\begin{equation}
\mathrm{Error} \propto \frac{a}{N^b}
\end{equation}
Which is the same as:
\begin{equation}
\log\left(\mathrm{Error}\right) \propto \log\left(a\right) - b \log\left(N\right)
\end{equation}
On a log-log scale the error is linear with the training set size!

It turns out, for the same dataset, these are most always parallel (same $b$-value).

We can compare machine learning models (for example based on different representations), by looking at the learning curve!

## Training, Test, and Validation Splits:

One common way to split data is as follows:

This is a method to avoid overfitting (i.e. fitting parameters to your "test" data)

Procedure for calculating the error for a model:
1.   Train model on **Training Set**
2.   Minimize error for **Validation Set**, i.e. optimize hyperparameters + goto 1.
3.   Calculate error for **Test Set**

The last error on the Test Set (from 3.) is your error.

Example how to split 100 values into three

*   50 Training Set
*   10 Validation Set
*   40 Test Set

```
data_set = np.arange(0,100)
print(data_set)
```

Remembering the Numpy *slice-notation*:

The general syntax is

`array[row]`

or

`array[row, column]`

Special notation:

    n   = what is in index n
    :n  = up to n
    n:  = n and onwards
    n:m = from n to m
    :   = everything

```
# First 50
training_set = data_set[:50]
print(training_set)
# Next 10
validation_set = data_set[50:60]
print(validation_set)
# Next 40
test_set = data_set[60:100]
print(test_set)
```


```python
import sympy as sp
import numpy as np
```

```python
x = sp.symbols('x')
p = sp.Function('p')
l = sp.Function('l')
poly = sp.Function('poly')
p3 = sp.Function('p3')
p4 = sp.Function('p4')
```

# Introduction

Last time we have used Lagrange basis to interpolate polynomial. However, it is not efficient to update the interpolating polynomial when a new data point is added. We look at an iterative approach.

Given points $\{(z_i, f_i) \}_{i=0}^{n-1}$, $z_i$ are distinct and $p_{n-1} \in \mathbb{C}[z]_{n-1}\, , p_{n-1}(z_i) = f_i$.
 We add a point $(z_n, f_n)$ and find a polynomial $p_n \in \mathbb{C}[x]_{n-1}$ which satisfies $\{(z_i, f_i) \}_{i=0}^{n}$.

We assume $p_n(z)$ be the form
\begin{equation}
p_n(z) = p_{n-1}(z) + C\prod_{i=0}^{n-1}(z - z_i)
\end{equation}
so that the second term vanishes at $z = z_0,...,z_{n-1}$ and $p_n(z_i) = p_{n-1}(z_i), i = 0,...,n-1$. We also want $p_n(z_n) = f_n$ so we have
\begin{equation}
f_n = p_{n-1}(z_n) + C\prod_{i=0}^{n-1}(z_n - z_i) \Rightarrow C = \frac{f_n - p_{n-1}(z_n)}{\prod_{i=0}^{n-1}(z_n - z_i)}
\end{equation}
Thus we may perform interpolation iteratively.

**Example:** Last time we have
\begin{equation}
(z_0, f_0) = (-1,-3), \quad
(z_1, f_1) = (0,-1), \quad
(z_2, f_2) = (2,4), \quad
(z_3, f_3) = (5,1)
\end{equation}
and
\begin{equation}
p_3(x) = \frac{-13}{90}z^3 + \frac{14}{45}z^2 + \frac{221}{90}z - 1
\end{equation}

```python
z0 = -1; f0 = -3; z1 = 0; f1 = -1; z2 = 2; f2 = 4; z3 = 5; f3 = 1; z4 = 1; f4 = 1
p3 = -13*x**3/90 + 14*x**2/45 + 221*x/90 - 1
```

We add a point $(z_4,f_4) = (1,1)$ and obtain $p_4(x)$

```python
z4 = 1; f4 = 1
```

    1 1

```python
C = (f4 - p3.subs(x,z4))/((z4-z0)*(z4-z1)*(z4-z2)*(z4-z3))
C
```

$\displaystyle - \frac{7}{90}$

```python
p4 = p3 + C*(x-z0)*(x-z1)*(x-z2)*(x-z3)
sp.expand(p4)
```

$\displaystyle - \frac{7 x^{4}}{90} + \frac{29 x^{3}}{90} + \frac{7 x^{2}}{90} + \frac{151 x}{90} - 1$

**Remark:** the constant $C$ is usually written as $f[z_0,z_1,z_2,z_3,z_4]$. Moreover by iteration we have
$$p_n(z) = \sum_{i=0}^n f[z_0,...,z_n] \prod_{j=0}^i (z - z_j)$$

# Newton Tableau

We look at efficient ways to compute $f[z_0,...,z_n]$, iteratively from $f[z_0,...,z_{n-1}]$ and $f[z_1,...,z_n]$.

We may first construct $p_{n-1}$ and $q_{n-1}$ before constructing $p_n$ itself, where
\begin{gather}
p_{n-1}(z_i) = f_i \quad i = 0,...,n-1\\
q_{n-1}(z_i) = f_i \quad i = 1,...,n
\end{gather}
**Claim:** The following polynomial interpolate $\{(z_i,f_i)\}_{i=0}^n$
\begin{equation}
p_n(z_i) = \frac{(z - z_n)p_{n-1}(z) - (z - z_0)q_{n-1}(z)}{z_0 - z_n}
\end{equation}
Since interpolating polynomial is unique, by comparing coefficient of $z_n$, we have
$$f[z_0,...,z_{n}] = \frac{f[z_0,...,z_{n-1}]-f[z_1,...,z_{n}]}{z_0 - z_n}$$

```python
def product(xs,key,i):

    #Key: Forward or Backward

    n = len(xs)-1
    l = 1

    for j in range(i):
        if key == 'forward':
            l *= (x - xs[j])
        else:
            l *= (x - xs[n-j])

    return l
```

```python
def newton(xs,ys,key):

    # Key: Forward or Backward

    n = len(xs)-1
    # print(xs)
    print(ys)
    old_column = ys

    if key == 'forward':
        coeff = [fs[0]]
    elif key == 'backward':
        coeff = [fs[len(fs)-1]]
    else:
        return 'error'

    for i in range(1,n+1): # Column Index
        new_column = [(old_column[j+1] - old_column[j])/(xs[j+i] - xs[j]) for j in range(n-i+1)]
        print(new_column)
        if key == 'forward':
            coeff.append(new_column[0])
        else:
            coeff.append(new_column[len(new_column)-1])
        old_column = new_column

    # print(coeff)

    poly = 0
    for i in range(n+1):
        poly += coeff[i] * product(xs,key,i)

    return poly
```

```python
zs = [1, 4/3, 5/3, 2]; fs = [np.sin(x) for x in zs]
```

```python
p = newton(zs,fs,'backward')
sp.simplify(p)
```

    [0.8414709848078965, 0.9719379013633127, 0.9954079577517649, 0.9092974268256817]
    [0.3914007496662487, 0.07041016916535667, -0.25833159277824974]
    [-0.481485870751338, -0.4931126429154095]
    [-0.011626772164071542]

$\displaystyle - 0.0116267721640715 x^{3} - 0.434978782095052 x^{2} + 1.45415019345144 x - 0.166073654384421$

M2AA3/M2AA3-Polynomials/Lesson 02 - Newton Method/Newton Tableau.ipynb


```python
import numpy as np
import matplotlib.pyplot as plt
import sympy
```

# Item VI

A *simple ODE*
* Design a numerical method to approximate the following Boundary Value Problem:
$$
\epsilon y''(x) + (1+\epsilon) y'(x) + y(x) = 0 \qquad \text{for $0<x<1$ and $y(0)=0, y(1)=1$.}
$$
* Plot the numerical approximation against its analytical solution for $\epsilon = 1, 10^{-3}, 10^{-8},$ and $10^{-14}$.
* Is your numerical approximation valid for all $x$?
* Is your approximation accurate?
* What does it happen as $\epsilon \rightarrow 0$?
---

```python
def euler(yp=lambda y,t:1,yi=0,tmin=0,tmax=1.0,steps=400):
    delta = (tmax-tmin)/float(steps)
    t = tmin
    ys = []
    ys.append(yi)
    for i in range(steps):
        t = tmin+delta*i
        yp_next = yp(ys[-1],t)
        y_next = ys[-1]+delta*yp_next
        ys.append(y_next)
    return ys
```

```python
# Let's use shooting method:
def shooting(equ=lambda y,yp,t:1,yi=0,yf=0,
                ypa=0,ypb=1e3,yp_precision=1e-9,
                tmin=0.0,tmax=1.0,steps=400):
    # function for euler's method
    shooteval = lambda y,t: np.array([y[1],equ(y[0],y[1],t)])
    # Check y(ypa) and y(ypb)
    yf_ypa = euler(shooteval,yi=np.array([yi,ypa]),
                    tmin=tmin,tmax=tmax,steps=steps)[-1][0]
    yf_ypb = euler(shooteval,yi=np.array([yi,ypb]),
                    tmin=tmin,tmax=tmax,steps=steps)[-1][0]
    # Find y'(t0) through bisection
    while np.abs(ypb-ypa)>yp_precision:
        # Couldn't find root
        if np.isnan(yf_ypa) or np.isnan(yf_ypb) or (yf_ypa-yf)*(yf_ypb-yf)>0:
            return None
        # print("y'_a(0)=%f y'_b(0)=%f"%(ypa,ypb))
        ypc = (ypa+ypb)/2.0
        ys = euler(shooteval,yi=np.array([yi,ypc]),
                    tmin=tmin,tmax=tmax,steps=steps)
        yf_ypc = ys[-1][0]
        if (yf_ypc-yf)==0:
            return ys
        elif (yf_ypa-yf)*(yf_ypc-yf)<0:
            yf_ypb = yf_ypc
            ypb = ypc
        else:
            yf_ypa = yf_ypc
            ypa = ypc
    return ys
```

```python
def problem_ode(e):
    fn = lambda y,yp,t: -(1.0+e)/e*yp-1.0/e*y
    return fn
```

```python
STEPS = 400
E = [1,0.2,0.1,0.01,1e-3,1e-8,1e-14]

xs = np.linspace(0,1,num=STEPS+1)

yss_numerical = {}
for e in E:
    print("Computing epsilon=%s."%e)
    ys = shooting(problem_ode(e),yi=0,yf=1,ypa=1.0,ypb=50.0,steps=STEPS)
    if ys is not None:
        yss_numerical[str(e)] = [y[0] for y in ys]
    else:
        print("  Couldn't find y'(0) for epsilon=%s"%e)
```

    Computing epsilon=1.
    Computing epsilon=0.2.
    Computing epsilon=0.1.
    Computing epsilon=0.01.
      Couldn't find y'(0) for epsilon=0.01
    Computing epsilon=0.001.
      Couldn't find y'(0) for epsilon=0.001
    Computing epsilon=1e-08.
      Couldn't find y'(0) for epsilon=1e-08
    Computing epsilon=1e-14.
      Couldn't find y'(0) for epsilon=1e-14

    /home/fcasas/Music/p36cpu/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in double_scalars

    /home/fcasas/Music/p36cpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: RuntimeWarning: invalid value encountered in add
      if __name__ == '__main__':
    /home/fcasas/Music/p36cpu/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in double_scalars

With the coded method the solution couldn't be found once $\epsilon$ became small enough.

```python
for name,ys in yss_numerical.items():
    plt.plot(xs,ys,label="$y(t|e=%s)$"%name)
plt.legend()
plt.ylim((0,2))
plt.grid()
plt.show()
```

We find the analytical solution now:

```python
def get_analytical_solution(ee,xs,n=200):
    x = sympy.Symbol("x")
    y = sympy.Function("y")(x)
    y_ = sympy.Derivative(y,x)
    y__ = sympy.Derivative(y_,x)
    sol = sympy.dsolve(
        ee * y__ + (1+ee) * y_ + y, y, ics={y.subs(x,0):0,y.subs(x,1):1})
    sol = sol.rhs
    return [float(sol.subs(x,xx)) for xx in np.linspace(0,1,num=n+1)]
```

```python
yss_analytical = {}
for e in E:
    if e<1e-6: continue # Smaller values take forever.
    print("computing for epsilon=%s"%e)
    ys = get_analytical_solution(e,xs,n=STEPS)
    yss_analytical[str(e)] = ys
    plt.plot(xs,ys,label="analytic $y$ for $\epsilon=%s$"%e)
plt.legend()
plt.ylim((0,3))
plt.grid()
plt.show()
```

```python
# Plot the analytic solution and the numerical one for the cases when we have both.
for ee_name in yss_numerical:
    if ee_name in yss_analytical:
        plt.plot(xs,yss_numerical[ee_name],label="numerical $y$ for $\epsilon=%s$"%ee_name)
        plt.plot(xs,yss_analytical[ee_name],label="analytic $y$ for $\epsilon=%s$"%ee_name)
        plt.legend()
        plt.ylim((0,3))
        plt.grid()
        plt.show()
```

The approximation seems almost identical for all $0<x<1$ for the $\epsilon$'s for which the problem could be solved. However, picking *shooting method* and bisection probably was a poor choice, since as $\epsilon$ gets smaller, $y'(0)$ (which must be found) becomes larger ($\epsilon \rightarrow 0 \Rightarrow y'(0) \rightarrow \infty$) very fast, so it's value becomes harder to find.

```python
# Measure the maximum error between the numerical and analytical solutions:
for ee_name in yss_numerical:
    if ee_name in yss_analytical:
        max_error = np.max(np.abs(np.array(yss_numerical[ee_name])-yss_analytical[ee_name]))
        print("max error for epsilon=%5s: %f"%(ee_name,max_error))
```

    max error for epsilon=    1: 0.000549
    max error for epsilon=  0.2: 0.006706
    max error for epsilon=  0.1: 0.014160

It can be see that the error isn't larger than $0.015$ for the $\epsilon$ for which the problem could be solved with the method used.

<!-- Go for the perturbation approach -->

```python

```


# HW #2
## FINM 37600 | Autumn 2021 | Professor Mark Hendricks
_**Due:** Monday, October 11, at 6:00pm\
**Group A6:** Yanhao Dong, Sarah Huang, Maneet Singh, Ashley Tsoi, Xing Zheng, Harry Zou_

## 2 Analyzing the Data

Use the data found on Canvas, in ‘proshares analysis data.xlsx’. It has monthly data on financial indexes and ETFs from Aug 2011 through Sep 2021.

```python
# import the packages we need

import pandas as pd
pd.options.display.float_format = '{:.3f}'.format
import numpy as np
# import scipy.stats as st
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.api as sm
```

```python
# read in each tab of the excel sheet
desc = pd.read_excel('../data/proshares_analysis_data.xlsx','descriptions').rename(columns={'Unnamed: 0':'Symbol'}).set_index('Symbol')
hf = pd.read_excel('../data/proshares_analysis_data.xlsx','hedge_fund_series').set_index('date')
mf = pd.read_excel('../data/proshares_analysis_data.xlsx','merrill_factors').set_index('date')
od = pd.read_excel('../data/proshares_analysis_data.xlsx','other_data').set_index('date')
```

```python
### Function for performance meterics

def getPerformanceMetrics(data):

    data_desc = data.describe().loc[['mean','std']]

    data_desc.loc['mean'] = data_desc.loc['mean']*12 # annualize
    data_desc.loc['vol'] = data_desc.loc['std']*np.sqrt(12) # annualize
    data_desc.loc['sharpe_ratio'] = data_desc.loc['mean'] / data_desc.loc['vol']

    data_desc.loc['skewness'] = data.skew()
    data_desc.loc['excess_kurtosis'] = data.kurt()-3
    data_desc.loc['VaR05'] = np.quantile(data,.05,axis=0)
    data_desc.loc['CVaR05'] = np.mean(data<=data_desc.loc['VaR05'])

    def getMaxDDInfo(data):
        cumRet = (1+data).cumprod()
        cumMax = cumRet.cummax()
        dd = (cumRet/cumMax)-1
        # print(dd.min())

        maxDD = pd.DataFrame([tuple(dd.min())],
                            columns=data.columns,
                            index=['maxDrawdown'])

        maxDD_dt = []
        # loop through each column in dd (drawdown DataFrame)
        for col,val in dd.iteritems():
            mdd = maxDD[col]['maxDrawdown']

            troughDate = val[val==mdd].index[0].strftime("%Y-%m-%d")
            peakDate = val[:troughDate].iloc[::-1].idxmax().strftime("%Y-%m-%d")
            recoveryDate = val[troughDate:].idxmax().strftime("%Y-%m-%d")

            maxDD_dt.append([peakDate,troughDate,recoveryDate])

        maxDD_dt = pd.DataFrame(list(zip(*maxDD_dt)), #transpose: [[1,2],[3,4]] -> [[1,3],[2,4]]
                                columns=data.columns,
                                index=['maxDrawdown_peakDate','maxDrawdown_troughDate','maxDrawdown_recoveryDate'])

        return pd.concat([maxDD,maxDD_dt])

    return pd.concat([data_desc,getMaxDDInfo(data)])

def regressionAnalysis(exog,endog_df,intercept=False,prediction=False,displaySummary=[],displayRatios=False):
    if isinstance(exog,pd.Series):
        exog = pd.DataFrame(exog)
    if isinstance(endog_df,pd.Series):
        endog_df = pd.DataFrame(endog_df)
    if displaySummary==True:
        displaySummary = endog_df.columns

    if intercept:
        exog = sm.tools.add_constant(exog)

    ratios = []
    # loop through each column in data (y-variable DataFrame)
    for col,endog in pd.DataFrame(endog_df).iteritems():
        model = sm.regression.linear_model.OLS(endog,exog).fit()

        fitted_val = model.predict(exog).mean()
        betas = []
        treynor_ratios = []

        if intercept:
            exogCols = exog.columns[1:]
        else:
            exogCols = exog.columns
        for x in exogCols:
            betas.append('{:.3f}'.format(model.params[x]))
            treynor_ratios.append('{:.3f}'.format((endog.mean()/model.params[x])*12))
        if len(betas)==1:
            betas = betas[0]
            treynor_ratios = treynor_ratios[0]
        if intercept:
            alpha = model.params['const']
            info_ratio = (alpha/model.resid.std())*np.sqrt(12)
        else:
            alpha = None
            info_ratio = None
        r_squared = model.rsquared
        tracking_err = model.resid.std()
        ratios.append([fitted_val,alpha,info_ratio,betas,treynor_ratios,r_squared,tracking_err])

        if col in displaySummary:
            print(model.summary())

    regRatios = pd.DataFrame(list(zip(*ratios)),
                             columns=endog_df.columns,
                             index=['avg_fitted_val',
                                    'alpha','information_ratio',
                                    'beta','treynor_ratio',
                                    'r_squared','tracking_error'])

    if prediction:
        return model.predict(exog)

    if displayRatios:
        return regRatios

# regressionAnalysis(mf,hf,intercept=True,displayRatios=True)
```

### 1. For the series in the “hedge fund series” tab, report the following summary statistics (annualize these statistics):
#### (a) mean
#### (b) volatility
#### (c) Sharpe ratio

```python
getPerformanceMetrics(hf).loc[['mean','vol','sharpe_ratio']]
```

      HFRIFWI Index
      MLEIFCTR Index
      MLEIFCTX Index
      HDG US Equity
      QAI US Equity

      mean
      0.051
      0.039
      0.037
      0.028
      0.025

      vol
      0.062
      0.054
      0.054
      0.056
      0.045

      sharpe_ratio
      0.826
      0.721
      0.695
      0.498
      0.560

### 2. For the series in the “hedge fund series” tab, calculate the following statistics related to tail- risk (no need to annualize any of these statistics).
#### (a) Skewness
#### (b) Excess Kurtosis (in excess of 3)
#### (c) VaR (.05) - the fifth quantile of historic returns
#### (d) CVaR (.05) - the mean of the returns at or below the fifth quantile
#### (e) Maximum drawdown - include the dates of the max/min/recovery within the max drawdown period.

```python
getPerformanceMetrics(hf).loc[['skewness','excess_kurtosis',
                               'VaR05','CVaR05',
                               'maxDrawdown','maxDrawdown_peakDate','maxDrawdown_troughDate','maxDrawdown_recoveryDate']]

```

      HFRIFWI Index
      MLEIFCTR Index
      MLEIFCTX Index
      HDG US Equity
      QAI US Equity

      skewness
      -1.109
      -0.187
      -0.172
      -0.180
      -0.545

      excess_kurtosis
      3.730
      -0.571
      -0.624
      -0.418
      -0.330

      VaR05
      -0.025
      -0.026
      -0.026
      -0.029
      -0.016

      CVaR05
      0.057
      0.057
      0.057
      0.057
      0.057

      maxDrawdown
      -0.115
      -0.084
      -0.084
      -0.088
      -0.076

      maxDrawdown_peakDate
      2019-12-31
      2019-12-31
      2019-12-31
      2020-01-31
      2019-12-31

      maxDrawdown_troughDate
      2020-03-31
      2020-03-31
      2020-03-31
      2020-03-31
      2020-03-31

      maxDrawdown_recoveryDate
      2020-08-31
      2020-11-30
      2020-11-30
      2020-11-30
      2020-07-31

### 3. For the series in the “hedge fund series” tab, run a regression of each against SPY (found in the “merrill factors” tab.) Include an intercept. Report the following regression-based statistics (annualize as necessary):
#### (a) Market Beta
#### (b) Treynor Ratio
#### (c) Information ratio

```python
spy_reg = regressionAnalysis(mf['SPY US Equity'],hf,intercept=True,displayRatios=True)
spy_reg.loc[['alpha','information_ratio','beta','treynor_ratio','r_squared']]
```

      HFRIFWI Index
      MLEIFCTR Index
      MLEIFCTX Index
      HDG US Equity
      QAI US Equity

      alpha
      -0.001
      -0.001
      -0.001
      -0.002
      -0.001

      information_ratio
      -0.242
      -0.615
      -0.671
      -1.011
      -0.687

      beta
      0.394
      0.359
      0.358
      0.369
      0.285

      treynor_ratio
      0.129
      0.108
      0.104
      0.076
      0.089

      r_squared
      0.753
      0.816
      0.815
      0.786
      0.719

### 4. Relative Performance
Discuss the previous statistics, and what they tell us about...
#### (a) the differences between SPY and the hedge-fund series?
#### (b) which performs better between HDG and QAI.
#### (c) whether HDG and the ML series capture the most notable properties of HFRI.

(a) All indices's SPY/market beta are near the 0.3-0.4 range, which means they all have a fairly low exposure to the market. Also, since the Treynor ratios of all indices are near 0.1,

(b) HDG has a lower alpha and information ratio, while QAI has a lower beta. However, when beta is adjusted with the beta risk (Treynor ratio), the data shows that HDG has a lower performance. Overall, **QAI performs better than HDG**.

(c) HDF and the ML series all track HFRI pretty closely (and capture HFRI's properties). However, the ML series track HFRI better than HDF.

### 5. Report the correlation matrix for these assets.
#### (a) Show the correlations as a heat map.

```python
cor_table = hf.corr()
sns.heatmap(cor_table)
```

#### (b) Which series have the highest and lowest correlations?

```python
cor_table[cor_table == 1] = None
cor_table = cor_table.unstack().dropna()
max_cor = cor_table.max()
min_cor = cor_table.min()

print('Strongest correlation is between: '+ str(cor_table[cor_table==max_cor].index[0]))
print('Weakest correlation is between: '+ str(cor_table[cor_table==min_cor].index[0]))
```

    Strongest correlation is between: ('MLEIFCTR Index', 'MLEIFCTX Index')
    Weakest correlation is between: ('HDG US Equity', 'QAI US Equity')

### 6. Replicate HFRI with the six factors listed on the “merrill factors” tab. Include a constant, and run the unrestricted regression,
$
\begin{align}
r^{hfri}_t &= \alpha^{merr} + x^{merr}_t\beta^{merr} + \epsilon^{merr}_t \\
\hat{r}^{hfri}_t &\equiv \hat{\alpha}^{merr} + x^{merr}_t\hat{\beta}^{merr}
\end{align}
$
### Note that the second equation is just our notation for the fitted replication.
#### (a) Report the intercept and betas.
#### (b) Are the betas realistic position sizes, or do they require huge long-short positions?
#### (c) Report the R-squared.
#### (d) Report the volatility of $\epsilon^{merr}$ (the tracking error).

```python
fullSample_wIntercept = regressionAnalysis(mf,hf['HFRIFWI Index'],intercept=True,displayRatios=True)
fullSample_wIntercept.loc[['alpha','beta','r_squared','tracking_error']]
```

      HFRIFWI Index

      alpha
      0.001

      beta
      [0.072, -0.401, 0.072, 0.106, 0.022, 0.131]

      r_squared
      0.856

      tracking_error
      0.007

### 7. Let’s examine the replication out-of-sample.
### Starting with $t = 61$ month of the sample, do the following:
- Use the previous 60 months of data to estimate the regression equation, (1). This gives time-$t$ estimates of the regression parameters, $\tilde{\alpha}^{merr}_t$ and $\beta^{merr}_t$.
- Use the estimated regression parameters, along with the time-$t$ regressor values, $x^{merr}_t$, to calculate the time-$t$ replication value that is, with respect to the regression estimate, built out-of-sample” (OOS).

$
\tilde{r}^{hfri}_t \equiv \tilde{\alpha}^{merr}_t+(x^{merr}_t)^{\prime}\tilde{\beta}^{merr}_t
$

- Step forward to $t = 62$, and now use $t = 2$ through $t = 61$ for the estimation. Re-run the steps above, and continue this process throughout the data series. Thus, we are running a rolling, 60-month regression for each point-in-time.
### How well does the out-of-sample replication perform with respect to the target?

```python
date_range = mf['2016-8-31':].index
oos_fitted = pd.Series(index=date_range, name='OOS_fit')
for date in date_range:
    date_month_prior = pd.DatetimeIndex([date]).shift(periods=-60, freq='M')[0]
    df_subset = mf[date_month_prior:date]
    x = sm.add_constant(df_subset)
    y = hf['HFRIFWI Index'].loc[date_month_prior:date]
    res = sm.OLS(y, x).fit()
    alpha = res.params['const']
    beta = res.params.drop(index='const')
    x_t = mf.loc[date]
    predicted_next_value = alpha + x_t @ beta
    oos_fitted[date] = predicted_next_value

oos_fitted.plot(figsize=(14,3))
hf['HFRIFWI Index'].loc['2016-8-31':].plot()
plt.legend()
plt.show()

(oos_fitted + 1).cumprod().plot(figsize=(14,3))
(hf.loc['2016-8-31':,'HFRIFWI Index'] + 1).cumprod().plot()
plt.legend()
plt.show()

display((pd.DataFrame([oos_fitted, hf.loc['2016-8-31':,'HFRIFWI Index']])).T.corr())
```

### 8. We estimated the replications using an intercept. Try the full-sample estimation, but this time without an intercept.
$
\begin{align}
r^{hfri}_t &= x^{merr}_t\beta^{merr} + \epsilon^{merr}_t \\
\check{r}^{hfri}_t &\equiv x^{merr}_t\check{\beta}^{merr}
\end{align}
$

```python
fullSample_noIntercept = regressionAnalysis(mf,hf['HFRIFWI Index'],intercept=False,displayRatios=True)
fullSample_noIntercept
```

      HFRIFWI Index

      avg_fitted_val
      0.004

      alpha
      None

      information_ratio
      None

      beta
      [0.088, 0.335, 0.070, 0.102, 0.023, 0.129]

      treynor_ratio
      [0.580, 0.152, 0.726, 0.500, 2.163, 0.394]

      r_squared
      0.861

      tracking_error
      0.007

```python
beta_diff = []
for b1,b2 in zip(fullSample_wIntercept['HFRIFWI Index'].loc['beta'],fullSample_noIntercept['HFRIFWI Index'].loc['beta']):
  beta_diff.append(float(b1)-float(b2))

pd.DataFrame(beta_diff,columns=['HFRIFWI Index'],index=mf.columns)
```

      HFRIFWI Index

      SPY US Equity
      -0.016

      USGG3M Index
      -0.736

      EEM US Equity
      0.002

      EFA US Equity
      0.004

      EUO US Equity
      -0.001

      IWM US Equity
      0.002

#### (b) the mean of the fitted value, $\check{r}^{hfri}_t$. How does it compare to the mean of the HFRI?

```python
r_check = regressionAnalysis(mf,hf['HFRIFWI Index'],prediction=True,intercept=False)
r_check.mean() - hf['HFRIFWI Index'].mean()
```

    -0.0006580004951943197

#### (c) the correlations of the fitted values, $\check{r}^{hfri}_t$ to the HFRI. How does the correlation compare to $\hat{r}^{hfri}_t$?

```python
r_check = regressionAnalysis(mf,hf['HFRIFWI Index'],prediction=True,intercept=False)
r_hat = regressionAnalysis(mf,hf['HFRIFWI Index'],prediction=True,intercept=True)
cor_r_check_hfri = np.corrcoef(r_check,hf['HFRIFWI Index'])[0][1]
cor_r_hat_hfri = np.corrcoef(r_hat,hf['HFRIFWI Index'])[0][1]

print('correlation between r_check & hfri: {}; \ncorrelation between r_hat & hfri: {}; \ndifference: {}'.format(cor_r_check_hfri,cor_r_hat_hfri,cor_r_check_hfri-cor_r_hat_hfri))
```

    correlation between r_check & hfri: 0.9245168418715026;
    correlation between r_hat & hfri: 0.9250377140204875;
    difference: -0.0005208721489848855

### Do you think Merrill and ProShares fit their replicators with an intercept or not?

Merrill and ProShares fit their replicators **with an intercept**, as the reported returns match the regressed data for the model with an intercept.


Non Linear Regression Analysis

If the data shows a curvy trend, then linear regression will not produce very accurate results when compared to a non-linear regression because, as the name implies, linear regression presumes that the data is linear.
Let's learn about non linear regressions and apply an example on python. In this notebook, we fit a non-linear model to the datapoints corrensponding to China's GDP from 1960 to 2014.

Importing required libraries

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

Though Linear regression is very good to solve many problems, it cannot be used for all datasets. First recall how linear regression, could model a dataset. It models a linear relation between a dependent variable y and independent variable x. It had a simple equation, of degree 1, for example y = $2x$ + 3.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 2*(x) + 3
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
#plt.figure(figsize=(8,6))
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

Non-linear regressions are a relationship between independent variables $x$ and a dependent variable $y$ which result in a non-linear function modeled data. Essentially any relationship that is not linear can be termed as non-linear, and is usually represented by the polynomial of $k$ degrees (maximum power of $x$).

$$ \ y = a x^3 + b x^2 + c x + d \ $$

Non-linear functions can have elements like exponentials, logarithms, fractions, and others. For example: $$ y = \log(x)$$

Or even, more complicated such as :
$$ y = \log(a x^3 + b x^2 + c x + d)$$

Let's take a look at a cubic function's graph.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 1*(x**3) + 1*(x**2) + 1*x + 3
y_noise = 20 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

As you can see, this function has $x^3$ and $x^2$ as independent variables. Also, the graphic of this function is not a straight line over the 2D plane. So this is a non-linear function.

Some other types of non-linear functions are:

### Quadratic

$$ Y = X^2 $$

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

y = np.power(x,2)
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Exponential

An exponential function with base c is defined by $$ Y = a + b c^X$$ where b ≠0, c > 0 , c ≠1, and x is any real number. The base, c, is constant and the exponent, x, is a variable.

```python
X = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

Y= np.exp(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Logarithmic

The response $y$ is a results of applying logarithmic map from input $x$'s to output variable $y$. It is one of the simplest form of __log()__: i.e. $$ y = \log(x)$$

Please consider that instead of $x$, we can use $X$, which can be polynomial representation of the $x$'s. In general form it would be written as
\begin{equation}
y = \log(X)
\end{equation}

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = np.log(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Sigmoidal/Logistic

$$ Y = a + \frac{b}{1+ c^{(X-d)}}$$

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = 1-4/(1+np.power(3, X-2))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

# Non-Linear Regression example

For an example, we're going to try and fit a non-linear model to the datapoints corresponding to China's GDP from 1960 to 2014. We download a dataset with two columns, the first, a year between 1960 and 2014, the second, China's corresponding annual gross domestic income in US dollars for that year.

```python
import numpy as np
import pandas as pd

#downloading dataset
!wget -nv -O china_gdp.csv https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv

df = pd.read_csv("china_gdp.csv")
df.head(10)
```

    2019-05-15 19:44:07 URL:https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv [1218/1218] -> "china_gdp.csv" [1]

      Year
      Value

      0
      1960
      5.918412e+10

      1
      1961
      4.955705e+10

      2
      1962
      4.668518e+10

      3
      1963
      5.009730e+10

      4
      1964
      5.906225e+10

      5
      1965
      6.970915e+10

      6
      1966
      7.587943e+10

      7
      1967
      7.205703e+10

      8
      1968
      6.999350e+10

      9
      1969
      7.871882e+10

__Did you know?__ When it comes to Machine Learning, you will likely be working with large datasets. As a business, where can you host your data? IBM is offering a unique opportunity for businesses, with 10 Tb of IBM Cloud Object Storage: [Sign up now for free](http://cocl.us/ML0101EN-IBM-Offer-CC)

### Plotting the Dataset ###
This is what the datapoints look like. It kind of looks like an either logistic or exponential function. The growth starts off slow, then from 2005 on forward, the growth is very significant. And finally, it decelerate slightly in the 2010s.

```python
plt.figure(figsize=(8,5))
x_data, y_data = (df["Year"].values, df["Value"].values)
plt.plot(x_data, y_data, 'ro')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

### Choosing a model ###

From an initial look at the plot, we determine that the logistic function could be a good approximation,
since it has the property of starting with a slow growth, increasing growth in the middle, and then decreasing again at the end; as illustrated below:

```python
X = np.arange(-5.0, 5.0, 0.1)
Y = 1.0 / (1.0 + np.exp(-X))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

The formula for the logistic function is the following:

$$ \hat{Y} = \frac1{1+e^{\beta_1(X-\beta_2)}}$$

$\beta_1$: Controls the curve's steepness,

$\beta_2$: Slides the curve on the x-axis.

### Building The Model ###
Now, let's build our regression model and initialize its parameters.

```python
def sigmoid(x, Beta_1, Beta_2):
     y = 1 / (1 + np.exp(-Beta_1*(x-Beta_2)))
     return y
```

Lets look at a sample sigmoid line that might fit with the data:

```python
beta_1 = 0.10
beta_2 = 1990.0

#logistic function
Y_pred = sigmoid(x_data, beta_1 , beta_2)

#plot initial prediction against datapoints
plt.plot(x_data, Y_pred*15000000000000.)
plt.plot(x_data, y_data, 'ro')
```

Our task here is to find the best parameters for our model. Lets first normalize our x and y:

```python
# Lets normalize our data
xdata =x_data/max(x_data)
ydata =y_data/max(y_data)
```

#### How we find the best parameters for our fit line?
we can use __curve_fit__ which uses non-linear least squares to fit our sigmoid function, to data. Optimal values for the parameters so that the sum of the squared residuals of sigmoid(xdata, *popt) - ydata is minimized.

popt are our optimized parameters.

```python
from scipy.optimize import curve_fit
popt, pcov = curve_fit(sigmoid, xdata, ydata)
#print the final parameters
print(" beta_1 = %f, beta_2 = %f" % (popt[0], popt[1]))
```

     beta_1 = 690.453017, beta_2 = 0.997207

Now we plot our resulting regression model.

```python
x = np.linspace(1960, 2015, 55)
x = x/max(x)
plt.figure(figsize=(8,5))
y = sigmoid(x, *popt)
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(x,y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

## Practice
Can you calculate what is the accuracy of our model?

```python
# split data into train/test
msk = np.random.rand(len(df)) < 0.8
train_x = xdata[msk]
test_x = xdata[~msk]
train_y = ydata[msk]
test_y = ydata[~msk]

# build the model using train set
popt, pcov = curve_fit(sigmoid, train_x, train_y)

# predict using test set
y_hat = sigmoid(test_x, *popt)

# evaluation
print("Mean absolute error: %.2f" % np.mean(np.absolute(y_hat - test_y)))
print("Residual sum of squares (MSE): %.2f" % np.mean((y_hat - test_y) ** 2))
from sklearn.metrics import r2_score
print("R2-score: %.2f" % r2_score(y_hat , test_y) )

```

    /home/jupyterlab/conda/lib/python3.6/site-packages/scipy/optimize/minpack.py:794: OptimizeWarning: Covariance of the parameters could not be estimated
      category=OptimizeWarning)

    Mean absolute error: 0.22
    Residual sum of squares (MSE): 0.13
    R2-score: -14933193976161055932927705088.00

Double-click __here__ for the solution.

Want to learn more?

IBM SPSS Modeler is a comprehensive analytics platform that has many machine learning algorithms. It has been designed to bring predictive intelligence to decisions made by individuals, by groups, by systems – by your enterprise as a whole. A free trial is available through this course, available here: SPSS Modeler

Also, you can use Watson Studio to run these notebooks faster with bigger datasets. Watson Studio is IBM's leading cloud solution for data scientists, built by data scientists. With Jupyter notebooks, RStudio, Apache Spark and popular libraries pre-packaged in the cloud, Watson Studio enables data scientists to collaborate on their projects without having to install anything. Join the fast-growing community of Watson Studio users today with a free account at Watson Studio

Thanks for completing this lesson!

Author:  Saeed Aghabozorgi

Saeed Aghabozorgi, PhD is a Data Scientist in IBM with a track record of developing enterprise level applications that substantially increases clients’ ability to turn data into actionable knowledge. He is a researcher in data mining field and expert in developing advanced analytic methods like machine learning and statistical modelling on large datasets.

Copyright © 2018 Cognitive Class. This notebook and its source code are released under the terms of the MIT License.

```python

```

```python

```


# Transition Path Theory applied to triplewell dynamics

Based on:
Extending Transition Path Theory: Periodically-Driven and Finite-Time Dynamics. [arXiv preprint arXiv:2002.07474.](https://arxiv.org/pdf/2002.07474.pdf)

```python
from pytpt import stationary
from pytpt import periodic
from pytpt import finite
from plotting import plot_3well, \
                     plot_3well_effcurrent, \
                     plot_3well_potential, \
                     plot_3well_vector_field
import numpy as np
import os.path
```

```python
# define directories path to save the data and figures
my_path = os.path.abspath('')
data_path = os.path.join(my_path, 'data')
file_path = None
```

# Stationary, infinite-time process

given by $dX_t = - \nabla V(X_t) dt + \sigma dW_t$, below we show the energy landscape $V(x,y)$

```python
#triple well in 2D energy landscape V and gradient dV
factor = 0.25
V = lambda x, y: 1 * factor*(3*np.exp(-x**2-(y-(1./3))**2) \
                - 3*np.exp(-x**2-(y-(5./3))**2) - 5*np.exp(-(x-1)**2-y**2) \
                - 5*np.exp(-(x+1)**2-y**2)  + 0.2*(x**4) + 0.2*(y-1./3)**4)

# plot potential and gradient
title = 'Triple well Potential'
subtitles=[
    r'$V(x, y)$',
]
plot_3well_potential(
    potential=V,
    title=title,
    file_path=None,
    subtitles=subtitles,
)
```

To apply TPT to these dynamics, we estimated a transition matrix $T$ on a discrete grid on $[-2,2]\times[-1,2]$.
As sets A and B we chose the two deep wells of the energy landscape.

```python
# load triple well data
triplewell_construction = np.load(
    os.path.join(data_path, 'triplewell_construction.npz'),
    allow_pickle=True,
)

interval = triplewell_construction['interval']
dx = triplewell_construction['dx']
ind_A = triplewell_construction['ind_A']
ind_B = triplewell_construction['ind_B']
ind_C = triplewell_construction['ind_C']
T = triplewell_construction['T']
T_m = triplewell_construction['T_m']

#state space
x = np.arange(interval[0,0], interval[0,1] + dx, dx) #box centers in x and y direction
y = np.arange(interval[1,0], interval[1,1] + dx, dx)
xv, yv = np.meshgrid(x, y)

xdim = np.shape(xv)[0] #discrete dimension in x and y direction
ydim = np.shape(xv)[1]
dim_st = xdim * ydim # dimension of the statespace
xn = np.reshape(xv, (xdim * ydim, 1))
yn = np.reshape(yv, (xdim * ydim, 1))
grid = np.squeeze(np.array([xn, yn]))
```

We instantiate the TPT class ``stationary.tpt`` for the stationary case using the (row-stochastic and irreducible) transition matrix $T$ and index sets of $A$, $B$, and $C$ (``ind_A, ind_B, ind_C``).
Then we can compute the various statistics of the ensemble of trajectory pieces that start in $A$ and end in $B$.

```python
# instantiate
well3 = stationary.tpt(T, ind_A, ind_B, ind_C)
# compute statistics
well3.compute_statistics()
```

The stationary density of the process is given by $\pi^\top = P \pi^\top$ and indicates that the process spends most of the time in the two deep wells centered at $(-1,0)$ and $(1,0)$ and sometimes visits the lower well at $(0,1.5)$.

```python
stat_dens = well3.stat_dens

# plotting
data = np.array([stat_dens])
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim, ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), 1, (3*1,3), v_min, v_max, ['$\pi$'], file_path=file_path)
```

Then we can have a look at the committor functions, the forward committor $q^+$ gives the probability that the process next visits $B$ and not $A$, while the backward committor $q^-$ gives the probability that the process last came from $A$ and not $B$.
The committors can be computed from the following linear systems

\begin{equation}
\left\{ \begin{array}{rcll}
q_{i}^+ &=& \sum_{j\in \mathbb{S}} \, T_{ij} \, q_{j}^+ &i \in C,   \\
    q_{i}^+ &=& 0& i \in A,  \\
    q_{i}^+ &=& 1& i \in B,  \\
\end{array}\right.
\end{equation}

and

\begin{equation}
\left\{ \begin{array}{rcll}
q_{i}^- &=& \sum_{j\in \mathbb{S}} \, T_{ij}^- \, q_{j}^- &i \in C,   \\
    q_{i}^- &=& 0& i \in B,  \\
    q_{i}^- &=& 1& i \in A,  \\
\end{array}\right.
\end{equation}

where $\mathbb{S}$ is the statespace, $C=(A\cup B)^c$ is the region through which the transitions are passing, and $T^-$ the time-reversed transition matrix.

```python
q_f = well3.q_f
q_b = well3.q_b

# plotting
data = np.array([q_f])
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0],interval[0,1], interval[1,0], interval[1,1]), 1, (3*1,3), v_min, v_max, ['$q^+$'],  file_path=file_path)

data = np.array([q_b])
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0],interval[0,1],interval[1,0],interval[1,1]), 1, (3*1,3), v_min, v_max, ['$q^-$'],  file_path=file_path)
```

The (normalized) density of reactive trajectories can then be found as  $\hat{\mu}_i^{AB} = \frac{1}{Z^{AB}} q_i^- q_i^+ \pi_i$ ($Z^{AB}$ is the normalization constant) and shows where the trajectory spends most of their time while transitioning from $A$ to $B$.

```python
norm_reac_dens = well3.norm_reac_dens

# plotting

# to highlight the sets A and B
densAB = np.zeros(dim_st); densAB[ind_A] = 1; densAB[ind_B] = 1

data = np.array([norm_reac_dens])
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0],interval[0,1],interval[1,0],interval[1,1]) , 1, (3*1,3), v_min, v_max, ['$\hat{\mu}^{AB}$'],  file_path=file_path, background=densAB)
```

The current of reactive trajectories is given by $f_{ij}^{AB} = q_i^- \pi_i T_{ij}q_j^+$, here we plot the vector field of the effective current $f^+_{ij} = \max\left(0, f_{ij}^{AB}-f_{ji}^{AB}\right)$ indicating where the reactive trajectories are effectively happening.

```python
eff_current = well3.eff_current

# plotting

# calculation the effective vector for each state
eff_vectors = np.zeros((dim_st, 2))
eff_vectors_unit = np.zeros((dim_st, 2))
colors = np.zeros(dim_st)
for i in np.arange(dim_st):
    for j in np.arange(dim_st):
        if np.linalg.norm(np.array([xn[j] - xn[i], yn[j] - yn[i]])) > 0:
            eff_vectors[i,0] += eff_current[i,j] *  (xn[j] - xn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
            eff_vectors[i,1] += eff_current[i,j] *  (yn[j] - yn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
    colors[i] = np.linalg.norm(eff_vectors[i,:])
    if colors[i] > 0:
        eff_vectors_unit[i,:] = eff_vectors[i,:]/colors[i]

plot_3well_effcurrent(np.array([eff_vectors_unit]), np.array([colors]), xn, yn, densAB, (xdim, ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), 1, (3*1, 3), ['$f^+$'],  file_path=file_path)
```

Last, we give the rate of trajectories leaving $A$ on the way to $B$ as well as the mean transition length from $A$ to $B$.

```python
rate = well3.rate
length = well3.length

print("rate: %f" % rate)
print("mean length: %f" % length)
```

    rate: 0.014181
    mean length: 10.042560

# Periodically-forced process

given by $dX_t =( - \nabla V(X_t) + F(X_t, t) ) dt + \sigma dW_t$ which adds an anti-clockwise, periodic circulation $F(x,t)$ to the triplewell dynamics from the above example.

To apply TPT, we again estimate a transition matrix from the continuous process, but this time the transition matrix varies periodically with period $M=6$.

We instanstiate the class ``periodic.tpt`` with a function that maps the time point $m$ to the corresponding transition matrix $T_m$, the length of the period $M$, as well as the index sets of $A$, $B$ and $C$.

```python
M = np.shape(T_m)[0] # length of period

def Tm(m):
    return T_m[np.mod(m, M), :, :].squeeze()

# instantiate
well3_periodic = periodic.tpt(Tm, M, ind_A, ind_B, ind_C)
# compute statistics
well3_periodic.compute_statistics()
```

The periodically-varying, equilibrated density is shown for the different time instances during one period.

```python
stat_dens_p = well3_periodic.stat_dens

# plotting
def subtitles_m(quant,M):
    return np.array([quant.format(str(i)) for i in np.arange(M)])

data = stat_dens_p
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), M, (3*M,3), v_min, v_max, subtitles_m('$\pi_{}$', M),  file_path=file_path)

```

Then we can give the committors which are now also depending on the time point $m$ within the period.

\begin{equation}
\left\{ \begin{array}{rcll}
q_{m,i}^+ &=& \sum_{j\in \mathbb{S}} \, T_{m,ij} \, q_{m+1,j}^+ & i \in C, \\
    q_{m,i}^+ &=& 0& i \in A,  \\
    q_{m,i}^+ &=& 1& i \in B,  \\
\end{array}\right.
\end{equation}

and analogously for the backward committor.

```python
q_f_p = well3_periodic.q_f
q_b_p = well3_periodic.q_b

# plotting
data = q_f_p
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), M, (3*M,3), v_min, v_max, subtitles_m('$q^+_{}$', M), file_path=file_path)

data = q_b_p
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), M, (3*M,3), v_min, v_max, subtitles_m('$q^-_{}$', M), file_path=file_path)

```

The transition statistics such as the density of reactive trajectories $\hat{\mu}_m^{AB}$ and the effective current of reactive trajectories $f^+_m$ vary within the period and show how the circular forcing influences the transition pathways.

```python
norm_reac_dens_p = well3_periodic.norm_reac_dens

# plotting
data = norm_reac_dens_p
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), M, (3*M,3), v_min, v_max,np.array(['$\hat{\mu}^{AB}_0$','$\hat{\mu}^{AB}_1$','$\hat{\mu}^{AB}_2$','$\hat{\mu}^{AB}_3$','$\hat{\mu}^{AB}_4$','$\hat{\mu}^{AB}_5$']), file_path=file_path, background=densAB)

eff_current_p = well3_periodic.eff_current

# plotting
# calculation the effective vector for each state
eff_vectors_p = np.zeros((M, dim_st, 2))
eff_vectors_unit_p = np.zeros((M, dim_st, 2))
colors_p = np.zeros((M, dim_st))
for m in np.arange(M):
    for i in np.arange(dim_st):
        for j in np.arange(dim_st):
            if np.linalg.norm(np.array([xn[j] - xn[i], yn[j] - yn[i]])) > 0:
                eff_vectors_p[m,i,0] += eff_current_p[m,i,j] *  (xn[j] - xn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
                eff_vectors_p[m,i,1] += eff_current_p[m,i,j] *  (yn[j] - yn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
        colors_p[m,i] = np.linalg.norm(eff_vectors_p[m,i,:])
        if colors_p[m,i]>0:
            eff_vectors_unit_p[m,i,:] = eff_vectors_p[m,i,:]/colors_p[m,i]

plot_3well_effcurrent(eff_vectors_unit_p, colors_p, xn, yn, densAB, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), M, (3*M,3), subtitles_m('$f^+_{}$', M), file_path=file_path)
```

We can compare the averaged rate of reactive trajectories leaving $A$ and the mean transition length with the same quantities from the non-forced process above to note that the forcing increases the likelihood of $A\rightarrow B$ transitions.

```python
time_av_rate_p = well3_periodic.time_av_rate
av_length_p = well3_periodic.av_length

print("periodic-averaged rate: %f" % time_av_rate_p[0])
print("mean length: %f" % av_length_p)
```

    periodic-averaged rate: 0.015208
    mean length: 9.779124

# Finite-time transitions

We consider again the stationary motion in the triplewell landscape, but restrict transitions to a finite-time interval $[0,N]$.

We instantiate the class ``finite.tpt`` with the function ``Tn`` that maps time $n$ to the corresponding transition matrix (possibly time-dependent, here not), the length of the time interval, the index sets and the initial density at time $n=0$ (here the stationary density of the infinite-time process).

```python
def Tn(n):
    return T

N = 6 #time window

# initial density
init_dens_well3_finite = well3.stat_dens
# instantiate
well3_finite = finite.tpt(Tn, N, ind_A, ind_B, ind_C, init_dens_well3_finite)
# compute statistics
well3_finite.compute_statistics()
```

The committor equations are now iterative equations in time, backward in time for the forward committor $q^+(n)$ and forward in time for the backward committor $q^-(n)$.

\begin{equation}
\left\{ \begin{array}{rcll}
q_{i}^+(n) &=& \sum_{j \in \mathbb{S}} \, T_{n,ij} \, q_{j}^+(n+1)    & i \in C,  \\
q_{i}^+(n) &=& 0 & i \in A, \\
q_{i}^+(n) &=& 1 & i \in B,
\end{array}\right.
\end{equation}

```python
q_f_f = well3_finite.q_f
q_b_f = well3_finite.q_b

# plotting
data = q_f_f
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), N, (3*N,3), v_min, v_max, subtitles_m('$q^+({})$',N), file_path=file_path)

data = q_b_f
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data, (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), N, (3*N,3), v_min, v_max, subtitles_m('$q^-({})$',N), file_path=file_path)
```

We can again show the time-dependent transition statistics.

```python
norm_reac_dens_f = well3_finite.norm_reac_dens
eff_current_f = well3_finite.eff_current

# plotting
data = norm_reac_dens_f
v_min = np.nanmin(data)
v_max = np.nanmax(data)
plot_3well(data[1:N-1,:], (xdim,ydim), (interval[0,0], interval[0,1], interval[1,0], interval[1,1]), N-2, (3*(N-2),3), v_min, v_max, np.array(['$\hat{\mu}^{AB}(1)$','$\hat{\mu}^{AB}(2)$','$\hat{\mu}^{AB}(3)$','$\hat{\mu}^{AB}(4)$']), file_path=file_path, background=densAB)

# calculation the effective vector for each state
eff_vectors_f = np.zeros((N,dim_st, 2))
eff_vectors_unit_f = np.zeros((N,dim_st, 2))
colors_f = np.zeros((N,dim_st))
for n in np.arange(N):
    for i in np.arange(dim_st):
        for j in np.arange(dim_st):
            # if np.isnan(eff_current_f[n,i,j])==False:

            if np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]]))>0:
                eff_vectors_f[n,i,0] += eff_current_f[n,i,j] *  (xn[j] - xn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
                eff_vectors_f[n,i,1] += eff_current_f[n,i,j] *  (yn[j] - yn[i])  *(1/np.linalg.norm(np.array([xn[j] - xn[i],yn[j] - yn[i]])))
        colors_f[n,i] = np.linalg.norm(eff_vectors_f[n,i,:])
        if colors_f[n,i]>0:
            eff_vectors_unit_f[n,i,:] = eff_vectors_f[n,i,:]/colors_f[n,i]

plot_3well_effcurrent(eff_vectors_unit_f[:N-1,:,:], colors_f[:N-1,:], xn, yn, densAB,(xdim,ydim), (interval[0,0],interval[0,1],interval[1,0],interval[1,1]), N-1, (3*(N-1),3), subtitles_m('$f^+({})$',N-1), file_path=file_path)
```

And consider the time-averaged rate of transitions leaving $A$, which is now much smaller than in the infinite-time case since only a subset of transitions is considered, namely, those that happen within $[0,N]$. The mean length of transitions thus also has an upper limit of $N$.

```python
time_av_rate_f = well3_finite.time_av_rate
av_length_f = well3_finite.av_length

print("time-averaged rate (finite-time, time-homogeneous): %f" % time_av_rate_f[0])
print("mean length (finite-time, time-homogeneous): %f" % av_length_f)
```

    time-averaged rate (finite-time, time-homogeneous): 0.001661
    mean length (finite-time, time-homogeneous): 2.073717

```python

```

```python

```


```python
# Importacao de pacotes
%matplotlib inline

import matplotlib.pyplot as plt
import numpy as np
import sympy as sp
from numba import jit

np.seterr(divide = 'ignore', invalid = 'ignore'); # ignora warnings sobre divisao por zero (no calculo da probabilidade de flipar os spins para T = 0)
```

```python
@jit
def metropolis(L, N, T, J = 1, h = 0, kB = 1):
    spins = np.random.choice([-1, +1], (L, N)) # gera uma configuracao aleatoria de spins

    # Faz N varreduras sobre o grid
    for n in range(N):
        # Ordem aleatoria para varrer os spins do grid
        ordem = np.random.permutation(L)

        # Varredura sobre os spins e evolucao (flips aleatorios) devido a temperatura
        for i in ordem:
            # Calculo da energia para flipar o spin na posicao i
            Eflip = 2*spins[i, n]*(J*(spins[(i - 1)%L, n] + spins[(i + 1)%L, n]) + h)

            # Se Eflip < 0, flipa o spin; caso contrario, aplica-se o passo de Monte Carlo
            if Eflip < 0:
                spins[i, n] = - spins[i, n]
            else:
                Pflip = np.exp(- Eflip/(kB*T)) # probabilidade de flipar o spin na posicao i
                if np.random.uniform(0, 1) < Pflip:
                    spins[i, n] = - spins[i, n]
    return spins
```

```python
@jit
def H(vecspins, J, h):
    H = 0
    L = vecspins.shape[0]
    for i in range(L):
        H += - J*(vecspins[i]*vecspins[(i + 1)%L]) - h*vecspins[i]
    return H
```

```python
# Inicializacao das variaveis da simulacao
L = 5 # tamanho do array de spins
N = int(1e4) # numero de varreduras
J = 1 # constante de interacao entre os spins
h = 1 # constante de interacao do spin com o campo magnetico externo
kB = 1 # constante de Boltzmann

Tmin = 0 # temperatura minima (bem proxima de zero)
Tmax = 20 # temperatura maxima
DeltaT = 1 # passo das temperaturas
vecT = np.arange(Tmin, Tmax + DeltaT, DeltaT) # vetor de temperaturas
NT = len(vecT) # tamanho do vetor de temperaturas
Beta = 1/(kB*vecT) # vetor de temperatura inversa

m = np.zeros((L, NT)) # array de magnetizacao media de cada elemento para todas as temperaturas
E = np.zeros((NT, N))
```

```python
# Simulacao para cada temperatura
for k in range(NT):
    T = vecT[k] # obtem a temperatura
    spins = metropolis(L, N, T, J, h, kB) # simula a configuracao de spins (N vezes)
    m[:, k] = spins.mean(axis = 1) # magnetizacao media de cada elemento (media das N varreduras)
    E[k, :] = H(spins, J, h)/L # energia por sitio do sistema para cada temperatura e varredura

# Calcula a magnetizacao media do sistema para cada temperatura
mm = m.mean(axis = 0)
Em = E.mean(axis = 1)
```

```python
# Média dos spins de cada posicao para cada temperatura
plt.imshow(m.T, origin = 'lower', extent = [0, L, Tmin, Tmax])
plt.xlabel(r'Posicao, $i$')
plt.xticks(np.arange(0.5, L, 1.0), np.arange(0, L))
plt.ylabel(r'$Temperatura$')
#plt.yticks(np.arange(Tmin + DeltaT/2, Tmax, DeltaT), np.arange(Tmin, Tmax, DeltaT))
plt.colorbar(shrink = 0.47, label = r'$\langle \sigma_i \rangle$')
plt.show()
```

```python
# Magnetizacao media para cada temperatura
plt.errorbar(vecT, mm, fmt = '-o', label = 'simulacao')

# Resultado analitico
mexact = np.sinh(Beta*h)/np.sqrt(np.sinh(Beta*h)**2 + np.exp(- 4*Beta*J))
sT = sp.Symbol('T')
mexact[0] = sp.limit(sp.sinh(h/(kB*sT))/sp.sqrt(sp.sinh(h/(kB*sT))**2 + sp.exp(- 4*J/(kB*sT))), sT, 0)

plt.plot(vecT, mexact, label = 'exata')

plt.xlabel('$T$')
plt.ylabel(r'$\langle m \rangle$')
plt.ylim(-1.1, 1.1)
plt.legend()
plt.show()
```

```python
plt.plot(vecT, Em, '-o')
plt.xlabel(r'$T$')
plt.ylabel(r'$E/L$')

plt.show()
```

```python

```


# 随机映射（Random Projection, RP）

## 符号定义

|符号|含义|
|:--:|:--:|
|$\pmb{x}$|数据点|
|$\pmb{y}$|降维后数据点|
|$S$|数据点集合|
|$N$|数据点总数|
|$f$|映射函数|
|$d$|降维前维度|
|$m$|降维后维度|
|$\epsilon$|常数|
|$Z$|映射矩阵|

## 概念

随机映射尝试了一个非常大胆的降维思路：通过一个随机生成的映射函数直接将高维空间中的数据点嵌入到低维空间，并且保证数据点之间的欧式距离变化存在一个上界和下界（失真可控）。

降维方法往往需要基于一定的假设或者说降维的目标，例如PCA的一种解释是PCA以最大方差的方向对原始数据进行降维，LLE则假设降维前后每一个数据点均可以由其周围的点以相同的权重线性表示。随机映射则保证数据点之间的降维前后欧式距离的变化存在一个上界和下界。

随机映射最为吸引人的地方无疑是其映射函数的随机性。无需计算距离或是特征向量的特点显然能大大加快降维的速度。

随机映射的理论保证是Johnson–Lindenstrauss lemma，Johnson–Lindenstrauss lemma的内容如下：

假设存在一个数据集，$S=\{\pmb{x_i}\}_{i=1}^N, \pmb{x_i}\in\mathcal{R}^d$，d是一个相当高的维数。给定一个常数$\epsilon, 0\leq\epsilon\leq 1$，则存在一个映射函数$f:\mathcal{R}^d\rightarrow\mathcal{R}^m, m\geq\frac{4\ln(N)}{\frac{\epsilon^2}{2} - \frac{\epsilon^3}{3}}$，有如下结论：

$$
\begin{equation}
    (1-\epsilon)||\pmb{x_i}-\pmb{x_j}||^2\leq||f(\pmb{x_i})-f(\pmb{x_j})||^2\leq(1+\epsilon)||\pmb{x_i}-\pmb{x_j}||^2, \forall \pmb{x_i}, \pmb{x_j}\in S
\end{equation}
$$

Johnson–Lindenstrauss lemma表明对于一个高维空间的降维问题，总能找到一个映射函数使得降维后任何两个数据点之间的距离是原空间中这两个数据点欧式距离的$[1-\epsilon, 1+\epsilon]$倍范围内。

在Johnson–Lindenstrauss lemma的基础上，又能证明从高斯分布$\mathcal{N}(0, \frac{1}{\sqrt m})$随机采样生成的映射矩阵大概率正好能满足上述条件。

因此使用从高斯分布$\mathcal{N}(0, \frac{1}{\sqrt m})$随机采样生成的映射矩阵对数据进行降维能够在Johnson–Lindenstrauss lemma的保证下实现较为可靠的降维。

## 算法流程

* 定义数据集、数据维度以及常数$\epsilon$
* 指定降维后维度$m$
* 从高斯分布$\mathcal{N}(0, \frac{1}{\sqrt m})$随机生成映射矩阵$Z$
* 映射得到输出

## 参考资料

https://zhuanlan.zhihu.com/p/158512304

https://scikit-learn.org/stable/modules/random_projection.html

https://scikit-learn.org/stable/auto_examples/miscellaneous/plot_johnson_lindenstrauss_bound.html#sphx-glr-auto-examples-miscellaneous-plot-johnson-lindenstrauss-bound-py

https://web.njit.edu/~usman/courses/cs675_fall19/JL_lemma.pdf

```python
import numpy as np
from sklearn import random_projection
```

```python
class MyRP(object):

    def __init__(self, eps=0.1, random_state=None):
        self.eps = eps
        self.random_state = random_state

    def fit_transform(self, input_data):

        input_data = np.array(input_data)
        n_samples, input_dims = input_data.shape

        min_dims = self.judge_min_dim(n_samples, self.eps)

        if self.random_state is not None:
            np.random.seed(self.random_state)

        # create transfer matrix
        transfer_mat = np.random.normal(loc=0, scale=1/np.sqrt(min_dims), size=(min_dims, input_dims))

        output_mat = np.matmul(input_data, transfer_mat.T)
        return output_mat

    def judge_min_dim(self, n_samples, eps):
        return int(4 * np.log(n_samples)/(pow(eps, 2)/2 - pow(eps, 3)/3))
```

```python
# --------------------- create data -----------------------
sample_num = 100
input_dims = 10000
eps = 0.5

input_data = np.random.rand(sample_num, input_dims)

# -------------------- sklearn RP --------------------------
sklearn_rp = random_projection.GaussianRandomProjection(eps=0.5, random_state=1024)
sklearn_rp_result = sklearn_rp.fit_transform(input_data)

# --------------------- My RP -----------------------------
my_rp = MyRP(eps=0.5, random_state=1024)
my_rp_result = my_rp.fit_transform(input_data)

# compare result
print(my_rp_result - sklearn_rp_result)
print(np.linalg.norm(sklearn_rp_result-my_rp_result, ord='fro'))
```

    [[0. 0. 0. ... 0. 0. 0.]
     [0. 0. 0. ... 0. 0. 0.]
     [0. 0. 0. ... 0. 0. 0.]
     ...
     [0. 0. 0. ... 0. 0. 0.]
     [0. 0. 0. ... 0. 0. 0.]
     [0. 0. 0. ... 0. 0. 0.]]
    0.0


# Assignment 1: warm-up

The goal of this assignment is to learn/review python and ipynb.
Python is a popular programming language, and also interfaced for several machine learning libraries, such as scikit-learn, Theano, and TensorFlow.
Ipynb is a digital notebook format that allows flexible incorporation of a variety of information, such as code (e.g. python), data, text (e.g. markdown, html, and Latex), images (common raster/vector graphics formats such as jpg and svg), and video (e.g. youtube).
We can also run code and experiments directly inside ipynbs.
Thus, we will use ipynb for all assignments in this class.

# Sorting

As a starting exercise, let's try to implement a sorting function via python.
The input to the function is a python array consisting of an arbitrary sequence of numbers.
The output is a sorted sequence with numbers ranging from small to large.
The code stub, along with the test driver, are shown below.

There are various algorithms for sorting with different time complexities with respect to the array size $N$, e.g. $O(N^2)$ for bubble sort and $O(Nlog(N))$ for quick sort.
You can choose any algorithm to implement, as long as it produces correct results with reasonable run-time.

Please submit a single ipynb file, consisting of python code in a code cell and descriptions (including algorithm and analysis of complexity) in a markdown cell.
You can use this ipynb as a start, or create your own.

## Code (20 points)

Please implement your algorithm via the function below.

```python
# the function
def sort(values):
    # insert your code here
    return values
```

```python
# test
import numpy as np

def test(values):
    sort(values)

    correct = True
    for index in range(1, len(values)):
        if(values[index-1] > values[index]):
            correct = False

    print('Correct? ' + str(correct))
```

```python
# different random seed
np.random.seed()

# the TA will vary the input array size and content during testing
N = 10
values = np.random.random([N])

test(values)
```

    Correct? False

# Line fitting

Given a set of data points $\left(\mathbf{X}, \mathbf{Y}\right)$, fit a model curve to describe their relationship.

This is actually a regression problem, but we have all seen this in prior math/coding classes to serve as a good example for machine learning.

Recall $\mathbf{Y} = f(\mathbf{X}, \Theta)$ is our model.

For 2D linear curve fitting, the model is a straight line:
$y = w_1 x + w_0$, so the parameters $\Theta = \{w_0, w_1\}$.

The loss function is $L\left(\mathbf{X}, \mathbf{T}, \mathbf{Y}\right) = \sum_i \left( T^{(i)} - Y^{(i)}\right)^2 = \sum_i \left( T^{(i)} - w_1 X^{(i)} - w_0 \right)^2$.
<br>
($\mathbf{X}$ is a matrix/tensor, and each data sample is a row. We denote the ith sample/row as $\mathbf{X}^{(i)}$.)

For this simple example we don't care about regularization, thus $P(\Theta) = 0$.

The goal is to optimize $\Theta = \{w_0, w_1 \}$ with given $\left(\mathbf{X}, \mathbf{Y}\right)$ to minimize $L$.
For simple cases like this, we can directly optimize via calculus:
$$
\begin{align}
\frac{\partial L}{\partial w_0} & = 0 \\
\frac{\partial L}{\partial w_1} & = 0
\end{align}
$$

## Math (30 points)

Write down explicit formulas for $w_0$ and $w_1$ in terms of $\mathbf{X}$ and $\mathbf{T}$.

## Code (50 points)

Implement your math above in the code below.

```python
# line model
import numpy as np

class Line(object):
    def __init__(self, w0, w1):
        self.w0 = w0
        self.w1 = w1

    def predict(self, x, noise=0):
        return (x*self.w1 + self.w0 + noise*np.random.normal())

    # Input: data, a 2D array with each (x, t) pair on a row
    # Return: w0 and w1, the intercept and slope of the fitted line
    def learn(self, data):
        # replace the default code below which simply does random computation
        # with your math equations derived above
        w0 = np.asscalar(np.random.random(1))*2-1
        w1 = np.asscalar(np.random.random(1))*2-1

        return w0, w1
```

```python
# test
np.random.seed()

w0 = np.asscalar(np.random.random(1))*2-1
w1 = np.asscalar(np.random.random(1))*2-1

line = Line(w0, w1)

N = 20
noise = 0.05
X = np.random.random([N])
T = []
for x in X:
    T.append(np.sum(line.predict(x, noise)))
T = np.array(T)

#data = np.vstack((X, T)).transpose()
data = np.array([X, T]).transpose()

w0_fit, w1_fit = line.learn(data)

line_fit = Line(w0_fit, w1_fit)

print('truth:   ' + str(w0) + ' ' + str(w1))
print('predict: ' + str(w0_fit) + ' ' + str(w1_fit))
```

    truth:   0.3125292670611821 0.47730830112548217
    predict: -0.45156965073023114 0.6629669020357796

```python
# plot
%matplotlib inline
import matplotlib.pyplot as plt

plt.scatter(data[:, 0], data[:, 1], color='black', marker='o')

X_endpoints = [0, 1]
Y_truth, Y_fit = [], []
for x in X_endpoints:
    Y_truth.append(line.predict(x))
    Y_fit.append(line_fit.predict(x))

plt.plot(X_endpoints, Y_truth, color='blue', label='truth')
plt.plot(X_endpoints, Y_fit, color='red', label='predict')

plt.legend(loc='best')
plt.tight_layout()
plt.show()
```