```python
import matplotlib.pyplot as plt
from scipy import optimize
import numpy as np
import sympy as sm
import func
```

```python
# Parameters
#Micro:
theta = 0.9
beta = 0.99
#Macro:
alpha = 1/3
delta = 0.05
n = 0.01

# produktion:
b = 1
k0 = 5
l0 = 1
```

```python
t = 40
weight = np.array([beta**i for i in range(t)])
```

```python
times = 50
ks = np.linspace(0.1,100,times)
sks = np.empty(times)
for i,k in enumerate(ks):
    sks[i] = func.optimal_sks(t, b, l0, n, weight, delta, alpha, theta, k)

```

```python
func.plot(ks,sks)
```

# Solowwalk

With fixed savings rate

```python
def capitalakku(b,k,l,sk,alpha,delta):
    return func.prod(k,l,alpha,b)*sk+(1-delta)*k
def prod(k,l,alpha,b):
    return (k**alpha)*((b*l)**(1-alpha))
```

```python
def solowwalk(k0, b, l0, n, sk, alpha, delta, theta,t_big):
    k_path = np.array([k0])
    l_path = np.array([l0*(1+n)**i for i in list(range(t_big))])
    y_path = np.array([prod(k_path[0],l_path[0],alpha,b)])

    for i in range(1,t_big):
        k_plus = capitalakku(b,k_path[i-1],l_path[i-1],sk,alpha,delta)
        y_plus = func.prod(k_plus,l_path[i-1],alpha,b)

        k_path = np.append(k_path, k_plus)
        y_path = np.append(y_path, y_plus)

    k_pr_path = k_path/l_path
    y_pr_path = y_path/l_path
    return y_path, k_path, k_pr_path, y_pr_path
```

```python
t_big = 150
sk = alpha
```

```python
y_path, k_path, k_pr_path, y_pr_path = solowwalk(k0, b, l0, n, sk, alpha, delta, theta,t_big)
k_pr_path[-1]
```

    13.071844455355642

```python
steady_state = ((b*sk)/(n+delta))**(1/(1-alpha))
steady_states = np.full(t_big,steady_state)
```

```python
func.plot2(range(t_big),k_pr_path,steady_states)
```

**Finding steady state for given savings rate:**

\\[
Y=B \cdot K^{\alpha} \cdot L^{1-\alpha} \Leftrightarrow y=B k^{\alpha}
\\]
\\[
K_{t+1}=s_{Kt} \cdot Y_{t} + \left(1-\delta\right)K_{t}
\\]
\\[
k_{t+1}=\frac{1}{1+n} \left(s_{Kt} B k^{\alpha} + \left(1-\delta\right)k_{t}\right)
\\]
Steady state implies:
\\[
k_{t+1}=k_{t}
\\]
Inserting gives:
\\[
k_{t}=\frac{1}{1+n} \left(s_{Kt} B k^{\alpha} + \left(1-\delta\right)k_{t}\right)
\\]
\\[
1=\frac{1}{1+n} \left(s_{Kt} B k^{\alpha -1} + \left(1-\delta\right)\right)
\\]
\\[
n+\delta = s_{Kt} B k^{\alpha -1}
\\]

```python
sm.init_printing(use_unicode=True)
```

```python
sk = sm.symbols('s_Kt')
# Ud fra de valg nogle udfald
# Først defineres variable
alpha = sm.symbols('alpha')
k =sm.symbols('k_{t}')
delta = sm.symbols('delta')
k_star = sm.symbols('k^{*}')
n = sm.symbols('n')
b = sm.symbols('B')
```

```python
sseq = sm.Eq(k,1/(1+n)*(sk*b*k**alpha+(1-delta)*k))
sseq
```

```python
ss_sk = sm.solve(sseq,sk)[0]
ss_k = sm.solve(sseq,k)[0]
```

```python
sm.Eq(sk,ss_sk)
```

```python
sm.Eq(k_star,ss_k)
```

```python
find_ssk_sk = sm.lambdify((k,b,delta,n,alpha),ss_sk)
find_ssk_k = sm.lambdify((sk,b,delta,n,alpha),ss_k)
```

```python
sm.init_printing(use_unicode=False)
```

```python
# Parameters
#Micro:
theta = 0.9
beta = 0.99
rho = 0.5
#Macro:
alpha = 1/3
delta = 0.05
n = 0.01

# produktion:
b = 1
k0 = 5
l0 = 1
```

```python
t = 40
weight = np.array([beta**i for i in range(t)])
```

```python
find_ssk_sk(10,b,delta,n,alpha)
```

```python
find_ssk_k(0.2784953300167668,b,delta,n,alpha)
```

```python
times = 30
ks = np.linspace(0.1,30,times)
sks = np.empty(times)
for i,k in enumerate(ks):
    sks[i] = func.optimal_sks(t, b, 1, n, weight, delta, alpha, theta, k)
```

```python
sksopt = find_ssk_sk(ks,b,delta,n,alpha)
```

```python
func.plot2(ks,sks,sksopt)
```

```python
def find_ss(t, b, n, beta, delta, alpha, theta,bracket):
    weight = np.array([beta**i for i in range(t)])
    obj = lambda k_star: find_ssk_sk(k_star,b,delta,n,alpha)-func.optimal_sks(t, b, 1, n, weight, delta, alpha, theta, k_star)
    res = optimize.root_scalar(obj,method='brentq',bracket=bracket)
    if res.converged:
        k_star = res.root
        sk_star = find_ssk_sk(k_star,b,delta,n,alpha)
        return k_star,sk_star
    else:
        print('Convergence failed')

```

```python
bracket =[0,100]
```

```python
find_ss(100, b, n, beta, delta, alpha, theta, bracket)
```

**With changing savings rate**

```python
def mod_solowwalk(k0, b, l0, n, alpha, delta, rho, beta, theta, t, t_big):
    weight = np.array([beta**i for i in range(t)])
    k_path = np.array([k0])
    l_path = np.array([l0*(1+n)**i for i in list(range(t_big))])
    y_path = np.array([func.prod(k_path[0],l_path[0],alpha,b)])

    sk_path = np.array([func.optimal_sks(
        t, b, l_path[0], n, weight, delta, alpha, theta, k_path[0])])

    for i in range(1,t_big):
        k_plus = capitalakku(b,k_path[i-1],l_path[i-1],sk_path[i-1],alpha,delta)
        y_plus = func.prod(k_plus,l_path[i-1],alpha,b)
        sk_plus = np.array([func.optimal_sks(t, b, l_path[i], n, weight, delta, alpha, theta, k_plus)])

        k_path = np.append(k_path, k_plus)
        y_path = np.append(y_path, y_plus)
        sk_path = np.append(sk_path,sk_plus)

    k_pr_path = k_path/l_path
    y_pr_path = y_path/l_path
    return y_path, k_path, k_pr_path, y_pr_path, sk_path
```

```python
t_big = 150
```

```python
y_path, k_path, k_pr_path, y_pr_path, sk_path = mod_solowwalk(
    k0, b, l0, n, alpha, delta, rho, beta,theta, t, t_big)
```

```python
brakcet = [0,30]
```

```python
k_star,sk_star = find_ss(t, b, n, beta, delta, alpha, theta, bracket)
```

```python
func.plot2(range(t_big),k_pr_path,np.full(t_big,k_star))
func.plot2(range(t_big), sk_path,np.full(t_big,sk_star))
func.plot(range(t_big), k_path)
```

```python
# capital savingsrate in ss.
sk_path[-1]
```

```python
# Capital pr. worker in ss.
k_pr_path[-1]
```

# R&D with fixed sr but adjustable sk

**Needs new micro fundation that understands growth in tech**

```python
def capitalakku_a(a,k,l,sk,sr,alpha,delta):
    return proda(k,l,alpha,a,sr)*sk+(1-delta)*k

def research(a,sr,l,rho,phi):
    return a+rho*a**phi*(l*sr)**lambd

def proda(k,l,alpha,a,sr):
    return (k**alpha)*((a*l)**(1-alpha))

def mod_solowwalk(k0,a0, l0, n, sr, alpha, delta, rho, phi, theta, beta, t, t_big):
    weight = np.array([beta**i for i in range(t)])
    k_path = np.array([k0])
    a_path = np.array([a0])
    l_path = np.array([l0*(1+n)**i for i in list(range(t_big))])
    y_path = np.array([proda(k_path[0],l_path[0],alpha,a_path[0],sr)])
    sk_path = np.array([func.optimal_sks(
        t, b, l_path[0], n, weight, delta, alpha, theta, k_path[0])])

    for i in range(1,t_big):
        k_plus = capitalakku_a(a_path[i-1],k_path[i-1],l_path[i-1],sk_path[i-1],sr,alpha,delta)
        a_plus = research(a_path[i-1],sr,l_path[i-1],rho,phi)
        y_plus = proda(k_plus,l_path[i],alpha,a_plus,sr)
        sk_plus = np.array([func.optimal_sks(t, b, l_path[i], n, weight, delta, alpha, theta, k_plus)])

        k_path = np.append(k_path, k_plus)
        a_path = np.append(a_path, a_plus)
        y_path = np.append(y_path, y_plus)
        sk_path = np.append(sk_path,sk_plus)

    k_tilde_path = k_path/(a_path*l_path)
    y_tilde_path = y_path/(a_path*l_path)
    return a_path, k_tilde_path, y_tilde_path, sk_path
```

```python
# Parameters
#Micro:
theta = 0.9
beta = 0.99
#Macro:
alpha = 1/3
delta = 0.05
n = 0.01
rho = 0.01
phi = 0.1
lambd = 0.1
sr = 0.01

# produktion:
l0 = 10
k0 = 150
a0 = 10

t_big = 300
t = 30
```

```python
a_path, k_tilde_path, y_tilde_path, sk_path = mod_solowwalk(k0,a0, l0, n, sr, alpha, delta, rho, phi, theta, beta, t, t_big)
```

```python
func.plot(range(t_big),k_tilde_path)
func.plot(range(t_big),y_tilde_path)
func.plot(range(t_big),sk_path)
func.plot(range(t_big),np.log(a_path))
```

# Speed

```python
import time
%load_ext line_profiler
```

```python
%timeit mod_solowwalk(k0, b, l0, n, alpha, delta, rho, beta,theta, t, t_big)
```

```python
%lprun -f mod_solowwalk mod_solowwalk(k0, b, l0, n, alpha, delta, rho, beta,theta, t, t_big)

```


## Markov switching autoregression models

This notebook provides an example of the use of Markov switching models in statsmodels to replicate a number of results presented in Kim and Nelson (1999). It applies the Hamilton (1989) filter the Kim (1994) smoother.

This is tested against the Markov-switching models from E-views 8, which can be found at http://www.eviews.com/EViews8/ev8ecswitch_n.html#MarkovAR or the Markov-switching models of Stata 14 which can be found at http://www.stata.com/manuals14/tsmswitch.pdf.

```python
%matplotlib inline

import numpy as np
import pandas as pd
import statsmodels.api as sm
import matplotlib.pyplot as plt
import requests
from io import BytesIO

# NBER recessions
from pandas_datareader.data import DataReader
from datetime import datetime
usrec = DataReader('USREC', 'fred', start=datetime(1947, 1, 1), end=datetime(2013, 4, 1))
```

### Hamilton (1989) switching model of GNP

This replicates Hamilton's (1989) seminal paper introducing Markov-switching models. The model is an autoregressive model of order 4 in which the mean of the process switches between two regimes. It can be written:

$$
y_t = \mu_{S_t} + \phi_1 (y_{t-1} - \mu_{S_{t-1}}) + \phi_2 (y_{t-2} - \mu_{S_{t-2}}) + \phi_3 (y_{t-3} - \mu_{S_{t-3}}) + \phi_4 (y_{t-4} - \mu_{S_{t-4}}) + \varepsilon_t
$$

Each period, the regime transitions according to the following matrix of transition probabilities:

$$ P(S_t = s_t | S_{t-1} = s_{t-1}) =
\begin{bmatrix}
p_{00} & p_{10} \\
p_{01} & p_{11}
\end{bmatrix}
$$

where $p_{ij}$ is the probability of transitioning *from* regime $i$, *to* regime $j$.

The model class is `MarkovAutoregression` in the time-series part of `statsmodels`. In order to create the model, we must specify the number of regimes with `k_regimes=2`, and the order of the autoregression with `order=4`. The default model also includes switching autoregressive coefficients, so here we also need to specify `switching_ar=False` to avoid that.

After creation, the model is `fit` via maximum likelihood estimation. Under the hood, good starting parameters are found using a number of steps of the expectation maximization (EM) algorithm, and a quasi-Newton (BFGS) algorithm is applied to quickly find the maximum.

```python
# Get the RGNP data to replicate Hamilton
dta = pd.read_stata('https://www.stata-press.com/data/r14/rgnp.dta').iloc[1:]
dta.index = pd.DatetimeIndex(dta.date, freq='QS')
dta_hamilton = dta.rgnp

# Plot the data
dta_hamilton.plot(title='Growth rate of Real GNP', figsize=(12,3))

# Fit the model
mod_hamilton = sm.tsa.MarkovAutoregression(dta_hamilton, k_regimes=2, order=4, switching_ar=False)
res_hamilton = mod_hamilton.fit()
```

```python
res_hamilton.summary()
```

Markov Switching Model Results

  Dep. Variable:	           rgnp	           No. Observations:  	    131

  Model:	           MarkovAutoregression	   Log Likelihood     	 -181.263

  Date:	              Tue, 24 Dec 2019	     AIC                	  380.527

  Time:	                  14:58:45	         BIC                	  406.404

  Sample:	               04-01-1951	        HQIC               	  391.042

  	                     - 10-01-1984

  Covariance Type:	        approx

Regime 0 parameters

    	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  const	    -0.3588	     0.265	    -1.356	  0.175	    -0.877	     0.160

Regime 1 parameters

    	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  const	     1.1635	     0.075	    15.614	  0.000	     1.017	     1.310

Non-switching parameters

     	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  sigma2	     0.5914	     0.103	     5.761	  0.000	     0.390	     0.793

  ar.L1	      0.0135	     0.120	     0.112	  0.911	    -0.222	     0.249

  ar.L2	     -0.0575	     0.138	    -0.418	  0.676	    -0.327	     0.212

  ar.L3	     -0.2470	     0.107	    -2.310	  0.021	    -0.457	    -0.037

  ar.L4	     -0.2129	     0.111	    -1.926	  0.054	    -0.430	     0.004

Regime transition parameters

     	        coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  p[0->0]	     0.7547	     0.097	     7.819	  0.000	     0.565	     0.944

  p[1->0]	     0.0959	     0.038	     2.542	  0.011	     0.022	     0.170

Warnings:
[1] Covariance matrix calculated using numerical (complex-step) differentiation.

We plot the filtered and smoothed probabilities of a recession. Filtered refers to an estimate of the probability at time $t$ based on data up to and including time $t$ (but excluding time $t+1, ..., T$). Smoothed refers to an estimate of the probability at time $t$ using all the data in the sample.

For reference, the shaded periods represent the NBER recessions.

```python
fig, axes = plt.subplots(2, figsize=(7,7))
ax = axes[0]
ax.plot(res_hamilton.filtered_marginal_probabilities[0])
ax.fill_between(usrec.index, 0, 1, where=usrec['USREC'].values, color='k', alpha=0.1)
ax.set_xlim(dta_hamilton.index[4], dta_hamilton.index[-1])
ax.set(title='Filtered probability of recession')

ax = axes[1]
ax.plot(res_hamilton.smoothed_marginal_probabilities[0])
ax.fill_between(usrec.index, 0, 1, where=usrec['USREC'].values, color='k', alpha=0.1)
ax.set_xlim(dta_hamilton.index[4], dta_hamilton.index[-1])
ax.set(title='Smoothed probability of recession')

fig.tight_layout()
```

From the estimated transition matrix we can calculate the expected duration of a recession versus an expansion.

```python
print(res_hamilton.expected_durations)
```

    [ 4.07604746 10.42589382]

In this case, it is expected that a recession will last about one year (4 quarters) and an expansion about two and a half years.

### Kim, Nelson, and Startz (1998) Three-state Variance Switching

This model demonstrates estimation with regime heteroskedasticity (switching of variances) and no mean effect. The dataset can be reached at http://econ.korea.ac.kr/~cjkim/MARKOV/data/ew_excs.prn.

The model in question is:

$$
\begin{align}
y_t & = \varepsilon_t \\
\varepsilon_t & \sim N(0, \sigma_{S_t}^2)
\end{align}
$$

Since there is no autoregressive component, this model can be fit using the `MarkovRegression` class. Since there is no mean effect, we specify `trend='nc'`. There are hypothesized to be three regimes for the switching variances, so we specify `k_regimes=3` and `switching_variance=True` (by default, the variance is assumed to be the same across regimes).

```python
# Get the dataset
ew_excs = requests.get('http://econ.korea.ac.kr/~cjkim/MARKOV/data/ew_excs.prn').content
raw = pd.read_table(BytesIO(ew_excs), header=None, skipfooter=1, engine='python')
raw.index = pd.date_range('1926-01-01', '1995-12-01', freq='MS')

dta_kns = raw.loc[:'1986'] - raw.loc[:'1986'].mean()

# Plot the dataset
dta_kns[0].plot(title='Excess returns', figsize=(12, 3))

# Fit the model
mod_kns = sm.tsa.MarkovRegression(dta_kns, k_regimes=3, trend='nc', switching_variance=True)
res_kns = mod_kns.fit()
```

```python
res_kns.summary()
```

    C:\ProgramData\Anaconda3\lib\site-packages\statsmodels\base\model.py:1286: RuntimeWarning: invalid value encountered in sqrt
      bse_ = np.sqrt(np.diag(self.cov_params()))
    C:\ProgramData\Anaconda3\lib\site-packages\scipy\stats\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater
      return (self.a < x) & (x < self.b)
    C:\ProgramData\Anaconda3\lib\site-packages\scipy\stats\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less
      return (self.a < x) & (x < self.b)
    C:\ProgramData\Anaconda3\lib\site-packages\scipy\stats\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal
      cond2 = cond0 & (x <= self.a)

Markov Switching Model Results

  Dep. Variable:	           0	          No. Observations:  	    732

  Model:	           MarkovRegression	   Log Likelihood     	 1001.895

  Date:	            Tue, 24 Dec 2019	   AIC                	 -1985.790

  Time:	                14:59:00	       BIC                	 -1944.428

  Sample:	             01-01-1926	      HQIC               	 -1969.834

  	                   - 12-01-1986

  Covariance Type:	      approx

Regime 0 parameters

     	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  sigma2	     0.0012	     0.000	     6.675	  0.000	     0.001	     0.002

Regime 1 parameters

     	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  sigma2	     0.0040	     0.000	     8.477	  0.000	     0.003	     0.005

Regime 2 parameters

     	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  sigma2	     0.0311	     0.006	     5.539	  0.000	     0.020	     0.042

Regime transition parameters

     	        coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  p[0->0]	     0.9747	       nan	       nan	    nan	       nan	       nan

  p[1->0]	     0.0195	     0.012	     1.681	  0.093	    -0.003	     0.042

  p[2->0]	  2.354e-08	     0.004	  5.61e-06	  1.000	    -0.008	     0.008

  p[0->1]	     0.0253	     0.018	     1.408	  0.159	    -0.010	     0.061

  p[1->1]	     0.9688	     0.014	    68.322	  0.000	     0.941	     0.997

  p[2->1]	     0.0493	     0.040	     1.223	  0.221	    -0.030	     0.128

Warnings:
[1] Covariance matrix calculated using numerical (complex-step) differentiation.

Below we plot the probabilities of being in each of the regimes; only in a few periods is a high-variance regime probable.

```python
fig, axes = plt.subplots(3, figsize=(10,7))

ax = axes[0]
ax.plot(res_kns.smoothed_marginal_probabilities[0])
ax.set(title='Smoothed probability of a low-variance regime for stock returns')

ax = axes[1]
ax.plot(res_kns.smoothed_marginal_probabilities[1])
ax.set(title='Smoothed probability of a medium-variance regime for stock returns')

ax = axes[2]
ax.plot(res_kns.smoothed_marginal_probabilities[2])
ax.set(title='Smoothed probability of a high-variance regime for stock returns')

fig.tight_layout()
```

### Filardo (1994) Time-Varying Transition Probabilities

This model demonstrates estimation with time-varying transition probabilities. The dataset can be reached at http://econ.korea.ac.kr/~cjkim/MARKOV/data/filardo.prn.

In the above models we have assumed that the transition probabilities are constant across time. Here we allow the probabilities to change with the state of the economy. Otherwise, the model is the same Markov autoregression of Hamilton (1989).

Each period, the regime now transitions according to the following matrix of time-varying transition probabilities:

$$ P(S_t = s_t | S_{t-1} = s_{t-1}) =
\begin{bmatrix}
p_{00,t} & p_{10,t} \\
p_{01,t} & p_{11,t}
\end{bmatrix}
$$

where $p_{ij,t}$ is the probability of transitioning *from* regime $i$, *to* regime $j$ in period $t$, and is defined to be:

$$
p_{ij,t} = \frac{\exp\{ x_{t-1}' \beta_{ij} \}}{1 + \exp\{ x_{t-1}' \beta_{ij} \}}
$$

Instead of estimating the transition probabilities as part of maximum likelihood, the regression coefficients $\beta_{ij}$ are estimated. These coefficients relate the transition probabilities to a vector of pre-determined or exogenous regressors $x_{t-1}$.

```python
# Get the dataset
filardo = requests.get('http://econ.korea.ac.kr/~cjkim/MARKOV/data/filardo.prn').content
dta_filardo = pd.read_table(BytesIO(filardo), sep=' +', header=None, skipfooter=1, engine='python')
dta_filardo.columns = ['month', 'ip', 'leading']
dta_filardo.index = pd.date_range('1948-01-01', '1991-04-01', freq='MS')

dta_filardo['dlip'] = np.log(dta_filardo['ip']).diff()*100
# Deflated pre-1960 observations by ratio of std. devs.
# See hmt_tvp.opt or Filardo (1994) p. 302
std_ratio = dta_filardo['dlip']['1960-01-01':].std() / dta_filardo['dlip'][:'1959-12-01'].std()
dta_filardo['dlip'][:'1959-12-01'] = dta_filardo['dlip'][:'1959-12-01'] * std_ratio

dta_filardo['dlleading'] = np.log(dta_filardo['leading']).diff()*100
dta_filardo['dmdlleading'] = dta_filardo['dlleading'] - dta_filardo['dlleading'].mean()

# Plot the data
dta_filardo['dlip'].plot(title='Standardized growth rate of industrial production', figsize=(13,3))
plt.figure()
dta_filardo['dmdlleading'].plot(title='Leading indicator', figsize=(13,3));
```

The time-varying transition probabilities are specified by the `exog_tvtp` parameter.

Here we demonstrate another feature of model fitting - the use of a random search for MLE starting parameters. Because Markov switching models are often characterized by many local maxima of the likelihood function, performing an initial optimization step can be helpful to find the best parameters.

Below, we specify that 20 random perturbations from the starting parameter vector are examined and the best one used as the actual starting parameters. Because of the random nature of the search, we seed the random number generator beforehand to allow replication of the result.

```python
mod_filardo = sm.tsa.MarkovAutoregression(
    dta_filardo.iloc[2:]['dlip'], k_regimes=2, order=4, switching_ar=False,
    exog_tvtp=sm.add_constant(dta_filardo.iloc[1:-1]['dmdlleading']))

np.random.seed(12345)
res_filardo = mod_filardo.fit(search_reps=20)
```

```python
res_filardo.summary()
```

Markov Switching Model Results

  Dep. Variable:	           dlip	           No. Observations:  	    514

  Model:	           MarkovAutoregression	   Log Likelihood     	 -586.572

  Date:	              Tue, 24 Dec 2019	     AIC                	 1195.144

  Time:	                  15:01:25	         BIC                	 1241.808

  Sample:	               03-01-1948	        HQIC               	 1213.433

  	                     - 04-01-1991

  Covariance Type:	        approx

Regime 0 parameters

    	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  const	    -0.8659	     0.153	    -5.658	  0.000	    -1.166	    -0.566

Regime 1 parameters

    	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  const	     0.5173	     0.077	     6.706	  0.000	     0.366	     0.668

Non-switching parameters

     	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  sigma2	     0.4844	     0.037	    13.172	  0.000	     0.412	     0.556

  ar.L1	      0.1895	     0.050	     3.761	  0.000	     0.091	     0.288

  ar.L2	      0.0793	     0.051	     1.552	  0.121	    -0.021	     0.180

  ar.L3	      0.1109	     0.052	     2.136	  0.033	     0.009	     0.213

  ar.L4	      0.1223	     0.051	     2.418	  0.016	     0.023	     0.221

Regime transition parameters

        	           coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  p[0->0].tvtp0	     1.6494	     0.446	     3.702	  0.000	     0.776	     2.523

  p[1->0].tvtp0	    -4.3595	     0.747	    -5.833	  0.000	    -5.824	    -2.895

  p[0->0].tvtp1	    -0.9945	     0.566	    -1.758	  0.079	    -2.103	     0.114

  p[1->0].tvtp1	    -1.7702	     0.508	    -3.484	  0.000	    -2.766	    -0.775

Warnings:
[1] Covariance matrix calculated using numerical (complex-step) differentiation.

Below we plot the smoothed probability of the economy operating in a low-production state, and again include the NBER recessions for comparison.

```python
fig, ax = plt.subplots(figsize=(12,3))

ax.plot(res_filardo.smoothed_marginal_probabilities[0])
ax.fill_between(usrec.index, 0, 1, where=usrec['USREC'].values, color='gray', alpha=0.2)
ax.set_xlim(dta_filardo.index[6], dta_filardo.index[-1])
ax.set(title='Smoothed probability of a low-production state');
```

Using the time-varying transition probabilities, we can see how the expected duration of a low-production state changes over time:

```python
res_filardo.expected_durations[0].plot(
    title='Expected duration of a low-production state', figsize=(12,3));
```

During recessions, the expected duration of a low-production state is much higher than in an expansion.


# Cost Function and Backpropagation

Now, we're going to study an algorithm for fitting the parameters of a Neural Network, known as the Backpropagation algorithm. Before doing that, we will define the cost function

## 1. Cost Function

To define the cost function for Neural Networks, let's consider the following general neural network with $L$ layers for **classification problems**:

Moreover, we have $m$ training examples $\{(x^{(1)}, y^{(1)}), \dots (x^{(m)}, y^{(m)})\}$.

- For binary classification we would have $y\in\{0,1\}$, and $K=1$ output units.

- For multi-class classification we have $y\in\mathbb{R}^K$ and $K$ output units. For example, for $K=4$:
  $$
  y = \left[\begin{array}{c}
  1 \\
  0 \\
  0 \\
  0
  \end{array}\right], \left[\begin{array}{c}
  0 \\
  0 \\
  1 \\
  0
  \end{array}\right]
  $$

### What happens from layer $l$ to layer $l+1$ ($l\in\{1,\dots,L-1\}$)?

Recall that we introduce the intermediate variables $z_j^{(l+1)}$ for $j\in\{1, \dots, s_{l+1}\}$, defined as

$$
z_j^{(l+1)} = \Theta_{j0}^{(l)} + \Theta_{j0}^{(l)}a_1^{(l)} + \dots + \Theta_{js_l}^{(l)} a_{s_l}^{(l)}.
$$

Then, the $j$-th unit activation is defined as

$$
a_j^{(l+1)} = g(z_j^{(l+1)}),
$$

where $g$ is the sigmoid function.

Equivalently,

\begin{align}
z^{(l+1)} & = \Theta^{(l)} \left[\begin{array}{c} 1 \\ a^{(l)} \end{array}\right] \\
a^{(l+1)} & = g(z^{(l+1)}),
\end{align}

where

$$
\Theta^{(l)} = \left[
\begin{array}{cccc}
\Theta_{10}^{(l)}       & \Theta_{11}^{(l)}       & \dots  & \Theta_{1s_l}^{(l)}  \\
\Theta_{20}^{(l)}       & \Theta_{21}^{(l)}       & \dots  & \Theta_{2s_l}^{(l)}  \\
\vdots                  & \vdots                  & \ddots & \vdots               \\
\Theta_{s_{l+1}0}^{(l)} & \Theta_{s_{l+1}1}^{(l)} & \dots  & \Theta_{s_{l+1}s_l}^{(l)}
\end{array}\right],
$$

### What is the form of the hypothesis function?

Following the above analysis, in the output layer $L$:

$$\left\{
\begin{align}
z^{(L)} & = \Theta^{(L-1)} \left[\begin{array}{c} 1 \\ a^{(L-1)} \end{array}\right] \in\mathbb{R}^{K} \\
a^{(L)} & = h_{\Theta}(x) = g(z^{(L)})\in\mathbb{R}^{K}.
\end{align}\right.
$$

In the hidden layers $l = 2,\dots, L-1$:

$$\left\{
\begin{align}
z^{(l)} & = \Theta^{(l-1)} \left[\begin{array}{c} 1 \\ a^{(l-1)} \end{array}\right] \in\mathbb{R}^{s_l} \\
a^{(l)} & = g(z^{(l})\in\mathbb{R}^{s_l}.
\end{align}\right.
$$

In the input layer:

$$
a^{(1)} = x \in\mathbb{R}^{n}.
$$

### What is the form of the cost function?

The cost function we use for this class of neural networks is a generalization of the cost function for the logistic regression:

$$
J(\Theta) = -\frac{1}{m} \left[\sum_{t=1}^{m} \sum_{k=1}^{K} y^{(t)}_k \log(h_{\Theta}(x^{(t)})_k) + (1-y^{(t)}_k) \log(1 - h_{\Theta}(x^{(t)})_k)\right] + \frac{\lambda}{2m} \sum_{l=1}^{L-1} \sum_{i=1}^{s_{l+1}} \sum_{j=1}^{s_l} \left(\Theta_{ij}^{l}\right)^2
$$

Thus, in order to use a gradient based optimization method, we need to compute

$$
\frac{\partial}{\partial \Theta_{ij}^{(l)}} J(\Theta),
$$

for each $i\in\{1, \dots, s_{l+1}\}$, $j\in\{0, \dots, s_{l}\}$, and $l\in\{1, \dots, L-1\}$.

Defining the *cost for the example $t$ at the output $k$*, $c_k^{(t)}$ as:

$$
c_k^{(t)} = -(y^{(t)}_k \log(h_{\Theta}(x^{(t)})_k) + (1-y^{(t)}_k) \log(1 - h_{\Theta}(x^{(t)})_k),
$$

we have that

$$
J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \sum_{k=1}^{K} c_k^{(t)}\right] + \frac{\lambda}{2m} \sum_{l=1}^{L-1} \sum_{i=1}^{s_{l+1}} \sum_{j=1}^{s_l} \left(\Theta_{ij}^{(l)}\right)^2.
$$

## 2. Computing the Gradient

Before starting our analysis, let's note that for the sigmoid function

$$g(z) = \frac{1}{1+e^{-z}},$$

its derivative is

$$
g'(z) = \frac{e^{-z}}{(1+e^{-z})^2} = \frac{1+e^{-z} - 1}{(1+e^{-z})^2} = g(z) - g(z)^2 = g(z) (1 - g(z)).
$$

### First step: For $l=L-1$

Let $i\in\{1, \dots, s_L=K\}$ and $j\in\{0, \dots, s_{L-1}\}$. Then,

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-1)}} J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \sum_{k=1}^{K} \frac{\partial}{\partial \Theta_{ij}^{(L-1)}}c_k^{(t)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(L-1)} \mathcal{I}\{j>0\},
$$

where $\mathcal{I}(\cdot)$ stands for the indicator function.

Clearly,

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-1)}}c_k^{(t)} = 0,
$$

whenever $k \neq i$. On the other hand, using the chain rule

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-1)}}c_i^{(t)} = \frac{\partial}{\partial z_{i}^{(t,L)}}c_i^{(t)} \frac{\partial}{\partial \Theta_{ij}^{(L-1)}} z_{i}^{(t,L)},
$$

where ($h_{\Theta}(x^{(t)})_k=a_k^{(t,L)}=g(z_k^{(t,L)})$)

\begin{align}
\frac{\partial}{\partial z_{i}^{(t,L)}}c_i^{(t)} & = y^{(t)}_i \frac{1}{a^{(t,L)}_i} a^{(t,L)}_i (1 - a^{(t,L)}_i) + (1-y^{(t)}_i) \frac{1}{1 - a^{(t,L)}_i} (-a^{(t,L)}_i (1 - a^{(t,L)}_i)) \\
& = y^{(t)}_i (1 - a^{(t,L)}_i) - (1-y^{(t)}_i) a^{(t,L)}_i \\
& = a^{(t,L)}_i - y_i^{(t)} \\
& =: \delta_i^{(t,L)},
\end{align}

and $\frac{\partial}{\partial \Theta_{ij}^{(L-1)}} z_{i}^{(t,L)} = a_j^{(t,L-1)}$.

Hence,

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-1)}} J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \delta_i^{(t,L)} a_j^{(t,L-1)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(L-1)} \mathcal{I}\{j>0\}.
$$

### Second step: For $l=L-2$

Let $i\in\{1, \dots, s_{L-1}\}$ and $j\in\{0, \dots, s_{L-2}\}$. Then,

Then,

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-2)}} J(\Theta) = \frac{1}{m} \sum_{t=1}^{m} \frac{\partial}{\partial \Theta_{ij}^{(L-2)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(L-2)} \mathcal{I}\{j>0\}.
$$

It is worth to notice here that every unit in the output layer $L$ depends on $\Theta_{ij}^{(L-2)}$, for every $i, j, k$:

Hence, applying the chain rule

$$
 \frac{\partial}{\partial \Theta_{ij}^{(L-2)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] = \frac{\partial}{\partial z_{i}^{(t,L-1)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] \frac{\partial}{\partial \Theta_{ij}^{(L-2)}} z_{i}^{(t,L-1)}
$$

where

\begin{align}
\frac{\partial}{\partial z_{i}^{(t,L-1)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] & = \sum_{k=1}^{K}\frac{\partial}{\partial z_{k}^{(t,L)}}c_k^{(t)} \frac{\partial}{\partial a_{i}^{(t,L-1)}}z_k^{(t)} \frac{\partial}{\partial z_{i}^{(t,L-1)}}a_{i}^{(t,L-1)} \\
& = \left[\sum_{k=1}^{K} \delta_k^{(t, L)} \Theta_{ki}^{(L-1)}\right] a_i^{(t,L-1)} (1 - a_i^{(t,L-1)})  \\
& = (\Theta_{:i}^{(L-1)})^T \delta^{(t, L)} a_i^{(t,L-1)} (1 - a_i^{(t,L-1)}) \\
& =: \delta_i^{(t, L-1)},
\end{align}

where $(\Theta_{:i}^{(L-1)})^T = \left[\Theta_{1i}^{(L-1)} \quad \Theta_{2i}^{(L-1)} \quad \dots \quad \Theta_{s_{L}i}^{(L-1)}\right]$ is the $i$-th column of $\Theta^{(L-1)}$ transposed, and $\delta^{(t, L)} = \left[\delta_1^{(t, L)} \quad \delta_2^{(t, L)} \quad \dots \quad \delta_{K}^{(t, L)}\right]$.

Moreover, $\frac{\partial}{\partial \Theta_{ij}^{(L-2)}} z_{i}^{(t,L-1)} = a_j^{(t, L-2)}$.

Thus,

$$
\frac{\partial}{\partial \Theta_{ij}^{(L-2)}} J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \delta_i^{(t, L-1)} a_j^{(t, L-2)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(L-2)} \mathcal{I}\{j>0\}.
$$

### Inductive step

Suppose that we've calculated all the partial derivatives of the cost function w.r.t. the parameters in the layer $l+1$ as

$$
\frac{\partial}{\partial \Theta_{ij}^{(l+1)}} J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \delta_i^{(t, l+2)} a_j^{(t, l+1)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(l+1)} \mathcal{I}\{j>0\}.
$$

for all $i\in\{1, \dots, s_{l+2}\}$ and $j\in\{0, \dots, s_{l+1}\}$, with

\begin{align}
\delta_i^{(t, l+2)} &:= \frac{\partial}{\partial z_{i}^{(t,L-1)}} \left[\sum_{k=1}^{K} c_k^{(t)}\right] \\
& = (\Theta_{:i}^{(l+2)})^T \delta^{(t, l+3)} a_i^{(t,l+2)} (1 - a_i^{(t,l+2)}).
\end{align}

Now, for the layer $l$, let $i\in\{1, \dots, s_{l+1}\}$ and $j\in\{0, \dots, s_{l}\}$. Then,

$$
\frac{\partial}{\partial \Theta_{ij}^{(l)}} J(\Theta) = \frac{1}{m} \sum_{t=1}^{m} \frac{\partial}{\partial \Theta_{ij}^{(l)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(l)} \mathcal{I}\{j>0\}.
$$

Applying the chain rule, we have

$$
\frac{\partial}{\partial \Theta_{ij}^{(l)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] = \frac{\partial}{\partial z_{i}^{(t,l+1)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] \frac{\partial}{\partial \Theta_{ij}^{(l)}} z_{i}^{(t,l+1)},
$$

where

\begin{align}
\frac{\partial}{\partial z_{i}^{(t,l+1)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] &=\sum_{p=1}^{s_{l+2}} \left( \frac{\partial}{\partial z_{p}^{(t,l+2)}}\left[\sum_{k=1}^{K} c_k^{(t)}\right] \frac{\partial}{\partial a_{i}^{(t,l+1)}} z_{p}^{(t,l+2)} \right) \frac{\partial}{\partial z_{i}^{(t,l+1)}} a_{i}^{(t,l+1)}  \\
& = \left[\sum_{p=1}^{s_{l+2}} \delta_p^{(t, l+2)} \Theta_{pi}^{(l+1)}\right] a_i^{(t,l+1)} (1 - a_i^{(t,l+1)}) \\
& = (\Theta_{:i}^{(l+1)})^T \delta^{(t, l+2)} a_i^{(t,l+1)} (1 - a_i^{(t,l+1)}) \\
& =: \delta_i^{(t,l+1)},
\end{align}

and $\frac{\partial}{\partial \Theta_{ij}^{(l)}} z_{i}^{(t,l+1)} = a_j^{(t,l)}$.

Thus,

$$
\frac{\partial}{\partial \Theta_{ij}^{(l)}} J(\Theta) = \frac{1}{m} \left[\sum_{t=1}^{m} \delta_i^{(t, l+1)} a_j^{(t, l)}\right] + \frac{\lambda}{m} \Theta_{ij}^{(l)} \mathcal{I}\{j>0\}.
$$

## 3. Backpropagation algorithm

Based on the analysis of the gradients above, one can come up with the following backpropagation algorithm:

Given the training set $\{(x^{(1)}, y^{(1)}), \dots (x^{(m)}, y^{(m)})\}$:

> - Set $D^{(l)} = 0\in\mathbb{R}^{s_{l+1} \times (1 + s_l)}$ for $l\in\{1, \dots, L-1\}$.
> - For $t=1, \dots, m$:
>     - Set $a^{(1)} = x^{(t)}$.
>     - Forward propagation algorithm:
>       $$\left\{
        \begin{align}
        z^{(l)} & = \Theta^{(l-1)} \left[\begin{array}{c} 1 \\ a^{(l-1)} \end{array}\right] \in\mathbb{R}^{s_l} \\
        a^{(l)} & = g(z^{(l})\in\mathbb{R}^{s_l}.
        \end{align}\right.
        $$
>       for $l\in\{2, \dots, L\}$.
>     - Compute $\delta^{(L)} = a^{(L)} - y^{(t)} \in \mathbb{R}^{s_L}$.
>     - Compute $\delta^{(l)} = (\Theta_{:,1:}^{(l)})^T \delta^{(l+1)} \circ a^{(l)} \circ (1 - a^{(l)})$, for $l\in\{L-1, \dots, 2\}$.
>     - Update $D^{(l)} := D^{(l)} + \delta^{(l+1)} \left[1 \quad (a^{(l)})^T\right]$, for $l\in\{1, \dots, L-1\}$.
> - Update $D^{(l)} := \frac{1}{m} \left(D^{(l)} + \lambda \left[0_{s_{l+1} \times 1} \quad | \quad 1_{s_{l+1} \times s_l}\right] \circ \theta^{(l)}\right)$, for $l\in\{1, \dots, L-1\}$.

Finally, we get that

$$
\frac{\partial}{\partial \Theta_{ij}^{(l)}} J(\Theta) = D^{(l)}_{ij}
$$

<footer id="attribution" style="float:right; color:#808080; background:#fff;">
Created with Jupyter by Esteban Jiménez Rodríguez. Based on the content of the Machine Learning course offered through coursera by Prof. Andrew Ng.
</footer>


# Symbolic Calculations of the Self Replicator Model
(c) This work is licensed under a Creative Commons Attribution License CC-BY 4.0. All code contained herein is licensed under an MIT license

```python

```

```python
import numpy as np
import pandas as pd
import altair as alt
import growth.viz
import sympy as sp
colors, palette = growth.viz.altair_style()
```

## Introduction

## The Governing Dynamics

## Steady-State Solutions

```python
# Define symbols
gamma_max = sp.Symbol(r'{{\gamma_{max}}}')
nu_max = sp.Symbol(r'{{\nu_{max}}}')
phi_R = sp.Symbol(r'{{\phi_R}}')
Kd = sp.Symbol(r'{{K_D}}')
cAA = sp.Symbol(r'{{c_{AA}}}')
mu = sp.Symbol(r'\mu')
t = sp.Symbol('t')
# Define the system
cAA_eq = (nu_max * (1 - phi_R) / mu) - 1
gamma = gamma_max * cAA_eq / (cAA_eq + Kd)
mu_eq = phi_R * gamma
mu_soln = sp.solve(mu_eq - mu, mu)
```

```python
mu_soln[0].simplify()
```

$\displaystyle \frac{- {{\gamma_{max}}} {{\phi_R}} + {{\nu_{max}}} {{\phi_R}} - {{\nu_{max}}} + \sqrt{- 4 {{K_D}} {{\gamma_{max}}} {{\nu_{max}}} {{\phi_R}}^{2} + 4 {{K_D}} {{\gamma_{max}}} {{\nu_{max}}} {{\phi_R}} + {{\gamma_{max}}}^{2} {{\phi_R}}^{2} + 2 {{\gamma_{max}}} {{\nu_{max}}} {{\phi_R}}^{2} - 2 {{\gamma_{max}}} {{\nu_{max}}} {{\phi_R}} + {{\nu_{max}}}^{2} {{\phi_R}}^{2} - 2 {{\nu_{max}}}^{2} {{\phi_R}} + {{\nu_{max}}}^{2}}}{2 \left({{K_D}} - 1\right)}$

```python

```

```python
dmu_dphiR = sp.diff(mu_soln[0], phi_R)
opt_phiR = sp.solve(dmu_dphiR, phi_R)

```

```python
opt_phiR[0]
```

$\displaystyle \frac{{{\nu_{max}}} \left(- 2 {{K_D}} {{\gamma_{max}}} + {{\gamma_{max}}} + {{\nu_{max}}}\right) + \sqrt{{{K_D}} {{\gamma_{max}}} {{\nu_{max}}}} \left(- {{\gamma_{max}}} + {{\nu_{max}}}\right)}{- 4 {{K_D}} {{\gamma_{max}}} {{\nu_{max}}} + {{\gamma_{max}}}^{2} + 2 {{\gamma_{max}}} {{\nu_{max}}} + {{\nu_{max}}}^{2}}$

```python
# Define the symbols
M = sp.Symbol('M')
Mr = sp.Symbol('{{M_R}}')
M0 = sp.Symbol('{{M_0}}')
Mr0 = sp.Symbol(r'{{M_{R,0}}}')
cAA_0 = sp.Symbol(r'{{c_{AA,0}}}')
gamma = cAA / (cAA + Kd)

# Define the fns
M_fn = sp.Function('{{M_f}}')
Mr_fn = sp.Function('{{Mr_f}}')
cAA_fn = sp.Function('{{cAA_f}}')

# Define the system
system = [sp.Eq(M_fn(t).diff(t), gamma * Mr_fn(t)),
          sp.Eq(Mr_fn(t).diff(t), phi_R * M_fn(t)),
          sp.Eq(cAA_fn(t).diff(t),  ((M_fn(t) - Mr_fn(t))/M_fn(t)) * nu_max - gamma * (Mr_fn(t) / M_fn(t)) - (cAA_fn(t) / M_fn(t))* M_fn(t))]
ics = {M_fn(0):M0,
       Mr_fn(0):Mr,
       cAA_fn(0):cAA_0}
sys_soln = sp.dsolve(system, [M_fn(t), Mr_fn(t), cAA_fn(t)], ics=ics)
```

```python
sys_soln[0]
```

$\displaystyle \operatorname{dCAA_{dt}}{\left(t \right)} = {{c_{AA,0}}} - \frac{t^{2} \left(- M {{\nu_{max}}} \left({{K_D}} + {{c_{AA}}}\right) + {{K_D}} {{M_R}} {{\nu_{max}}} + {{M_R}} {{c_{AA}}} \left({{\nu_{max}}} + {{c_{AA}}} + 1\right)\right)}{2 M \left({{K_D}} + {{c_{AA}}}\right)}$

```python

```


## Introduction

This notebook accompanies the paper "Impact of non-parabolic electronic band structure on the optical, defect, and transport properties of photovoltaic materials". It reproduces some of the key results from the paper and includes supplementary information. It makes extensive use of the [effmass](http://github.com/lucydot/effmass) package which is available to download on github.

The effmass package contains the [data files](https://github.com/lucydot/effmass/tree/master/paper/data) needed to reproduce all results in the paper. Here we focus upon a subset of that data to calculate:
- Effective mass and non-parabolicity of CdTe
- Burstein Moss shift and optical effective mass in MAPI

In both cases we use data calculated using the HSE06 functional with spin-orbit coupling. However, the notebook can be easily adapted to explore other materials and levels of theory if required.

## Imports and preamble

```python
# plots displayed within notebook
%matplotlib inline

# import scientific libraries
import math
import matplotlib.pyplot as plt
import numpy as np

# import modules from the effmass package
from effmass import inputs, analysis, extrema, outputs, dos, ev_to_hartree
```

## Effective mass and non-parabolicity of CdTe

### Settings
First we use the `inputs` module to create a `Settings` object. The `extrema_search_depth` attribute tells us how far from the CBM/VBM we would like to search for the bandstructure minima/maxima. The `energy_range` attribute sets the energy range for each band `Segment`.

```python
settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.25)
```

### Import bandstructure data
We now use the `inputs` module to create a `DataVasp` object which automatically imports the vasp data from the files specified. We manually specify how many k-points to ignore at the start of each file. These are the k-points which are included as part of the non-self-consistent bandstructure calculation, but which do not form part of the bandstructure itself.

```python
data = inputs.DataVasp("./paper/data/CdTe/HSE06_SoC/OUTCAR","./paper/data/CdTe/HSE06_SoC/PROCAR", ignore=216)
```

To import FHI-Aims data we use a similar command, this time specifying the path to the directory containing `geometry.in`, `control.in` and `band1.out` / `band2.out`. The rest of the workflow is independant of the DFT calculator used, although the DOS-related functionality is only supported for Vasp.

```python
Aims_data = inputs.DataAims("./tests/data_aims/Ge_sp_aims/")
```

### Generate band segments
We can now use our `Settings` and `Data` objects to generate a list of `Segment` objects using the `extrema` module.

```python
segments = extrema.generate_segments(settings,data)
```

Each Segment has a string method which gives the energy of the Segment extrema (referenced to the VBM) and the start- and end- points of the Segment in reciprocal space.

```python
str(segments[-1])
```

    '0.03 eV; [0.5 0.5 0.5]-->[0.49  0.49  0.519]'

Note: If `effmass` is struggling to find the extrema you are interested in, then you can pass a specific band index and kpoint index to the `generate_segments` function using the optional `bk` argument.

### Visualise band segments
We can visualise the `Segment` objects created using the `outputs` module. The plot is annotated with the `Segment`'s direction and segments argument index.

```python
outputs.plot_segments(data,settings,segments)
```

    (<Figure size 576x576 with 1 Axes>, <AxesSubplot:>)

```python
outputs.plot_segments(data,settings,[segments[-1],segments[-3]])
```

### Calculate parabolic effective mass
There are a number of methods associated with each `Segment` object. We can use these to calculate the different definitions of effective mass, assuming a parabolic dispersion $E= \frac{\hbar^2k^2}{2m^*}$.

```python
segments[-1].five_point_leastsq_effmass()
```

    0.09061386126453852

```python
segments[-1].finite_difference_effmass()
```

    0.092155278777688338

```python
segments[-1].weighted_leastsq_effmass()
```

    0.095174661409246172

We can use inbuilt documentation to find out more about a particular method.

```python
analysis.Segment.weighted_leastsq_effmass?
```

### Calculate parameters for the Kane dispersion
The alpha parameter quantifies the extent of non-parabolicity in the kane dispersion $\frac{\hbar^2k^2}{2m^*_0} = E(1 + \alpha E) $
where $m^*_0$ is the mass at the band edge ($E=0$).

```python
segments[-1].alpha() # note that atomic units are used (hartree^-1)
```

    27.782410846735118

```python
segments[-1].kane_mass_band_edge()
```

    0.089285653580707308

### Summarise results

The `print_results` function in the `outputs` module summarises the results for a segment.

```python
outputs.print_results(segments[-1], data, settings)
```

## Burstein Moss shift and optical effective mass in MAPI

### Settings, Data and Segments
As in the previous example we create a `Settings` object and `Data` object then use these to create a list of `Segment` objects.

```python
settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.75)
data = inputs.DataVasp("./paper/data/MAPI/HSE06_SoC/OUTCAR","./paper/data/MAPI/HSE06_SoC/PROCAR",ignore=216)
segments = extrema.generate_segments(settings,data)
```

### Kane dispersion parameters
To calculate the non-parabolic burstein-moss shift we need to know the electron alpha parameter and bandedge transport effective mass. First, let's see the segments we have generated

```python
outputs.plot_segments(data,settings,segments)
```

We want to calculate the kane dispersion parameters for `segments[-4]`, `segments[-5]` and `segments[-6]`. These parameters depend upon the `Settings.energy_range` attribute and the order of the polyfit used for calculating the transport mass. We can adjust the `energy_range` and polyfit order until we get a good fit to data.

```python
# energy_range=0.25, polyfit_order=6 for optimal fitting in 001 direction

settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.25)
segments = extrema.generate_segments(settings, data)
outputs.print_results(segments[-4],data,settings,polyfit_order=6)
```

```python
# energy_range=0.5, polyfit_order=6 for optimal fitting in 101 direction

settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.5)
segments = extrema.generate_segments(settings, data)
outputs.print_results(segments[-5],data,settings,polyfit_order=6)
```

```python
# energy_range=0.4, polyfit_order=4 for optimal fitting in 111 direction

settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.4)
segments = extrema.generate_segments(settings, data)
outputs.print_results(segments[-6],data,settings,polyfit_order=4)
```

### Burstein Moss shift

The Burstein Moss shift is calculated using analytic expression $\Delta_{BM} =\frac{\hbar^2}{2m^*}(3\pi^2n_e)^{2/3}$ where the effective mass $m^*$ is constant (in the case of a parabolic dispersion) or takes the form $m^*(E) = m_0^*(1+2 \alpha E)$ (for a Kane dispersion).

```python
def burstein_moss_parabolic(mass,concentration):
    # the expression comes from the fermi energy for a given concentration, assuming parabolic dispersion
    # the fermi wavevector is simply gotten from considering volume of sphere in reciprocal space / volume of each eigenstate (factor 2 for spin)
    concentration = concentration * ((5.29E-9)**3) # convert cm-3 --> bohr-3
    return (((3*math.pi*math.pi*concentration)**(2/3))/(2*mass))/ev_to_hartree

def burstein_moss_kane(concentration,mass,alpha):
    # this expression is an adaptation of the one above where the mass is now dependant upon the shift m_t = m_o(1+2 \alpha E)
    concentration = concentration * ((5.29E-9)**3) # convert cm-3 --> bohr-3
    return (analysis._solve_quadratic(2*alpha,1,[-((3*math.pi*math.pi*concentration)**(2/3)/(2*mass))])[0])/ev_to_hartree
```

We take a mean average of the alpha value across the three directions

```python
average_alpha = (2.214+1.499+0.16)/(3*(ev_to_hartree)) # convert back to atomic units
```

The geometric average of the mass is calculated for electrons and holes. Unlike silicon which has 6 equivalent minima and a degeneracy of 6, the minima here are between $\Gamma-R$ which has a multiplicity of 1. These average values are then combined into a reduced mass.

```python
def dos_average(m1,m2,m3,degeneracy=1):
    # density of states mass which is used instead of conductivity effective mass as electron concentration populates entire 3D brillouin zone
    # this perhaps needs a weighting factor to account for not mx,my,mz
    return ((m1*m2*m3)**(1/3))*(degeneracy**(2/3))

def reduced_mass(m_e,m_h):
    return 1/((1/m_e)+(1/m_h))

# effective mass at bandedge calculated from bandstructure
em1 = 0.19
em2 = 0.10
em3 = 0.18
hm1 = 0.23
hm2 = 0.10
hm3 = 0.12

average_mass=reduced_mass(dos_average(em1,em2,em3),dos_average(hm1,hm2,hm3))

```

Calculate the Burstein Moss shifts for concentration range $1 \times 10^{16}$ to $ 3 \times 10^{20}$

```python
concentrations = np.logspace(16,np.log10(3E20),100)
parabolic_bandshift = [burstein_moss_parabolic(average_mass,x) for x in concentrations]
kane_bandshift = [burstein_moss_kane(x,average_mass,average_alpha) for x in concentrations]
```

### Import density of states data

Note: DOS-related functionality is support for Vasp only.

We can compare these values to density of states data. We start by parsing the DOSCAR file.

```python
data.parse_DOSCAR("./paper/data/MAPI/HSE06_SoC/DOSCAR")
```

We can use this data and the `dos` module to calculate the band filling level for a given concentration. We also need to supply the volume of the unit cell in $\unicode{x212B} ^3$.

```python
volume = 251.13
dos_bandshift = [dos.electron_fill_level(data, volume, x, dos.find_dos_CBM_index(data))for x in concentrations]
```

### Compare results

Let's plot the three bandshift results:

```python
fig = plt.figure()
plt.plot(concentrations,parabolic_bandshift, "-",label="parabolic",color="black")
plt.plot(concentrations,kane_bandshift,"--",label="Kane",color="black")
plt.plot(concentrations,dos_bandshift, ":",label="DFT DOS",color="black")
plt.legend(prop={'size': 8},loc=2)
plt.xlabel(r"concentration cm$^{-3}$")
plt.ylabel("E (eV)")
plt.tight_layout()
plt.savefig("burstein_moss_MAPI_hybrid_SoC.pdf")
plt.xlim([0,3E20])
plt.ylim([0,2.5])
```

### Calculate optical effective mass as a function of concentration
Our results show that the Kane dispersion is a good approximation to the density of states data (which makes no assumptions about the band dispersions). We can calculate an optical effective mass:
$\begin{equation}
\frac{1}{m_o} = \frac{\sum_{l} \int f(E_k(k),T) \frac{\delta^2 E_k(k)}{\delta k^2} dk}{\sum_{l} \int f(E_k(k),T) dk}
\end{equation}$

with $E(K)$ set to the Kane dispersion. The Fermi level in the Fermi-Dirac distribution $f(E_k(k),T)$ is set to the burstein-moss shift as calculated using the Kane dispersion.

```python
settings = inputs.Settings(extrema_search_depth=0.075, energy_range=0.25)  # kane dispersion valid up to energy_range 0.25eV
segments = extrema.generate_segments(settings, data)
concentrations = np.logspace(16,np.log10(2E19),100) # if we set the concentration higher we exceed energies where the kane dispersion is valid and receive a warning

optical_mass_111 = [segments[-6].optical_effmass_kane_dispersion(fermi_level=data.CBM+burstein_moss_kane(x,average_mass,average_alpha),alpha=0.16/ev_to_hartree,mass_bandedge=0.155,upper_limit=0.1) for x in concentrations]
optical_mass_110 = [segments[-5].optical_effmass_kane_dispersion(fermi_level=data.CBM+burstein_moss_kane(x,average_mass,average_alpha),alpha=1.499/ev_to_hartree,mass_bandedge=0.098,upper_limit=0.1) for x in concentrations]
optical_mass_100 = [segments[-4].optical_effmass_kane_dispersion(fermi_level=data.CBM+burstein_moss_kane(x,average_mass,average_alpha),alpha=2.214/ev_to_hartree,mass_bandedge=0.156,upper_limit=0.1) for x in concentrations]

## Plot results

fig,ax1 = plt.subplots()

ax1.plot(np.log10(concentrations),optical_mass_111,":",label="(111) ",color="black")
ax1.plot(np.log10(concentrations),optical_mass_110,"-.",label="(110) ",color="black")
ax1.plot(np.log10(concentrations),optical_mass_100,"--",label="(100) ",color="black")

ax1.set_xticks([16,17,18,19])
ax1.set_xticklabels([r"$10^{16}$",r"$10^{17}$",r"$10^{18}$",r"$10^{19}$"])
ax1.set_xlim([16,np.log10(2E19)])
ax1.set_ylim([0,0.4])

ax1.set_xlabel(r"concentration (cm$^{-3}$)")
ax1.set_ylabel(r"optical $\frac{m^*}{m_e}$")
```

```python

```

```python

```


# Automated Model Reduction with AutoReduce

Assumptions such as quasi-steady state approximation (QSSA) and conservation laws are common in modeling of biological systems. [AutoReduce](https://github.com/ayush9pandey/autoReduce) is a Python package that can perform model reduction using QSSA and conservation laws on given SBML models. In addition to model reduction, AutoReduce can also perform sensitivity analysis and deterministic simulations for SBML models.

This notebook gives a short introduction to AutoReduce by importing an SBML model and performing model reduction on the given model.

# Detailed Chemical Reaction Network (CRN):

We have the SBML file called "gene_expression.xml" which models the expression of two genes according to the following reactions:

Transcription:

$ DNA + RNAP \leftrightharpoons C_1 $

$ C_1 \rightarrow mRNA + DNA + RNAP $

Translation:

$ mRNA + Ribosome \leftrightharpoons C_2 $

$C_2 \rightarrow Protein + mRNA + Ribosome $

Degradation:

$ mRNA + RNAase \leftrightharpoons C_3 $

$ C_3 \rightarrow RNAase $

$ Protein \rightarrow \varnothing $

$ mRNA \rightarrow \varnothing $

The SBML file may be generated using any SBML model builder tool such as BioCRNpyler, COPASI, iBioSim, or others. We load this SBML file in AutoReduce and apply model reduction techniques:

# Package Installation

If you do not have AutoReduce installed in your environment, uncomment the following line to install:

```python
# !pip install autoreduce
```

## Check if it worked by importing the packages:

```python
from autoreduce import *
import numpy as np
from sympy import symbols
```

# Load SBML in AutoReduce:

```python
from autoreduce.converters import load_sbml
sys = load_sbml('gene_expression.xml', outputs = ['protein_X'])
```

## Explore the System object

AutoReduce stores models using the following syntax for each model:

$ \dot{x} = f(x, params)$

$ y = Cx$ (if linear)

or

$ y = h(x, params)$

Here, $x$ are the state variables of the model (the list of species, if the model is a CRN), the params are the list of parameters (the reaction rates, if the model is a CRN). The outputs $y$ are the measured variables of the system.

```python
sys.x
```

    [dna_X,
     protein_RNAP_machinery,
     rna_X,
     complex_dna_X_protein_RNAP_machinery_,
     protein_Ribo_machinery,
     protein_X,
     complex_protein_Ribo_machinery_rna_X_,
     protein_RNAase_machinery,
     dna_cellular_processes,
     rna_cellular_processes,
     complex_dna_cellular_processes_protein_RNAP_machinery_,
     protein_cellular_processes,
     complex_protein_Ribo_machinery_rna_cellular_processes_,
     complex_protein_RNAase_machinery_rna_X_,
     complex_complex_protein_Ribo_machinery_rna_X__protein_RNAase_machinery_,
     complex_protein_RNAase_machinery_rna_cellular_processes_,
     complex_complex_protein_Ribo_machinery_rna_cellular_processes__protein_RNAase_machinery_]

```python
# sys.f
sys.params
```

    [kb__,
     ku_strong_,
     ktx_strong_,
     ku_weak_,
     ktl__,
     kb__transcription,
     ku__transcription,
     ktx__transcription,
     kb__translation,
     ku__translation,
     ktl__translation,
     ku__,
     kdeg__rna_degredation_mm,
     kdil__]

```python
sys.x_init
```

    [0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0]

```python
sys.C
```

    array([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
            0.]])

# Set up model reduction:

```python
from autoreduce.utils import get_reducible
sys_reduce = get_reducible(sys)
sys_reduce.nstates_tol_min = 3 # Minimum number of states tbat you want in the reduced model
```

## Conservation Laws

```python
G, P, C1, T, R, C2, E, C3, X = sys.x
conserved_quantities = [P + C1 - P_tot, R + C2 - R_tot, E + C3 - E_tot]
states_to_eliminate = [1,4,6]
f_cons = sys_reduce.set_conservation_laws(conserved_quantities, states_to_eliminate)
```

```python

```


# Bayesian Parametric Regression

    Notebook version: 1.3 (Sep 26, 2016)

    Author: Jerónimo Arenas García (jarenas@tsc.uc3m.es)
            Jesús Cid-Sueiro (jesus.cid@uc3m.es)

    Changes: v.1.0 - First version
             v.1.1 - ML Model selection included
             v.1.2 - Some typos corrected
             v.1.3 - Rewriting text, reorganizing content, some exercises.

    Pending changes: * Include regression on the stock data

```python
# Import some libraries that will be necessary for working with data and displaying plots

# To visualize plots in the notebook
%matplotlib inline
from IPython import display

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import scipy.io       # To read matlab files
import pylab
import time
```

## 1. Model-based parametric regression

### 1.1. The regression problem.

Given an observation vector ${\bf x}$, the goal of the regression problem is to find a function $f({\bf x})$ providing *good* predictions about some unknown variable $s$. To do so, we assume that a set of *labelled* training examples, $\{{\bf x}^{(k)}, s^{(k)}\}_{k=1}^K$ is available.

The predictor function should make good predictions for new observations ${\bf x}$ not used during training. In practice, this is tested using a second set (the *test set*) of labelled samples.

NOTE: In the following, we will use capital letters, ${\bf X}$, $S$, ..., to denote random variables, and lower-case letters ${\bf x}$, s, ..., to the denote the values they can take. When there is no ambigüity, we will remove subindices of the density functions, $p_{{\bf X}, S}({\bf x}, s)= p({\bf x}, s)$ to simplify the mathematical notation.

### 1.2. Model-based parametric regression

Model-based regression methods assume that all data in the training and test dataset habe been generated by some stochastic process. In parametric regression, we assume that the probability distribution generating the data has a known parametric form, but the values of some parameters are unknown.

In particular, in this notebook we will assume the target variables in all pairs $({\bf x}^{(k)}, s^{(k)})$ from the training and test sets have been generated independently from some posterior distribution $p(s| {\bf x}, {\bf w})$, were ${\bf w}$ is some unknown parameter. The training dataset is used to estimate ${\bf w}$.

Once $p(s|{\bf x},{\bf w})$ is known or can be estimated, Estimation Theory can be applied to estimate $s$ for any input ${\bf x}$. For instance, any of these classical estimates can be used:

* Maximum A Posterior (MAP): $\qquad\hat{s}_{\text{MAP}} = \arg\max_s p(s| {\bf x}, {\bf w})$
* Minimum Mean Square Error (MSE): $\qquad\hat{s}_{\text{MSE}} = \mathbb{E}\{S |{\bf x}, {\bf w}\}$

#### 1.3.1. Maximum Likelihood (ML) parameter estimation

One way to estimate ${\bf w}$ is to apply the maximum likelihood principle: take the value ${\bf w}_\text{ML}$ maximizing the joint distribution of the target variables given the inputs and given ${\bf w}$, i.e.

$$
{\bf w}_\text{ML} = \arg\max_{\bf w} p({\bf s}|{\bf X}, {\bf w})
$$

where ${\bf s} = \left(s^{(1)}, \dots, s^{(K)}\right)^\top$ is the vector of target variables and ${\bf X} = \left({\bf x}^{(1)}, \dots, {\bf x}^{(K)}\right)^\top$ is the input matrix.

NOTE: Since the training data inputs are known, all probability density functions and expectations in the remainder of this notebook will be conditioned on ${\bf X}$. To simplify the mathematical notation, from now on we will remove ${\bf X}$ from all conditions. Keep in mind that, in any case, all probabilities and expectations may depend on ${\bf X}$ implicitely.

#### 1.3.2. The Gaussian case

A particularly interesting case arises when the data model is Gaussian:

$$p(s|{\bf x}, {\bf w}) =
    \frac{1}{\sqrt{2\pi}\sigma_\varepsilon}
    \exp\left(-\frac{(s-{\bf w}^\top{\bf z})^2}{2\sigma_\varepsilon^2}\right)
$$

where ${\bf z}=T({\bf x})$ is a vector with components which can be computed directly from the observed variables. Such expression includes a linear regression model, where ${\bf z} = [1; {\bf x}]$, as well as any other non-linear model as long as it can be expressed as a "linear in the parameters" model.

In that case, it can be shown that the likelihood function $p({\bf s}| {\bf w})$ ($\equiv p({\bf s}| {\bf X}, {\bf w})$) is given by

$$
p({\bf s}| {\bf w})
    = \left(\frac{1}{\sqrt{2\pi}\sigma_\varepsilon}\right)^K
      \exp\left(-\frac{1}{2\sigma_\varepsilon^2}\|{\bf s}-{\bf Z}{\bf w}\|^2\right)
$$

which is maximum for the Least Squares solution

$$
{\bf w}_{ML} = ({\bf Z}^\top{\bf Z})^{-1}{\bf Z}^\top{\bf s}
$$

### 1.4. Limitations of the ML estimators.

Since the ML estimation is equivalent to the LS solution under a Gaussian data model, it has the same drawbacks of LS regression. In particular, ML estimation is prone to overfiting. In general, if the number of parameters (i.e. the dimension of ${\bf w}$) is large in relation to the size of the training data, the predictor based on the ML estimate may have a small square error over the training set but a large error over the test set. Therefore, in practice, som cross validation procedures is required to keep the complexity of the predictor function under control depending on the size of the training set.

## 2. Bayesian Regression

One of the reasons why the ML estimate is prone to overfitting is that the prediction function uses ${\bf w}_\text{ML}$ without taking into account how much uncertain the true value of ${\bf w}$ is.

Bayesian methods utilize such information but considering ${\bf w}$ as a random variable with some prior distribution $p({\bf w})$. The posterior distribution $p({\bf w}|{\bf s})$ will be our measure of the uncertainty about the true value of the model parameters.

In fact, this posterior distribution is a key component of the predictor function. Indeed, the minimum MSE estimate can be computed as

$$
\hat{s}_\text{MSE}
   = \mathbb{E}\{s|{\bf s}, {\bf x}\}
   = \int \mathbb{E}\{s|{\bf w}, {\bf s}, {\bf x}\} p({\bf w}|{\bf s}) d{\bf w}
$$

Since the samples are i.i.d. $\mathbb{E}\{s|{\bf w}, {\bf s}, {\bf x}\} = \mathbb{E}\{s|{\bf w}, {\bf x}\}$ and, thus

$$
\hat{s}_\text{MSE}
   = \int \mathbb{E}\{s|{\bf w}, {\bf x}\} p({\bf w}|{\bf s}) d{\bf w}
$$

Noting that $\mathbb{E}\{s|{\bf w}, {\bf s}, {\bf x}\}$ is the minimum MSE prediction for a given value of ${\bf w}$, we observe that the Bayesian predictor is a weighted sum of these predictions, weighted by its posterior probability (density) of being the correct one.

### 2.1. Posterior weight distribution

We will express our a priori belief of models using a prior distribution $p({\bf w})$. Then we can infer the a posteriori distribution using Bayes' rule:

   $$p({\bf w}|{\bf s}) = \frac{p({\bf s}|{\bf w})~p({\bf w})}{p({\bf s})}$$

   Where:
   - $p({\bf s}|{\bf w})$: is the likelihood function
   - $p({\bf w})$: is the prior distribution of the weights (assumptions are needed here)
   - $p({\bf s})$: is the marginal distribution of the observed data, which could be obtained integrating the expression in the numerator

The previous expression can be interpreted in a rather intuitive way:

   - Since ${\bf w}$ are the parameters of the model, $p({\bf w})$ express our belief about which models should be preferred over others before we see any data. For instance, since parameter vectors with small norms produce smoother curves, we could assign (a priori) a larger pdf value to models with smaller norms

   - The likelihood function $p({\bf s}|{\bf w})$ tells us how well the observations can be explained by a particular model

   - Finally, the posterior distribution $p({\bf w}|{\bf s})$ expresses the estimated goodness of each model (i.e., each parameter vector ${\bf w}$) taking into consideration both the prior and the likelihood of $\bf w$. Thus, a model with large $p({\bf w})$ would have a low posterior value if it offers a poor explanation of the data (i.e., if $p({\bf s}|{\bf w})$ is small), whereas models that fit well with the observations would get emphasized

The posterior distribution of weights opens the door to working with several models at once. Rather thank keeping the estimated best model according to a certain criterion, we can now use all models parameterized by ${\bf w}$, assigning them different degrees of confidence according to $p({\bf w}|{\bf s})$.

#### 2.1.1. A Gaussian Prior

Since each value of ${\bf w}$ determines a regression functions, by stating a prior distributions over the weights we state also a prior distribution over the space of regression functions.

For instance, we will consider a particular example in which we assume a Gaussian prior for the weights given by:

$${\bf w} \sim {\cal N}\left({\bf 0},{\pmb \Sigma}_{p} \right)$$

#### Example

Assume that the true target variable is related to the input observations through the equation
$$
s = {\bf w}^\top{\bf z} + \varepsilon
$$
where ${\bf z} = T({\bf x})$ is a polynomial transformation of the input, $\varepsilon$ is a Gaussian noise variable and ${\bf w}$ some unknown parameter vector.

Assume a Gausian prior weigh distribution, ${\bf w} \sim {\cal N}\left({\bf 0},{\pmb \Sigma}_{p} \right)$. For each parameter vector ${\bf w}$, there is a polynomial $f({\bf x}) = {\bf w}^\top {\bf z}$ associated to it. Thus, by drawing samples from $p({\bf w})$ we can generate and plot their associated polynomial functions. This is carried out in the following example.

You can check the effect of modifying the variance of the prior distribution.

```python
n_points = 20
n_grid = 200
frec = 3
std_n = 0.2
degree = 3
nplots = 20

#Prior distribution parameters
sigma_eps = 0.1
mean_w = np.zeros((degree+1,))
sigma_w = 0.03  ### Try increasing this value
var_w = sigma_w * np.eye(degree+1)

X_tr = 3 * np.random.random((n_points,1)) - 0.5
S_tr = - np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)

xmin = np.min(X_tr)
xmax = np.max(X_tr)
X_grid = np.linspace(xmin-0.2*(xmax-xmin), xmax+0.2*(xmax-xmin),n_grid)
S_grid = - np.cos(frec*X_grid) #Noise free for the true model

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(X_tr,S_tr,'b.',markersize=10)

for k in range(nplots):

    #Draw weigths fromt the prior distribution
    w_iter = np.random.multivariate_normal(mean_w,var_w)
    S_grid_iter = np.polyval(w_iter,X_grid)
    ax.plot(X_grid,S_grid_iter,'g-')

ax.set_xlim(xmin-0.2*(xmax-xmin), xmax+0.2*(xmax-xmin))
ax.set_ylim(S_tr[0]-2,S_tr[-1]+2)
ax.set_xlabel('$x$')
ax.set_ylabel('$s$')
plt.show()
```

### 2.2. Summary

Summarizing, the steps to design a Bayesian parametric regresion algorithm are the following:

1. Assume a parametric data model $p(s| {\bf x},{\bf w})$ and a prior distribution $p({\bf w})$.
2. Using the data model and the i.i.d. assumption, compute $p({\bf s}|{\bf w})$.
3. Applying the bayes rule, compute the posterior distribution $p({\bf w}|{\bf s})$.
4. Compute the MSE estimate of $s$ given ${\bf x}$.

## 3. Bayesian regression for a Gaussian model.

We will apply the above steps to derive a Bayesian regression algorithm for a Gaussian model.

### 3.1. Step 1: The Gaussian model.

Let as assume that the likelihood function is given by the Gaussian model described in Sec. 1.3.2.

$$
s~|~{\bf w} \sim {\cal N}\left({\bf z}^\top{\bf w}, \sigma_\varepsilon^2 {\bf I} \right)
$$

and that the prior is also Gaussian

$$
{\bf w} \sim {\cal N}\left({\bf 0},{\pmb \Sigma}_{p} \right)
$$

### 3.2. Step 2: Complete data likelihood

Using the i.i.d. assumption,

$$
{\bf s}~|~{\bf w} \sim {\cal N}\left({\bf Z}{\bf w},\sigma_\varepsilon^2 {\bf I} \right)
$$

### 3.3. Step 3: Posterior weight distribution

The posterior distribution of the weights can be computed using the Bayes rule

$$p({\bf w}|{\bf s}) = \frac{p({\bf s}|{\bf w})~p({\bf w})}{p({\bf s})}$$

Since both $p({\bf s}|{\bf w})$ and $p({\bf w})$ follow a Gaussian distribution, we know also that the joint distribution and the posterior distribution of ${\bf w}$ given ${\bf s}$ are also Gaussian. Therefore,

$${\bf w}~|~{\bf s} \sim {\cal N}\left({\bf w}_\text{MSE}, {\pmb\Sigma}_{\bf w}\right)$$

After some algebra, it can be shown that mean and the covariance matrix of the distribution are:

$${\pmb\Sigma}_{\bf w} = \left[\frac{1}{\sigma_\varepsilon^2} {\bf Z}^{\top}{\bf Z} + {\pmb \Sigma}_p^{-1}\right]^{-1}$$

$${\bf w}_\text{MSE} = {\sigma_\varepsilon^{-2}} {\pmb\Sigma}_{\bf w} {\bf Z}^\top {\bf s}$$

#### Exercise 1:

Consider the dataset with one-dimensional inputs given by

```python
# True data parameters
w_true = 3
std_n = 0.4

# Generate the whole dataset
n_max = 64
X_tr = 3 * np.random.random((n_max,1)) - 0.5
S_tr =  w_true * X_tr + std_n * np.random.randn(n_max,1)
```

Fit a Bayesian linear regression model assuming ${\bf z}={\bf x}$ and

```python
# Model parameters
sigma_eps = 0.1
mean_w = np.zeros((1,))
sigma_p = 1e6 * np.eye(1)
```

To do so, compute the posterior weight distribution using the first $k$ samples in the complete dataset, for $k = 1,2,4,8,\ldots, 128$. Draw all these posteriors along with the prior distribution in the same plot.

```python
# No. of points to analyze
n_points = [1, 2, 4, 8, 16, 32, 64]

# Prepare plots
w_grid = np.linspace(2.7, 3.4, 5000)   # Sample the w axiss
plt.figure()

# Plot the prior distribution
# p = <FILL IN>

plt.plot(w_grid, p.flatten(),'g-')

for k in n_points:

    # Select the first k samples
    Zk = X_tr[0:k, :]
    Sk = S_tr[0:k]

    # Compute the parameters of the posterior distribution
    # Sigma_w = <FILL IN>
    # w_MSE = <FILL IN>

    w_MSE = np.array(w_MSE).flatten()

    # Draw weights from the posterior distribution
    # p = <FILL IN>

    p = p.flatten()

    plt.plot(w_grid, p,'g-')
    plt.fill_between(w_grid, 0, p, alpha=0.8, edgecolor='#1B2ACC', facecolor='#089FFF',
        linewidth=1, antialiased=True)
    plt.xlim(w_grid[0], w_grid[-1])
    plt.ylim(0, np.max(p))
    plt.xlabel('$w$')
    plt.ylabel('$p(w|s)$')

    display.clear_output(wait=True)
    display.display(plt.gcf())
    time.sleep(0.5)

# Remove the temporary plots and fix the last one
display.clear_output(wait=True)
plt.show()

# Print the weight estimate based on the whole dataset
print w_MSE
```

#### Exercise 2:

Note that, in the example above, the model assumptions are correct: the target variables have been generated by a linear model with noise standard deviation `sigma_n` which is exactly equal to the value assumed by the model, stored in variable `sigma_eps`. Check what happens if we take `sigma_eps=4*sigma_n` or `sigma_eps=sigma_n/4`.

* Does the algorithm fails in that cases?
* What differences can you observe with respect to the ideal case `sigma_eps=sigma_n`?

### 3.4. Step 4: MSE estimate

Noting that
$$
\mathbb{E}\{s|{\bf w}, {\bf x}\} = {\bf w}^\top {\bf z}
$$

we can write

$$
\hat{s}_\text{MSE}
   = \int {\bf w}^\top {\bf z} p({\bf w}|{\bf s}) d{\bf w}
   = \left(\int {\bf w} p({\bf w}|{\bf s}) d{\bf w}\right)^\top {\bf z}
   = {\bf w}_\text{MSE}^\top {\bf z}
$$

where
$$
{\bf w}_\text{MSE}
   = \int {\bf w} p({\bf w}|{\bf s}) d{\bf w}
   = {\sigma_\varepsilon^{-2}} {\pmb\Sigma}_{\bf w} {\bf Z}^\top {\bf s}
$$

Therefore, in the Gaussian case, the weighted integration of prediction function is equivalent to apply a unique model, with weights ${\bf w}_\text{MSE}$.

#### Exercise 3:

Plot the minimum MSE predictions of $s$ for inputs $x$ in the interval [-1, 3].

```python
# <SOL>
# </SOL>
```

### 3.5 Maximum likelihood vs Bayesian Inference.  Making predictions

   - Following an ML approach, we retain a single model, ${\bf w}_{ML} = \arg \max_{\bf w} p({\bf s}|{\bf w})$. Then, the predictive distribution of the target value for a new point would be obtained as:

   $$p({s^*}|{\bf w}_{ML},{\bf x}^*) $$

   For the generative model of Section 3.1.2 (additive i.i.d. Gaussian noise), this distribution is:

   $$p({s^*}|{\bf w}_{ML},{\bf x}^*) = \frac{1}{\sqrt{2\pi\sigma_\varepsilon^2}} \exp \left(-\frac{\left(s^* - {\bf w}_{ML}^\top {\bf z}^*\right)^2}{2 \sigma_\varepsilon^2} \right)$$

      * The mean of $s^*$ is just the same as the prediction of the LS model, and the same uncertainty is assumed independently of the observation vector (i.e., the variance of the noise of the model).

      * If a single value is to be kept, we would probably keep the mean of the distribution, which is equivalent to the LS prediction.

   - Using Bayesian inference, we retain all models. Then, the inference of the value $s^* = s({\bf x}^*)$ is carried out by mixing all models, according to the weights given by the posterior distribution.

   \begin{align}p({s^*}|{\bf x}^*,{\bf s})
                         & = \int p({s^*}~|~{\bf w},{\bf x}^*) p({\bf w}~|~{\bf s}) d{\bf w}\end{align}

   where:

      * $p({s^*}|{\bf w},{\bf x}^*) = \displaystyle\frac{1}{\sqrt{2\pi\sigma_\varepsilon^2}} \exp \left(-\frac{\left(s^* - {\bf w}^\top {\bf z}^*\right)^2}{2 \sigma_\varepsilon^2} \right)$
      * $p({\bf w}~|~{\bf s})$: Is the posterior distribution of the weights, that can be computed using Bayes' Theorem.

The following fragment of code draws random vectors from $p({\bf w}|{\bf s})$, and plots the corresponding regression curves along with the training points. Compare these curves with those extracted from the prior distribution of ${\bf w}$ and with the LS solution.

```python
n_points = 15
n_grid = 200
frec = 3
std_n = 0.2
degree = 12
nplots = 6

#Prior distribution parameters
sigma_eps = 0.1
mean_w = np.zeros((degree+1,))
sigma_p = .3 * np.eye(degree+1)

X_tr = 3 * np.random.random((n_points,1)) - 0.5
S_tr = - np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)
X_grid = np.linspace(-.5,2.5,n_grid)
S_grid = - np.cos(frec*X_grid) #Noise free for the true model

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(X_tr,S_tr,'b.',markersize=10)

# Compute matrix with training input data for the polynomial model
Z = []
for x_val in X_tr.tolist():
    Z.append([x_val[0]**k for k in range(degree+1)])
Z=np.asmatrix(Z)

#Compute posterior distribution parameters
Sigma_w = np.linalg.inv(np.dot(Z.T,Z)/(sigma_eps**2) + np.linalg.inv(sigma_p))
posterior_mean = Sigma_w.dot(Z.T).dot(S_tr)/(sigma_eps**2)
posterior_mean = np.array(posterior_mean).flatten()

for k in range(nplots):

    #Draw weights from the posterior distribution
    w_iter = np.random.multivariate_normal(posterior_mean,Sigma_w)
    #Note that polyval assumes the first element of weight vector is the coefficient of
    #the highest degree term. Thus, we need to reverse w_iter
    S_grid_iter = np.polyval(w_iter[::-1],X_grid)
    ax.plot(X_grid,S_grid_iter,'g-')

#We plot also the least square solution
w_LS = np.polyfit(X_tr.flatten(), S_tr.flatten(), degree)
S_grid_iter = np.polyval(w_LS,X_grid)
ax.plot(X_grid,S_grid_iter,'m-',label='LS regression')

ax.set_xlim(-.5,2.5)
ax.set_ylim(S_tr[0]-2,S_tr[-1]+2)
ax.legend(loc='best')
plt.show()
```

#### Posterior distribution of the target

   - Since $f^* = f({\bf x}^*) = {\bf w}^\top{\bf z}$, $f^*$ is also a Gaussian variable whose posterior mean and variance can be calculated as follows:

   $$\mathbb{E}\{{{\bf z}^*}^\top {\bf w}~|~{\bf s}, {\bf z}^*\} = {{\bf z}^*}^\top \mathbb{E}\{{\bf w}|{\bf s}\} = {\sigma_\varepsilon^{-2}} {{\bf z}^*}^\top  {\pmb\Sigma}_{\bf w} {\bf Z}^\top {\bf s}$$

   $$\text{Cov}\left[{{\bf z}^*}^\top {\bf w}~|~{\bf s}, {\bf z}^*\right] = {{\bf z}^*}^\top \text{Cov}\left[{\bf w}~|~{\bf s}\right] {{\bf z}^*} = {{\bf z}^*}^\top {\pmb \Sigma}_{\bf w} {{\bf z}^*}$$

   - Therefore, $f^*~|~{\bf s}, {\bf x}^* \sim {\cal N}\left({\sigma_\varepsilon^{-2}} {{\bf z}^*}^\top  {\pmb\Sigma}_{\bf w} {\bf Z}^\top {\bf s}, {{\bf z}^*}^\top {\pmb \Sigma}_{\bf w} {{\bf z}^*} \right)$

   - Finally, for $s^* = f^* + \varepsilon^*$, the posterior distribution is  $s^*~|~{\bf s}, {\bf z}^* \sim {\cal N}\left({\sigma_\varepsilon^{-2}} {{\bf z}^*}^\top  {\pmb\Sigma}_{\bf w} {\bf Z}^\top {\bf s}, {{\bf z}^*}^\top {\pmb \Sigma}_{\bf w} {{\bf z}^*} + \sigma_\varepsilon^2\right)$

```python
n_points = 15
n_grid = 200
frec = 3
std_n = 0.2
degree = 12
nplots = 6

#Prior distribution parameters
sigma_eps = 0.1
mean_w = np.zeros((degree+1,))
sigma_p = .5 * np.eye(degree+1)

X_tr = 3 * np.random.random((n_points,1)) - 0.5
S_tr = - np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)
X_grid = np.linspace(-1,3,n_grid)
S_grid = - np.cos(frec*X_grid) #Noise free for the true model

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(X_tr,S_tr,'b.',markersize=10)

#Compute matrix with training input data for the polynomial model
Z = []
for x_val in X_tr.tolist():
    Z.append([x_val[0]**k for k in range(degree+1)])
Z=np.asmatrix(Z)

#Compute posterior distribution parameters
Sigma_w = np.linalg.inv(np.dot(Z.T,Z)/(sigma_eps**2) + np.linalg.inv(sigma_p))
posterior_mean = Sigma_w.dot(Z.T).dot(S_tr)/(sigma_eps**2)
posterior_mean = np.array(posterior_mean).flatten()

#Plot the posterior mean
#Note that polyval assumes the first element of weight vector is the coefficient of
#the highest degree term. Thus, we need to reverse w_iter
S_grid_iter = np.polyval(posterior_mean[::-1],X_grid)
ax.plot(X_grid,S_grid_iter,'g-',label='Predictive mean, BI')

#Plot confidence intervals for the Bayesian Inference
std_x = []
for el in X_grid:
    x_ast = np.array([el**k for k in range(degree+1)])
    std_x.append(np.sqrt(x_ast.dot(Sigma_w).dot(x_ast)[0,0]))
std_x = np.array(std_x)
plt.fill_between(X_grid, S_grid_iter-std_x, S_grid_iter+std_x,
    alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF',
    linewidth=4, linestyle='dashdot', antialiased=True)

#We plot also the least square solution
w_LS = np.polyfit(X_tr.flatten(), S_tr.flatten(), degree)
S_grid_iter = np.polyval(w_LS,X_grid)
ax.plot(X_grid,S_grid_iter,'m-',label='LS regression')

ax.set_xlim(-1,3)
ax.set_ylim(S_tr[0]-2,S_tr[-1]+2)
ax.legend(loc='best')
```

Not only do we obtain a better predictive model, but we also have confidence intervals (error bars) for the predictions.

## 4 Maximum evidence model selection

We have already addressed with Bayesian Inference the following two issues:

   - For a given degree, how do we choose the weights?

   - Should we focus on just one model, or can we use several models at once?

However, we still needed some assumptions: a parametric model (i.e., polynomial function and a priori degree selection) and several parameters needed to be adjusted.

Though we can recur to cross-validation, Bayesian inference opens the door to other strategies.

   - We could argue that rather than keeping single selections of these parameters, we could use simultaneously several sets of parameters (and/or several parametric forms), and average them in a probabilistic way ... (like we did with the models)

   - We will follow a simpler strategy, selecting just the most likely set of parameters according to an ML criterion

### 4.1 Model evidence

The evidence of a model is defined as

$$L = p({\bf s}~|~{\cal M})$$

where ${\cal M}$ denotes the model itself and any free parameters it may have. For instance, for the polynomial model we have assumed so far, ${\cal M}$ would represent the degree of the polynomia, the variance of the additive noise, and the a priori covariance matrix of the weights

Applying the Theorem of Total probability, we can compute the evidence of the model as

$$L = \int p({\bf s}~|~{\bf f},{\cal M}) p({\bf f}~|~{\cal M}) d{\bf f} $$

For the linear model $f({\bf x}) = {\bf w}^\top{\bf z}$, the evidence can be computed as

$$L = \int p({\bf s}~|~{\bf w},{\cal M}) p({\bf w}~|~{\cal M}) d{\bf w} $$

It is important to notice that these probability density functions are exactly the ones we computed on the previous section. We are just making explicit that they depend on a particular model and the selection of its parameters. Therefore:

   - $p({\bf s}~|~{\bf w},{\cal M})$ is the likelihood of ${\bf w}$

   - $p({\bf w}~|~{\cal M})$ is the a priori distribution of the weights

### 4.2 Model selection via evidence maximization

   - As we have already mentioned, we could propose a prior distribution for the model parameters, $p({\cal M})$, and use it to infer the posterior. However, this can be very involved (usually no closed-form expressions can be derived)

   - Alternatively, maximizing the evidence is normally good enough

   $${\cal M}_{ML} = \arg\max_{\cal M} p(s~|~{\cal M})$$

Note that we are using the subscript 'ML' because the evidence can also be referred to as the likelihood of the model

### 4.3 Example: Selection of the degree of the polynomia

For the previous example we had (we consider a spherical Gaussian for the weights):

   - ${\bf s}~|~{\bf w},{\cal M}~\sim~{\cal N}\left({\bf Z}{\bf w},\sigma_\varepsilon^2 {\bf I} \right)$

   - ${\bf w}~|~{\cal M}~\sim~{\cal N}\left({\bf 0},\sigma_p^2 {\bf I} \right)$

In this case, $p({\bf s}~|~{\cal M})$ follows also a Gaussian distribution, and it can be shown that

   - $L = p({\bf s}~|~{\cal M}) = {\cal N}\left({\bf 0},\sigma_p^2 {\bf Z} {\bf Z}^\top+\sigma_\varepsilon^2 {\bf I} \right)$

If we just pursue the maximization of $L$, this is equivalent to maximizing the log of the evidence

$$\log(L) = -\frac{M}{2} \log(2\pi) -{\frac{1}{2}}\log\mid\sigma_p^2 {\bf Z} {\bf Z}^\top+\sigma_\varepsilon^2 {\bf I}\mid - \frac{1}{2} {\bf s}^\top \left(\sigma_p^2 {\bf Z} {\bf Z}^\top+\sigma_\varepsilon^2 {\bf I}\right)^{-1} {\bf s}$$

where $M$ denotes the length of vector ${\bf z}$ (the degree of the polynomia minus 1).

The following fragment of code evaluates the evidence of the model as a function of the degree of the polynomia

```python
from math import pi

n_points = 15
frec = 3
std_n = 0.2
max_degree = 12

#Prior distribution parameters
sigma_eps = 0.2
mean_w = np.zeros((degree+1,))
sigma_p = 0.5

X_tr = 3 * np.random.random((n_points,1)) - 0.5
S_tr = - np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)

#Compute matrix with training input data for the polynomial model
Z = []
for x_val in X_tr.tolist():
    Z.append([x_val[0]**k for k in range(degree+1)])
Z=np.asmatrix(Z)

#Evaluate the posterior evidence

logE = []
for deg in range(max_degree):
    Z_iter = Z[:,:deg+1]
    logE_iter = -((deg+1)*np.log(2*pi)/2) \
                -np.log(np.linalg.det((sigma_p**2)*Z_iter.dot(Z_iter.T) + (sigma_eps**2)*np.eye(n_points)))/2 \
                -S_tr.T.dot(np.linalg.inv((sigma_p**2)*Z_iter.dot(Z_iter.T) + (sigma_eps**2)*np.eye(n_points))).dot(S_tr)/2
    logE.append(logE_iter[0,0])

plt.plot(np.array(range(max_degree))+1,logE)
plt.xlabel('Polynomia degree')
plt.ylabel('log evidence')

```

The above curve may change the position of its maximum from run to run.

We conclude the notebook by plotting the result of the Bayesian inference for M=6

```python
n_points = 15
n_grid = 200
frec = 3
std_n = 0.2
degree = 5 #M-1
nplots = 6

#Prior distribution parameters
sigma_eps = 0.1
mean_w = np.zeros((degree+1,))
sigma_p = .5 * np.eye(degree+1)

X_tr = 3 * np.random.random((n_points,1)) - 0.5
S_tr = - np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)
X_grid = np.linspace(-1,3,n_grid)
S_grid = - np.cos(frec*X_grid) #Noise free for the true model

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(X_tr,S_tr,'b.',markersize=10)

#Compute matrix with training input data for the polynomial model
Z = []
for x_val in X_tr.tolist():
    Z.append([x_val[0]**k for k in range(degree+1)])
Z=np.asmatrix(Z)

#Compute posterior distribution parameters
Sigma_w = np.linalg.inv(np.dot(Z.T,Z)/(sigma_eps**2) + np.linalg.inv(sigma_p))
posterior_mean = Sigma_w.dot(Z.T).dot(S_tr)/(sigma_eps**2)
posterior_mean = np.array(posterior_mean).flatten()

#Plot the posterior mean
#Note that polyval assumes the first element of weight vector is the coefficient of
#the highest degree term. Thus, we need to reverse w_iter
S_grid_iter = np.polyval(posterior_mean[::-1],X_grid)
ax.plot(X_grid,S_grid_iter,'g-',label='Predictive mean, BI')

#Plot confidence intervals for the Bayesian Inference
std_x = []
for el in X_grid:
    x_ast = np.array([el**k for k in range(degree+1)])
    std_x.append(np.sqrt(x_ast.dot(Sigma_w).dot(x_ast)[0,0]))
std_x = np.array(std_x)
plt.fill_between(X_grid, S_grid_iter-std_x, S_grid_iter+std_x,
    alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF',
    linewidth=4, linestyle='dashdot', antialiased=True)

#We plot also the least square solution
w_LS = np.polyfit(X_tr.flatten(), S_tr.flatten(), degree)
S_grid_iter = np.polyval(w_LS,X_grid)
ax.plot(X_grid,S_grid_iter,'m-',label='LS regression')

ax.set_xlim(-1,3)
ax.set_ylim(S_tr[0]-2,S_tr[-1]+2)
ax.legend(loc='best')
```

We can check, that now the model also seems quite appropriate for LS regression, but keep in mind that selection of such parameter was itself carried out using Bayesian inference.

```python

```