## Imports

```python
#Imports
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import math
import pandas as pd
import seaborn as sns
import scipy as sp
import sympy as sm
import plotly.graph_objects as go
import plotly.express as px
# from google.colab import files
from mpl_toolkits import mplot3d

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster
from sklearn.preprocessing import StandardScaler

#Warnings
import warnings
warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)

#Sinonimos
sym = sm
ip = sp.interpolate
la = np.linalg
cos = np.cos
sin = np.sin
e = np.e
pi = np.pi
cm = mpl.cm
Voronoi = sp.spatial.Voronoi
voronoi_plot_2d = sp.spatial.voronoi_plot_2d

#Styles
plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = [16, 7]
plt.rcParams['axes.grid'] = True
mpl.rcParams['axes.facecolor'] = 'F2D7D5'
mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=['c', 'k', 'r'])
np.set_printoptions(precision=5, suppress=True)
```

## Guía 1
Interpolación (funciones de Tchebyshev)

```python
def T(n, a = -1, b = 1):
    '''
    Devuelve el polinomio de Chebyshev de n raíces.
    Se le puede extraer los ceros con T(n).r
    '''
    if n==0:
        return (np.poly1d([1]))
    if n==1:
        return (np.poly1d([1,0]))
    else:
        return (2*np.poly1d([1,0])* T(n-1) - T(n-2))

def cheb_zeros(n):
    '''
    Devuelve los ceros del Chebyshev de grado n como array de np
    '''
    zeros = np.polynomial.chebyshev.chebpts1(n)
    return np.array(zeros)

def cheb_zeros_plot(n):
    '''
    Devuelve los ceros de Chebyshev y los grafica con su Polinomio
    '''
    x = np.linspace(-1, 1, 200)
    zeros = np.polynomial.chebyshev.chebpts1(n)
    y = np.zeros(n)
    T = np.cos(n * np.arccos(x))
    plt.plot(zeros, y, lw=0, marker='o', ms=10)
    plt.plot(zeros, y, lw=3, color='grey')
    plt.plot(x, T, color='b')
    plt.grid()
    return np.array(zeros)

def domain_change(zeros, x_min, x_max):
    ''' Devuelve el array de zeros trasladado al nuevo dominio. zeros debe ser un np.array.
    Se puede usar directamente con np.polynomial.chebyshev.chebpts1(n) '''
    return x_min + (zeros + 1) * (x_max - x_min) / 2
```

## Guía 2
Resolución numérica de ecuaciones

```python
def Biseccion(f, a, b, n, tol=False, preview=False, p=False):
    '''
    Aproxima el valor de una raíz de f en el intervalo [a, b] utilizando el método de bisección, n veces.
    Si el ultimo intervalo es menor a tol, se devuelve el punto medio de dicho intervalo sin llegar a la iteración n
    Tanto p o preview = True van mostrando los valores para cada ciclo. Si es falso, la función devuelve sin mostrar nada.
    '''
    a = a
    b = b
    tol = tol
    for k in range(n):
        f_a = f(a)
        f_b = f(b)
        c = (a+b)/2
        f_c = f(c)
        prod = f_a * f_c
        if f_c == 0:
            return c
        if tol:
            max_err = abs(b - a)
            if max_err < tol:
                return c
        if prod > 0:
            if preview or p is True:
                print(f'Paso {k+1}: Intervalo = [{a} ; {b}]. Raíz ≈ {c}, Prod = +')
            a = c
        elif prod < 0:
            if preview or p is True:
                print(f'Paso {k+1}: Intervalo = [{a} ; {b}]. Raíz ≈ {c}, Prod = -')
            b = c
    return c

def RegulaFalsi(f, a, b, n, tol=False, preview=False, p=False):
    '''
    Aproxima el valor de una raíz de f en el intervalo [a, b] utilizando el método de bisección, n veces.
    Si el ultimo intervalo es menor a tol, se devuelve el punto medio de dicho intervalo sin llegar a la iteración n
    Tanto p o preview = True van mostrando los valores para cada ciclo. Si es falso, la función devuelve sin mostrar nada.
    '''
    a = a
    b = b
    tol = tol
    q = 0
    for k in range(n):
        f_a = f(a)
        f_b = f(b)
        c = (a * f_b - b * f_a)/(f_b-f_a)
        f_c = f(c)
        prod = f_a * f_c
        if f_c == 0:
            return c
        if tol:
            max_err = abs(b - a)
            if max_err < tol:
                return c
        if prod > 0:
            if preview or p is True:
                print(f'Paso {k+1}: Intervalo = [{a} ; {b}]. Raíz ≈ {c}, Prod = +')
            a = c
        elif prod < 0:
            if preview or p is True:
                print(f'Paso {k+1}: Intervalo = [{a} ; {b}]. Raíz ≈ {c}, Prod = -')
            b = c
        q = c
    if tol:
        print('No se alcanzó la tolerancia para ', n, 'iteraciones')
    return q

def NewtonRaphson(f, df, x0, n, tol, p=False, all_roots = False):
    tol = tol
    xn = x0
    roots = {}
    if p:
        print('x0 =', xn)
    for k in range(n):
        f_n = f(xn)
        if abs(f_n) < tol:
            if p:
                print('Raíz:', xn, '. Iteraciones:', k)
            if all_roots:
                return roots
            return xn
        df_n = df(xn)
        if df_n == 0:
            raise ValueError('df(xn) debe ser != 0')
        if all_roots:
            roots[k] = xn
        xn = xn - f_n/df_n
        if p:
            print('x'+str(k+1) + ' =', xn)
    print('No se supero la tolerancia después de ', n, 'iteraciones.')
    if all_roots:
        return roots
    return xn

def Secante(f, x0, x1, n, tol, p=False, all_roots=False):
    tol = tol
    x0 = x0
    x1 = x1
    roots = {}
    for k in range(n):
        f_0 = f(x0)
        f_1 = f(x1)
        x2 = x1 - (f_1/(f_1-f_0)*(x1-x0))
        f_2 = f(x2)
        if all_roots:
            roots[k] = x2
        if abs(f_2) < tol:
            if p:
                print('Raíz:', x2, '. Iteraciones:', k)
            if all_roots:
                return roots
            return x2
        if p:
            print(
            f'x{k}: {x0}\nx{k+1}: {x1}\nx{k+2}: {x2}\n ---------------------------------------------')
        x0 = x1
        x1 = x2

    print('No se supero la tolerancia después de ', n, 'iteraciones.')
    if all_roots:
        return roots
    return x2

def PuntoFijo(g, x0, n, tol, p=False, all_roots=False):
    x = x0
    if p:
        print('x'+str(0)+' =', x)
    if all_roots:
        all_roots = []
        all_roots.append(x)
    for k in range(n):
        x = g(x)
        if p:
            print('x'+str(k+1)+' =', x)
        if all_roots:
            all_roots.append(x)
        if abs(x-g(x)) < tol:
            if all_roots:
                return all_roots
            return x
    print('No se alcanzó la tolerancia. Último x:', x)
    if all_roots:
        return all_roots
    return x

def GaussSeidel(A, b, X0, n, tol, conv = False, p = False, all_roots = False):
    D = np.diag(np.diag(A))
    L = np.tril(A, -1)
    U = np.triu(A, 1)
    DL = D + L
    DLinv = la.inv(DL)
    X = X0
    if all_roots:
        all_roots = {}
        all_roots[0] = X
    Bgs = - DLinv @ U
    Cgs = DLinv @ b
    if conv:
        lambdas = la.eigvals(Bgs)
        if all(np.abs(l) < 1 for l in lambdas):
            print('El método de Gauss-Seidel converge para este sistema. Autovalores:\n', lambdas)
        else:
            print('Gauss-Seidel no converge para este sistema. Autovalors:\n', lambdas)
        print('--------------------------------------------------------------')
    if p:
        print('X' + str(0) + ' =\n', X)
    for k in range(1, n + 1):
        old_X = X
        X = Bgs @ old_X + Cgs
        if p:
            print('X' + str(k) + ' =\n', X)
        if all_roots:
            all_roots[k] = X
        if la.norm(X - old_X) < tol:
            if all_roots:
                return all_roots
            return X
    print('No se alcanzó la tolerancia. Último X =\n', X)
    if all_roots:
        return all_roots
    return X

def Jacobi(A, b, X0, n, tol, conv = False, p = False, all_roots = False):
    D = np.diag(np.diag(A))
    L = np.tril(A, -1)
    U = np.triu(A, 1)
    D_inv = la.inv(D)
    X = X0
    if all_roots:
        all_roots = {}
        all_roots[0] = X
    Bj = - D_inv @ (L + U)
    Cj = D_inv @ b
    if conv:
        lambdas = la.eigvals(Bj)
        if all(np.abs(l) < 1 for l in lambdas):
            print('El método de Jacobi converge para este sistema. Autovalores:\n', lambdas)
        else:
            print('Jacobi no converge para este sistema. Autovalors:\n', lambdas)
        print('--------------------------------------------------------------')
    if p:
        print('X' + str(0) + ' =\n', X)
    for k in range(1, n + 1):
        old_X = X
        X = Bj @ old_X + Cj
        if p:
            print('X' + str(k) + ' =\n', X)
        if all_roots:
            all_roots[k] = X
        if la.norm(X - old_X) < tol:
            if all_roots:
                return all_roots
            return X
    print('No se alcanzó la tolerancia. Último X =\n', X)
    if all_roots:
        return all_roots
    return X

def NR2D(F, D, x0, y0, N, tol):
    X0 = np.array([x0, y0])
    X1 = X0 - np.linalg.solve(J(X0[0], X0[1]), F(X0[0], X0[1]))
    i = 1
    print(f'i = {i}. X{i} = {X1}')
    while (np.linalg.norm(X0 - X1) > tol and i < N):
        X0 = X1
        X1 = X0 - np.linalg.solve(J(X0[0], X0[1]), F(X0[0], X0[1]))
        i = i + 1
        print(f'i = {i}. X{i} = {X1}')
    return X1
```

## Guía 3
Resolución numérica de ecuaciones difeenciales

```python
def Euler(f, x0, t0, TF, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for k in range(1, N + 1):
        x[k] = x[k-1] + h * f(t[k-1], x[k-1])
    return t, x

def EulerImplicito(f, x0, t0, TF, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for k in range(1, N + 1):
        x[k] = x[k-1] + h * f(t[k], x[k])
    return t, x

def Taylor2(f, ft, fx, t0, TF, x0, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for i in range(1, N + 1):
        x[i] = x[i-1]+ h * f(t[i-1], x[i-1]) + (h**2 / 2) *(ft(t[i-1], x[i-1]) + fx(t[i-1], x[i-1]) * f(t[i-1], x[i-1]))
    return t, x

def RK2Especifico(f, t0, TF, x0, h = False, N = False, A1 = 0, A2 = 1, alpha = .5):
    '''
    Por defecto es Euler Modificado (A1 = 0, A2 = 1, alpha = 1)
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if A1 + A2 != 1:
        return ValueError('A1 + A2 debe dar exactamente 1')
    if alpha * A2 != .5:
        return ValueError('A2 * alpha debe dar exactamente 1/2')
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for i in range(1, N+1):
        x[i] = x[i-1] + h*A1*f(t[i-1], x[i-1]) + h*A2*f(t[i-1] + alpha*h, x[i-1] + alpha*h*f(t[i-1], x[i-1]))
    return t, x

def Heun(f, t0, TF, x0, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for i in range(1, N+1):
        K1 = f(t[i-1], x[i-1])
        K2 = f(t[i-1] + h, x[i-1] + h*K1)
        x[i] = x[i-1] + h*(K1+K2) / 2
    return t, x

RK2 = Heun

def EulerModificado(f, t0, TF, x0, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for i in range(1, N+1):
        x[i] = x[i-1] + h * f(t[i-1] + h/2, x[i-1] + h/2 * f(t[i-1], x[i-1]))
    return t, x

def RK4(f, t0, TF, x0, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo.
    Revisar las condiciones de los K
    '''
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    t = np.linspace(t0, TF, N+1)
    x = np.zeros(N + 1)
    x[0] = x0
    for i in range(1, N+1):
        K1 = f(t[i-1], x[i-1])
        K2 = f(t[i-1] + h/2, x[i-1] + (h/2)*K1)
        K3 = f(t[i-1] + h/2, x[i-1] + (h/2)*K2)
        K4 = f(t[i-1] + h, x[i-1] + h * K3)
        x[i] = x[i-1] + (h/6)*(K1+2*K2 + 2*K3 + K4)
    return t, x

def EulerSistemas(F, X0, t0, TF, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if len(X0) != F.__code__.co_argcount:
        raise ValueError('X0 debe tener igual cantidad de variables que F')
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    n = len(X0)
    t = np.linspace(t0, TF, N+1)
    X = np.zeros((n, N + 1))
    X[:, 0] = X0
    for k in range(1, N + 1):
        X[:, k] = X[:, k-1] + h * F(t[k-1], X[:, k-1])
    return t, X

def EulerImplicitoSistemas(F, X0, t0, TF, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if len(X0) != F.__code__.co_argcount:
        raise ValueError('X0 debe tener igual cantidad de variables que F')
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    n = len(X0)
    t = np.linspace(t0, TF, N+1)
    X = np.zeros((n, N + 1))
    X[:, 0] = X0
    for k in range(1, N + 1):
        X[:, k] = X[:, k-1] + h * f(t[k], X[:, k])
    return t, X

def EulerModificadoSistemas(F, X0, t0, TF, h = False, N = False):
    '''
    Sirve tanto para N como para h, pero hay que explicitarlo
    '''
    if len(X0) != F.__code__.co_argcount:
        raise ValueError('X0 debe tener igual cantidad de variables que F')
    if h:
        N = int((TF - t0) / h)
    elif N:
        h = (TF - t0) / N
    else:
        raise ValueError('Debe haber valor de h o de N asignado')
    n = len(X0)
    t = np.linspace(t0, TF, N+1)
    X = np.zeros((n, N + 1))
    X[:, 0] = X0
    for i in range(1, N+1):
        X[:, i] = X[:, i-1] + h * F(t[i-1] + h/2, X[:,i-1] + h/2 * F(t[i-1], X[:,i-1]))
    return t, X

def EncontrarEquilibrios(F1, F2, xy_max = 100):
    '''
    BruteForce para encontrar equilibrios en sistemas de dos ecuaciones difs.
    Las funciones no deben estar definidas en f(x, y), NO en f(t, X)
    '''
    eqs = []
    for x in range(xy_max):
        for y in range(xy_max):
            if ((F1(x, y) == 0) and (F2(x, y) == 0)):
                eqs.append((x, y))
    return eqs
```

## Guía 4
Cadenas de Markov

```python
def MarkovIters(M, v0, n, p = True):
    nd = len(M)
    assert np.sum(M) == np.sum(np.ones(nd)), 'La matriz M no es de Markov'
    assert np.sum(v0) == np.sum(np.ones(1)), 'El vector inicial no es un vector de proporción'
    assert M.shape[0] == M.shape[1], 'La matriz M no es cuadrada'
    assert (v0.ndim == 1), 'El vector inicial no es de nx1 dimensiones (escribirlo como única fila)'
    vn = v0
    vl = 0
    if p:
        print(f'v0 = {v0}')
    for i in range(n):
        try:
            vl = vm
        except:
            pass
        vm = np.round(vn, 3)
        vn = np.round(M @ vn, 3)
        if p:
            print(f'v{i+1} = {vn}')
        if np.array_equal(vm, vn) and i >= 1:
            if p:
                print(f'v{i} y v{i+1} son iguales:\n{vm} es Estado Límite\n')
                return vm
        if np.array_equal(vn, vl):
            if p:
                print(f'v{i} y v{i+2} son iguales:\nLa Matriz no tiene Estado Límite. Alterna entre\n{vn} y {vm}')
                return None

    return vn

def MarkovEqs(M, p = True):
    nd = len(M)
    assert np.sum(M) == np.sum(np.ones(nd)), 'La matriz M no es de Markov'
    assert M.shape[0] == M.shape[1], 'La matriz M no es cuadrada'
    e = np.linalg.eig(M)
    autovals = np.round(e[0], 5)
    autovecs = e[1]
    autovals_list = list(autovals)
    autovecs = np.round((autovecs)/sum(np.abs(autovecs)), 5)
    if p:
        print('Autovalores:', autovals_list)
        print('\nAutovectores =\n', autovecs)
    eqs = []
    q = 0
    for val in autovals:
        if np.linalg.norm(val)==1:
            idx = autovals_list.index(val)
            eq = autovecs[:, idx + q]
            if p:
                print(f'\nav{q+1} = autovector para lambda {val} = \n {eq}\n')
            eqs.append(eq)
            q += 1
            autovals_list.pop(idx)
    if len(eqs) > 1:
        return eqs
    else:
        return eqs[0]
```

## Guía 5
Cuadrados Mínimos

```python
# Funcs.
def CuadradosMinimos(X, Y, R2=True, p=True):
    """Devuelve alfa y beta que minimian S(alfa, beta).
    Si R2=True, devuelve un array con [{poly1d de regresión}, {R2}]
    """
    n = len(X)
    unos = np.ones(n)
    At = np.array([unos, X])
    A = np.transpose(At)
    AtA = At @ A
    AtY = At @ Y
    params = np.linalg.solve(AtA, AtY)
    reg = np.poly1d(np.flipud(params))
    if R2:
        Y_media = sum(Y)/len(Y)
        Y_star = reg(X)
        R2 = np.linalg.norm(Y_star - Y_media, 2)**2 / np.linalg.norm(Y - Y_media, 2) ** 2
        if p:
            print(reg)
            print('R2 = ', R2)
        return np.array([reg, R2])
    if p and R2==False:
        print(reg)
        return reg
    else:
        return reg

RegresionLineal = CuadradosMinimos

def CuadradosMinimosW(X, Y, W, p=True):
    """Si R2=True, devuelve un array con [{poly1d de regresión}, {R2}]
    W debe estar en forma de vector (1D)
    """
    n = len(X)
    unos = np.ones(n)
    W = np.diag(W)
    At = np.array([unos, X])
    A = np.transpose(At)
    AtWA = At @ W @ A
    AtWY = At @ W @ Y
    params = np.linalg.solve(AtWA, AtWY)
    reg = np.poly1d(np.flipud(params))
    if p:
        print(reg)
    return reg

RegresionLinealW = CuadradosMinimosW

def CombinacionLineal(X, Y, Fs, p=True):
    lx = len(X)
    ly = len(Y)
    if lx != ly:
        raise ValueError('La cantidad de datos en x e y es distinta')
    A = np.c_[Fs[0](X)]
    for f in range(1, len(Fs)):
        A = np.c_[A, Fs[f](X)]
    At = A.T
    AtA = At @ A
    AtY = At @ Y
    alphas = np.linalg.solve(AtA, AtY)
    if p:
        for q in range(len(alphas)):
            print(f'Alfa {q} = {alphas[q]}')
    return alphas

def RegresionLinealMultiple(X1, X2, Y, R2 = False, p=True):
    lx = len(X1)
    if lx != len(X2) != len(Y):
        raise ValueError('Las dimensiones no tienen igual cantidad de datos')
    ones = np.ones(lx)
    At = np.array([ones, X1, X2])
    A = At.T
    AtA = At @ A
    AtY = At @ Y
    reg = np.linalg.solve(AtA, AtY)
    if p:
        print('Alpha 0 =', reg[0], '\nAlpha 1 =', reg[1], '\nAlpha 2 =', reg[2])
    return reg

def ObtenerR2(X, Y, regresion):
    Y_media = sum(Y)/len(Y)
    Y_star = regresion(X)
    R2 = np.linalg.norm(Y_star - Y_media, 2)**2 / np.linalg.norm(Y - Y_media, 2) ** 2
    return R2

def ObtenerR2ML(X1, X2, Y, regresion):
    Y_media = sum(Y)/len(Y)
    Y_star = regresion(X1, X2)
    R2 = np.linalg.norm(Y_star - Y_media, 2)**2 / np.linalg.norm(Y - Y_media, 2) ** 2
    return R2

def ObtenerError(X, Y, regresion):
    Y_vals = regresion(X)
    err = np.linalg.norm(Y - Y_vals, 2)
    return err**2

def ObtenerErrorML(X1, X2, Y, regresion):
    Y_vals = regresion(X1, X2)
    err = np.linalg.norm(Y - Y_vals, 2)
    return err**2
```

## Guía 6
PCA y Clustering

```python
def PCA(X_array, Return=None, theta=1.0, z_max=None, normalizar_X=True, p=False):
    '''
    Returns: Z: Todos los z\nlambdas: autovals y U\nX_star: X*\ncov o A: A\nNada: Coefs. para var objetivo
    '''
    #Calculo de X*
    l = len(X_array[:, 0])
    if normalizar_X:
        X_star =np.c_[ X_array[:, 0] - sum(X_array[:, 0])/l]
        for n in range(1, len(X_array[0])):
            col = X_array[:, n]
            X_star = np.c_[X_star, col - sum(col)/l]
        if p:
            print('X* =\n', X_star, '\n-------------------------')
    else:
        X_star = X_array

    #Calculo de Matriz de Cov
    X_star_T = X_star.T
    A = X_star_T @ X_star / l

    # Orden de autovals y autovecs
    autovals, U = np.linalg.eig(A)
    idx = autovals.argsort()[::-1]
    autovals = autovals[idx]
    U = U[:, idx]
    if p:
        print(f'Autovalores = {autovals}\nMatriz de autovectores:\n{U}\n-------------------------')
    Z = X_star @ U
    varTotal = sum(autovals)
    varAcum = autovals[0]
    varObj = theta * varTotal
    i = 1
    proporcion = 100 * varAcum / varTotal
    if p:
        print(f'Varianza total = {varTotal}')
        print(f'Varianza objetivo = {varObj}')
        print(f'Con {i} componente, la varianza es de {varAcum}\nRepresenta un {proporcion}% de la varianza total')
    while varAcum < varObj:
        varAcum += autovals[i]
        proporcion = 100 * varAcum / varTotal
        i = i+1
        if p:
            print(f'Con {i} componentes, la varianza acumulada es de {varAcum}\nRepresenta un {proporcion}% de la varianza total)')
    if p:
        print('-------------------------\nComponentes principales:\n', Z[:, 0: i])
    if Return == 'Z':
        return Z
    elif Return == 'lambdas':
        return [autovals, U]
    elif Return == 'cov' or Return =='A':
        return A
    elif Return == 'X_star':
        return X_star
    elif z_max:
        return Z[:, 0:z_max]
    else:
        return Z[:, 0:i]

def ObtenerZyXstar(X_array, p=True):
    #Calculo de X*
    l = len(X_array[:, 0])
    X_star =np.c_[ X_array[:, 0] - sum(X_array[:, 0])/l]
    for n in range(1, len(X_array[0])):
        col = X_array[:, n]
        X_star = np.c_[X_star, col - sum(col)/l]
    if p:
        print('X* =\n', X_star, '\n-------------------------')

    #Calculo de Matriz de Cov
    X_star_T = X_star.T
    A = X_star_T @ X_star / l

    # Orden de autovals y autovecs
    autovals, U = np.linalg.eig(A)
    idx = autovals.argsort()[::-1]
    autovals = autovals[idx]
    U = U[:, idx]
    Z = X_star @ U
    if p:
        print(f'Z = \n', Z)
    return np.array([Z, X_star])

def PlotZvsXstar(Z, X_star, z_min = -5, z_max = 5):
    n = 0
    m = 0
    k = 0
    x_plot = np.linspace(z_min, z_max, 100)
    for z in Z.T:
        n += 1
        m = 0
        for x in X_star.T:
            k += 1
            plt.subplot(len(X_star.T), len(Z.T), k)
            m += 1
            plt.plot(z, x, 'k.')
            plt.xlabel(f'z{n}')
            plt.ylabel(f'x{m}*', rotation=0)
            f, R2 = RegresionLineal(z, x, p=False)
            plt.plot(x_plot, f(x_plot), label=f'R2 = {R2}')
            plt.legend()

def Colors(clusters):
    assign = []
    cols = ['r', 'b', 'g', 'c', 'orange', 'y', 'magenta' 'k']
    for j in clusters:
        assign.append(cols[j])
    return assign
```


# Tutorial

In this Jupyter notebook, we will do a walkthrough of using PyFDAM to fit a mock impedance data and using it to validate a galvanostatic discharge data. Before you run this tutorial, make sure you've installed PyFDAM!

First, let's import the library. We will also import `numpy` as it will be useful later on.

```python
import pyfdam
import numpy as np
```

Next, we'll define the **model** and its **parameters**. In PyFDAM, a model is a formula $ Z(s) $ for the impedance data, where $ s $ is the frequency argument. In Python, the model is written as a function followed by parameters as an additional argument:

```python
def Zmodel(s,parameters):
    # Define your model which takes in 'parameters' as argument
```

The argument ` parameters ` cannot be defined/managed arbitrarily! PyFDAM provides an object called ` Params ` that will manage the parameters, how they're labeled, and whether they should be designated as fitting vs. fixed parameters.

For this tutorial, our model is defined as a symmetric cell with a porous electrode of thickness $ L $ and specific interfacial area $ a$. The contact surface area of the cell is denoted as $A$. Also, the electrolyte's conductivity is denoted by $ \kappa $ and the current collector's conductivity is denoted by $ \sigma $. The cell is connected to an external resistor $ R_\mathrm{ext} $, which model's separator resistance + other external resistance contributions, as well as an inductance element $ L_\mathrm{ind} $ in series.  Finally, double layer capacitance at the electrode/electrolyte interface is modelled as a simple capacitance $ C_\mathrm{dl} $.

In total we have 8 parameters ($ L, a, A, \kappa, \sigma, R_\mathrm{ext}, L_\mathrm{ind}, C_\mathrm{dl} $). For this tutorial, we assume that $ L, a, A, \sigma $ will be known and is therefore fixed while the rest will be treated as fitting parameters. To do this, we will create the `Params` object and add a list of parameters to it

```python
#Step 1 -  Define the Parameters List
myparams = pyfdam.Params()

#Step 2 - Write the fixed parameters
#Electrode thickness (cm)
myparams.add('L',0.0242,vary=False)
#Conducitivity of current collector (S/cm)
myparams.add('sig',8,vary=False)
#The interfacial area (cm^2/cm^3)
myparams.add('a',6*10**6,vary=False)
#The cell area (cm^2).
myparams.add('A',71.43,vary=False)

#Step 3 - Write the fitting parameters
#Seperator/solution resistance, (ohm)
myparams.add('Rext',1.0,vary=True,min=0)
#External inductance (F/cm2)
myparams.add('Lind',1e-5,vary=True,min=0)
#double layer capacitance (F/cm2)
myparams.add('Cdl',1e-5,vary=True,min=0)
#Conductivity of electrolyte (S/cm)
myparams.add('kap',1.0,vary=True,min=0)
```

Note that each parameter can be accessed via `.value` method:

```python
#As an example, we can print the conducitivity of the electrolyte
print(myparams['kap'].value)

#Or print the electrode thickness
print(myparams['L'].value)
```

    1.0
    0.0242

Once the parameters have been defined, we can actually write the model's function. Let $ Z(s) $ be the cell's total impedance, then the equation for the $ Z(s) $ can be written as

\begin{gather}
Z(s) = \frac{2 L}{(\kappa + \sigma)} \left[1+\frac{2+\left(\frac{\kappa}{\sigma}+\frac{\sigma}{\kappa}\right)\cosh \nu}{\nu \sinh \nu}\right] +R_\mathrm{ext} +s \cdot L_\mathrm{ind}
\\
\nu = \sqrt{a \left(s \cdot C_\mathrm{dl} \right) L^2 \left(\frac{1}{\kappa}+\frac{1}{\sigma}\right)}
\end{gather}
where the first term is the porous electrode contribution.

The corresponding Python function is

```python
def Zmodel(s, params):
    #Access the raw numbers from the Params object
    Rext = params['Rext'].value
    Cdl = params['Cdl'].value
    Lind = params['Lind'].value
    K = params['kap'].value
    sig = params['sig'].value
    L = params['L'].value
    a = params['a'].value
    A = params['A'].value

    res = Cdl*s
    nu = (a*res*L**2*(1/sig+1/K))**0.5
    imp = (1+(2+(sig/K+K/sig)*np.cosh(nu))/(nu*np.sinh(nu)))

    return (2*L*imp/(A*(K+sig))+Rext+Lind*(s))
```

Once we've defined the parameters and model, we can now use them to do impedance fitting. To do this we construct an `ImpedanceFitter` object that will take our defined model, parameters, and impedance data filename as an input.

```python
MyFit = pyfdam.ImpedanceFitter(model=Zmodel,parameters=myparams,namedata="EIS_Mock_Data.txt")
```

Note that the datafile must be formatted with three columns and no headers! The first column is frequency, the second column is the real part of the impedance, and the third column is the imaginary part of the impedance. In this tutorial, the experimental data is `EIS_Mock_Data.txt` so check that file for reference!.

Finally, we can run the fitting process using the `.run()` method. The fitting run is done by selecting a suitable numerical method as an initial error minimization step. This is then followed by some number of cycles, denoted by `ncycles`, of Leveberg-Marquadt algorithm to further converge the parameters and obtain their uncertainty.

```python
MyFit.run(method='nelder',ncycles=1) #In here, we've chosen Nelder-Mead ad the initial method
```

    [[Variables]]
        L:     0.0242 (fixed)
        sig:   8 (fixed)
        a:     6000000 (fixed)
        A:     71.43 (fixed)
        Rext:  0.01531600 +/- 1.1009e-04 (0.72%) (init = 0.02352427)
        Lind:  4.8185e-06 +/- 1.6489e-07 (3.42%) (init = 3.985611e-06)
        Cdl:   7.6876e-06 +/- 7.8261e-09 (0.10%) (init = 7.660804e-06)
        kap:   0.01425053 +/- 1.6015e-04 (1.12%) (init = 3.835463)
    [[Correlations]] (unreported correlations are < 0.100)
        C(Rext, kap) =  0.690

A list of numerical methods for the initial fitting can be found in [the LMFIT package's documentation](https://lmfit.github.io/lmfit-py/fitting.html).

Next, we can take a look at the result of the fitting. To do this, we need to grab the output result as well as the experimental impedance data.

```python
#Get both the frequency (expfreq) and impedance data (expdata) of from the loaded textfile!
expfreq, expdata = MyFit.get_impedanceexp()

#Get the impedance output from the model using .get_impedancemodel() method. It takes a list of frequencies as an argument
modeldata = MyFit.get_impedancemodel(expfreq)
```

Afterwards, we can plot the data!

```python
%matplotlib inline

import matplotlib.pyplot as plt

plt.figure(figsize=(6,6))
#Plotting model data
plt.plot(modeldata.real,-modeldata.imag)

#Plotting experimental data
plt.plot(expdata.real,-expdata.imag,'o-',alpha=0.5)

#Impedance plot should be equal axes
plt.gca().set_aspect('equal', adjustable='box')

#Some labelling
plt.xlabel('Re Z')
plt.xlabel('-Im Z')
#Adjust the range as necessarry!
plt.ylim([-0.02,0.07])
plt.xlim([0,0.07])
```

We can also save the impedance data for later plotting

```python
#We use the list of frequencies from the experimental data, although we could use any other list of frequencies
MyFit.save_impedance(expfreq,'EIS_Model_Data.txt')
```

What's next? Well we can use this set of parameters to run a discharge experiment. The data that we will compare  with starts  at a fully charged cell at 2.4 V, which was discharging at a current of 200 A. To setup the galvanostatic discharge simulation, we need to create a `Galvanostatic` object and feed it wth out model and the fitted parameters, which we can get from the previous `ImpedanceFitter` object via the `.get_params()` method

```python
MyGalvanostatic = pyfdam.Galvanostatic(model=Zmodel,parameters=MyFit.get_params())
```

To run the simulation, we can use the `.run()` method which will use the Gaver-Stehfest (GV) method to perform numerical inverse Laplace transform. The `.run()` method takes in the following arguments:
1. `time` ($ t $) is an array of time which starts from **intial time larger than zero** (the GV method cannot handle t=0) and up to some final time.
2. `DeltaV0` ($ \Delta V_0 $)the initial total difference in potential across cell
3. `Iapp` ($ I_\mathrm{app} $) is the applied current, with (-)/(+) sign indicating discharging/charging experiments respectively,
4. `A` ($ A $) is an optional parameter that is divided against the impedance function $ Z(s) $. This is used only if the original model is not described in the correct Ohmic units, e.g., the model impedance is multiplied by surface area so we need `A` to be the cell's area. By default, it is set to one. For our tutorial, `A` will be the surface area parameter $ A $ since the original model function $ Z(s) $ is expressed in terms of Ohm cm^2.
5. `Nterms`($ N_\mathrm{terms} $) is a parameter for the Gaver-Stehfest method, which should be around 6 to 20 to obtain good convergence of the time-domain profile. By default, it is set to 10.

Under the hood, the code is inverting the following function in Laplace space:

\begin{equation}
V(s) = \frac{\Delta V_0}{s}+\frac{I_\mathrm{app} Z(s)}{s \cdot A}
\end{equation}

by using the following series expansion:

\begin{gather}
V(t) \approx \Delta V_0+\frac{\ln 2}{t} \sum_{n=1}^{N_\mathrm{terms}} c_{n} \frac{I_\mathrm{app} t}{A(n \ln 2)} \cdot Z\left(\frac{n \ln 2}{t}\right)
\\
c_{n}=(-1)^{n+\frac{N_\mathrm{terms}}{2}} \sum_{k=\frac{n+1}{2}}^{\min (n, N_\mathrm{terms} / 2)} \frac{k^{N_\mathrm{terms} / 2}(2 k) !}{(N_\mathrm{terms} / 2-k) ! k !(k-1) !(n-k) !(2 k-n) !}
\end{gather}

PyFDAM automates the whole series expansion computation so all we need to do is just feed in the right parameters!

```python
#Load the discharge data
expdata = np.loadtxt("Discharge_Mock_Data.txt")
#The time array is obtained from the mock data. Note that we also exclude the initial time!
MyGalvanostatic.run(time = expdata[1:,0], V0=2.4,Iapp=-200,A=myparams['A'].value)

```

Similar to the impedance run, we can get the data, which is done through the `.get_output()` method, save the data, which is done through the `.save_output` method, and plot the results

```python
#Grabbing the output as well as time used
t, modeldata = MyGalvanostatic.get_output()
#Saving the output
MyGalvanostatic.save_output("Discharge_Data_Results.txt")

plt.figure(figsize=(5,5))
plt.plot(expdata[::2,0],expdata[::2,1],'o', label="Experimental Data")
plt.plot(t,modeldata,label="Model Validation")
plt.xlabel("time (s)")
plt.ylabel("Potential (V)")
plt.legend(loc=0)
plt.show()
```

## Extra Notes

The module only consists of one file, which is just an `__init__.py` file. All codes are implemented in there. PyFDAM also do not implement the fitting routines by itself, and delegate it to the LMFIT package. Any issues pertaining to the fitting process can be traced directly to the LMFIT package. I highly recommend looking at LMFIT package for more troubleshooting because non-linear curve fitting, just by the nature of the mathematical problem, can be difficult!


 Text provided under a Creative Commons Attribution license, CC-BY. All code is made available under the FSF-approved MIT license. (c) Kyle T. Mandli

```python
from __future__ import print_function
from __future__ import absolute_import

%matplotlib inline
import numpy
import matplotlib.pyplot as plt
```

# Numerical Differentiation

**GOAL:**  Given a set of $N+1$ points $(x_i, y_i)$ compute the derivative of a given order to a specified accuracy.

**Approach:** Find the interpolating polynomial $P_N(x)$ and differentiate that.

### Newton Polynomials

While the interpolating polynomial $P_N(x)$ through $N+1$ distinct points is unique,  it can be expressed in terms of many bases.

So far we have considered the monomial basis $< 1,x,x^2,\ldots,x^n>$ and the Lagrange basis $\ell_i(x)$

However, for our purposes here it's somewhat more convenient to use the *Newton Polynomial* Basis

\begin{align}
    n_0 &= 1\\
    n_j(x) &= \prod^{j-1}_{i=0} (x - x_i)\quad\mathrm{for}\quad j>0 \\
\end{align}

### Newton Polynomials

Unrolling them, we see
\begin{align}
    n_0(x) &= 1\\
    n_1(x) &= (x-x_0) \\
    n_2(x) &= (x - x_0)(x-x_1)\\
        &\vdots \\
    n_N(x) &= (x - x_0)(x - x_1)\cdots(x-x_{N-1})\\
\end{align}

Note:  The Newton polynomials have some features common to both the monomials and the Lagrange polynomials

* Like the monomials they are monic polynomials of increasing degree in $x$, i.e. $n_2(x)$ is quadratic
* Like the Lagrange polynomials, some Newton polynomials vanish identically at the interpolation points $x_i$.  In particular

$$
    n_j(x_i) = 0 \quad \mathrm{for}\quad j> i
$$

### Newton's Form

This last feature makes it convenient to solve for the interpolating polynomial.

$$P_N(x) = \sum^N_{j=0} a_j n_j(x)$$

as the $(N+1)\times (N+1)$ linear system

$$
    P_N(x_i) = y_i\quad\mathrm{for}\quad i=0,\ldots,N
$$

#### Becomes the lower triangular system

$$
\begin{bmatrix}
1 & 0 & 0 & \ldots & 0 \\
1 & (x_1-x_0) & 0 & \ldots & 0 \\
1 & (x_2-x_0) & (x_2-x_0)(x_2-x_1) & \ldots & 0 \\
\vdots & \vdots & \vdots & \vdots & \vdots  \\
1 & n_1(x_n) & n_2(x_n) & \ldots & n_n(x_n) \\
\end{bmatrix}\begin{bmatrix}
a_0 \\
a_1 \\
a_2 \\
\vdots \\
a_n \\
\end{bmatrix}=
\begin{bmatrix}
y_0 \\
y_1 \\
y_2 \\
\vdots \\
y_n \\
\end{bmatrix}
$$

Which can be solved sequentially starting with

\begin{align}
    a_0 & = y_0\\
    a_1 & = \frac{y_1 - y_0}{x_1 - x_0} \\
    a_2 & = \frac{y_2 - y_0 - a_1(x_2 - x_0)}{(x_2-x_0)(x_2-x_1)} \\
    &\vdots \\
\end{align}

### The divided differences

The $a_j = [y_0, \ldots, y_j]$ are also known as *the divided differences* which are defined recursively as

$$[y_i] = y_i \quad i \in \{0,\ldots, N+1\}$$

and

$$[y_i, \ldots , y_{i+j}] = \frac{[y_{i+1}, \ldots , y_{i + j}] - [y_{i},\ldots,y_{i+j-1}]}{x_{i+j} - x_{i}} \quad i \in \{0,\ldots,N+1 - j\} \quad j \in \{1,\ldots, N+1\}$$

These formulas are recursively defined but not so helpful, here are a few examples to start out with:

$$[y_0] = y_0$$

$$[y_0, y_1] = \frac{y_1 - y_0}{x_1 - x_0}$$

$$[y_0, y_1, y_2] = \frac{[y_1, y_2] - [y_0, y_1]}{x_{2} - x_{0}} = \frac{\frac{y_2 - y_1}{x_2 - x_1} - \frac{y_1 - y_0}{x_1 - x_0}}{x_2 - x_0} = \frac{y_2 - y_1}{(x_2 - x_1)(x_2 - x_0)} - \frac{y_1 - y_0}{(x_1 - x_0)(x_2 - x_0)}$$

The benefit of writing a polynomial like this is that it isolates the $x$ dependence (we can easily take derivatives of this form).

In general then $P_N(x)$ can be written in Newton's form as

$$P_N(x) = y_0 + (x-x_0)[y_0, y_1] + (x - x_0) (x - x_1) [y_0, y_1, y_2] + \cdots + (x-x_0) (x-x_1) \cdots (x-x_{N-1}) [y_0, y_1, \ldots, y_{N}]$$

```python
def divided_difference(x, y, N=50):
    """Compute the Nth divided difference using *x* and *y*"""
    if N == 0:
        raise Exception("Reached recursion limit!")

    # Reached the end of the recurssion
    if y.shape[0] == 1:
        return y[0]
    elif y.shape[0] == 2:
        return (y[1] - y[0]) / (x[1] - x[0])
    else:
        return (divided_difference(x[1:], y[1:], N=N-1) - divided_difference(x[:-1], y[:-1], N=N-1)) / (x[-1] - x[0])

def newton_basis(x, x0):
    """ construct newton basis for the interpolating polynomial given interpolation points x0"""
    basis = numpy.ones((len(x0), len(x)))
    for j in range(len(x0)):
        for i in range(j):
            basis[j, :] *= (x - x0[i])
    return basis

def P_newton(x, data):
    """ construct the interpolating polynomial of order N through the N+1 data points data using a Newton basis"""
    P = numpy.zeros(x.shape)
    basis = newton_basis(x, data[:,0])
    for j in range(data.shape[0]):
        P += divided_difference(data[:j + 1, 0], data[:j + 1, 1]) * basis[j, :]

    return P
```

```python
# Calculate a polynomial in Newton Form
data = numpy.array([[-2.0, 1.0], [-1.5, -1.0], [-0.5, -3.0], [0.0, -2.0], [1.0, 3.0], [2.0, 1.0]])
x = numpy.linspace(-2.0, 2.0, 100)
basis = newton_basis(x, data[:,0])
P = P_newton(x, data)
N = data.shape[0] - 1
```

```python
# Plot basis and interpolant
fig = plt.figure(figsize = (16,6))
fig.subplots_adjust(hspace=.5)

axes = [None, None]
axes[0] = fig.add_subplot(1, 2, 1)
axes[1] = fig.add_subplot(1, 2, 2)

for j in range(N + 1):
    axes[0].plot(x, basis[j, :])
    axes[1].plot(data[j, 0], data[j, 1],'ko')
axes[1].plot(x, P)

axes[0].set_title("Newton Polynomial Basis")
axes[0].set_xlabel("x")
axes[0].set_ylabel("$n_j(x)$")
axes[0].grid()

axes[1].set_title("Interpolant $P_%s(x)$" % N)
axes[1].set_xlabel("x")
axes[1].set_ylabel("$P_%s(x)$" % N)
axes[1].grid()

plt.show()
```

As another concrete example consider the general quadratic polynomial written in Newton's form

$$P_2(x) = [y_0] + (x - x_0) [y_0, y_1] + (x - x_0)(x - x_1) [y_0, y_1, y_2] \\= y_0 + (x - x_0) \frac{y_1 - y_0}{x_1 - x_0} + (x - x_0)(x - x_1) \left ( \frac{y_2 - y_1}{(x_2 - x_1)(x_2 - x_0)} - \frac{y_1 - y_0}{(x_1 - x_0)(x_2 - x_0)} \right )$$

Recall that the interpolating polynomial of degree $N$ through these points is unique!

### Error Analysis

Given $N + 1$ points we can form an interpolant $P_N(x)$ of degree $N$ where

$$f(x) = P_N(x) + R_N(x)$$

And we know from Lagrange's Theorem that the remainder term looks like

$$R_N(x) = (x - x_0)(x - x_1)\cdots (x - x_{N})(x - x_{N+1}) \frac{f^{(N+1)}(c)}{(N+1)!}$$

noting that we need to require that $f(x) \in C^{N+1}$ on the interval of interest.  Taking the derivative of the interpolant $P_N(x)$ then leads to

$$P_N'(x) = [y_0, y_1] + ((x - x_1) + (x - x_0)) [y_0, y_1, y_2] + \cdots + \left(\sum^{N-1}_{i=0}\left( \prod^{N-1}_{j=0,~j\neq i} (x - x_j) \right )\right ) [y_0, y_1, \ldots, y_N]$$

Similarly we can find the derivative of the remainder term $R_N(x)$ as

$$R_N'(x) = \left(\sum^{N}_{i=0} \left( \prod^{N}_{j=0,~j\neq i} (x - x_j) \right )\right ) \frac{f^{(N+1)}(c)}{(N+1)!}$$

Now if we consider the approximation of the derivative evaluated at one of our data points $(x_k, y_k)$ these expressions simplify such that

$$f'(x_k) = P_N'(x_k) + R_N'(x_k)$$

If we let $\Delta x = \max_i |x_k - x_i|$ we then know that the remainder term will be $\mathcal{O}(\Delta x^N)$ as $\Delta x \rightarrow 0$ thus showing that this approach converges and we can find arbitrarily high order approximations (ignoring floating point error).

### Caution

High order does not necessarily imply high-accuracy!

```python
def PN_prime(x, data):
    """ calculate derivative of interpolating polynomail at point x"""
    # General form of derivative of P_N'(x)
    P_prime = numpy.zeros(x.shape)
    newton_basis_prime = numpy.empty(x.shape)
    product = numpy.empty(x.shape)
    N = data.shape[0] - 1
    for n in range(N + 1):
        newton_basis_prime = 0.0
        for i in range(n):
            product = 1.0
            for j in range(n):
                if j != i:
                    product *= (x - data[j, 0])
            newton_basis_prime += product
        P_prime += divided_difference(data[:n+1, 0], data[:n+1, 1]) * newton_basis_prime

    return P_prime
```

```python
num_points = 4
xmin = -2.
xmax = 2.
data = numpy.empty((num_points, 2))
data[:, 0] = numpy.linspace(xmin, xmax, num_points)
data[:, 1] = numpy.sin(data[:, 0])
x = numpy.linspace(xmin, xmax, 100)
P_prime = PN_prime(x, data)
```

```python
fig = plt.figure(figsize=(8, 6))
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, numpy.cos(x), 'k', label="$f'(x)$")
axes.plot(x, P_prime, 'r--', label="$Pn'(x)$")
axes.plot(data[:,0], numpy.cos(data[:,0]), 'ro')
axes.legend(loc='best')
axes.set_title("$f'(x)$")
axes.set_xlabel("x")
axes.set_ylabel("$f'(x)$ and $\hat{f}'(x)$")
axes.grid()

plt.show()
```

**Note:** In the parlance of numerical PDEs we can improve the estimate either by $p$-refinement (i.e. increasing the order of the polynomial) or $h$-refinement (decreasing the 'step-size', i.e. playing with the $\Delta x$ term).

### Examples

Often in practice we only use a small number of data points to derive a differentiation formula.  In the context of differential equations we also often have $f(x)$ so that $f(x_k) = y_k$ and we can approximate the derivative of a known function $f(x)$.

### Example 1:  1st order Forward and Backward Differences

Using 2 points we can get an approximation that is $\mathcal{O}(\Delta x)$:

$$f'(x) \approx P_1'(x) = [y_0, y_1] = \frac{y_1 - y_0}{x_1 - x_0} = \frac{y_1 - y_0}{\Delta x} = \frac{f(x_1) - f(x_0)}{\Delta x}$$

We can also calculate the error as

$$R_1'(x) = -\Delta x \frac{f''(c)}{2}$$

We can also derive the "forward" and "backward" formulas by considering the question slightly differently.  Say we want $f'(x_n)$, then the "forward" finite-difference can be written as

$$f'(x_n) \approx D_1^+ = \frac{f(x_{n+1}) - f(x_n)}{\Delta x}$$

and the "backward" finite-difference as

$$f'(x_n) \approx D_1^- = \frac{f(x_n) - f(x_{n-1})}{\Delta x}$$

Note these approximations should be familiar to use as the limit as $\Delta x \rightarrow 0$ these are no longer approximations but equivalent definitions of the derivative at $x_n$.

```python
f = lambda x: numpy.sin(x)
f_prime = lambda x: numpy.cos(x)

# Use uniform discretization
x = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, 1000)
N = 20
x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N)
delta_x = x_hat[1] - x_hat[0]

# Compute forward difference using a loop
f_prime_hat = numpy.empty(x_hat.shape)
for i in range(N - 1):
    f_prime_hat[i] = (f(x_hat[i+1]) - f(x_hat[i])) / delta_x
f_prime_hat[-1] = (f(x_hat[-1]) - f(x_hat[-2])) / delta_x

# Vector based calculation
# f_prime_hat[:-1] = (f(x_hat[1:]) - f(x_hat[:-1])) / (delta_x)

# Use first-order differences for points at edge of domain
f_prime_hat[-1] = (f(x_hat[-1]) - f(x_hat[-2])) / delta_x  # Backward Difference at x_N
```

```python
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)

axes.plot(x, f_prime(x), 'k')
axes.plot(x_hat + 0.5 * delta_x, f_prime_hat, 'ro')
axes.set_xlim((x[0], x[-1]))
axes.set_ylim((-1.1, 1.1))
axes.grid()

plt.show()
```

#### Aside:  Computing Order of Convergence

Say we had the error $E(\Delta x)$ and we wanted to make a statement about the rate of convergence (note we can replace $E$ here with the $R$ from above).  Then we can do the following:
$$\begin{aligned}
    E(\Delta x) &= C \Delta x^n \\
    \log E(\Delta x) &= \log C + n \log \Delta x
\end{aligned}$$

The slope of the line is $n$ when modeling the error like this!  We can also match the first point by solving for $C$:

$$
    C = e^{\log E(\Delta x) - n \log \Delta x}
$$

```python
f = lambda x: numpy.sin(x)
f_prime = lambda x: numpy.cos(x)

# Compute the error as a function of delta_x
N_range = numpy.logspace(1, 4, 10, dtype=int)
delta_x = numpy.empty(N_range.shape)
error = numpy.empty((N_range.shape[0], 4))
for (i, N) in enumerate(N_range):
    x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N)
    delta_x[i] = x_hat[1] - x_hat[0]

    # Compute forward difference
    f_prime_hat = numpy.empty(x_hat.shape)
    f_prime_hat[:-1] = (f(x_hat[1:]) - f(x_hat[:-1])) / (delta_x[i])

    # Use first-order differences for points at edge of domain
    f_prime_hat[-1] = (f(x_hat[-1]) - f(x_hat[-2])) / delta_x[i]  # Backward Difference at x_N

    # The differences in error computations is interesting here.  Note that the L_\infty norm returns a single
    # point-wise like error where as the L_2 and L_1 are adding error up so they exhbit less convergence
    error[i, 0] = numpy.linalg.norm(numpy.abs(f_prime(x_hat) - f_prime_hat), ord=numpy.infty)
    error[i, 1] = numpy.linalg.norm(numpy.abs(f_prime(x_hat + 0.5 * delta_x[i]) - f_prime_hat), ord=numpy.infty)
    error[i, 2] = numpy.linalg.norm(numpy.abs(f_prime(x_hat) - f_prime_hat), ord=1)
    error[i, 3] = numpy.linalg.norm(numpy.abs(f_prime(x_hat) - f_prime_hat), ord=2)

error = numpy.array(error)
delta_x = numpy.array(delta_x)

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)
fig.set_figheight(fig.get_figheight() * 2)
# plt.rc('legend', fontsize=6)
# plt.rc('font', size=6)

error_type = ['$L^\infty$', 'offset $L^\infty$', '$L^1$', '$L^2$']
for i in range(error.shape[1]):
    axes = fig.add_subplot(2, 2, i + 1)
    axes.loglog(delta_x, error[:, i], 'ko', label="Error")

    order_C = lambda delta_x, error, order: numpy.exp(numpy.log(error) - order * numpy.log(delta_x))
    axes.loglog(delta_x, order_C(delta_x[0], error[0, i], 1.0) * delta_x**1.0, 'r--', label="1st Order")
    axes.loglog(delta_x, order_C(delta_x[0], error[0, i], 2.0) * delta_x**2.0, 'b--', label="2nd Order")
    axes.legend(loc=4)
    axes.set_title("Convergence of 1st Order Differences - %s" % error_type[i])
    axes.set_xlabel("$\Delta x$")
    axes.set_ylabel("$|f'(x) - \hat{f}'(x)|$")

plt.show()
```

### Example 2: 2nd Order Centered Difference

Now lets use 3 points to calculate the 2nd order accurate finite-difference.  Consider the points $(x_{n}, y_{n})$, $(x_{n-1}, y_{n-1})$, and $(x_{n+1}, y_{n+1})$, from before we have

$$\begin{aligned}
    P_2(x) &= [f(x_0)] + (x - x_0) [f(x_0), f(x_1)] + (x - x_0)(x - x_1) [f(x_0), f(x_1), f(x_2)] \\
    &= f(x_0) + (x - x_0) \frac{f(x_1) - f(x_0)}{x_1 - x_0} + (x - x_0)(x - x_1) \left ( \frac{f(x_2) - f(x_1)}{(x_2 - x_1)(x_2 - x_0)} - \frac{f(x_1) - f(x_0)}{(x_1 - x_0)(x_2 - x_0)} \right )
\end{aligned}$$
Compute the formula for the derivative.  Assume that the distance between the $x_i$ are equal.

$$\begin{aligned}
    P_2'(x) &= [f(x_n), f(x_{n+1})] + ((x - x_n) + (x - x_{n+1})) [f(x_n), f(x_{n+1}), f(x_{n-1})] \\
    &= \frac{f(x_{n+1}) - f(x_n)}{x_{n+1} - x_n}  + ((x - x_n) + (x - x_{n+1}))\\ &* \left ( \frac{f(x_{n-1}) - f(x_{n+1})}{(x_{n-1} - x_{n+1})(x_{n-1} - x_n)} - \frac{f(x_{n+1}) - f(x_n)}{(x_{n+1} - x_n)(x_{n-1} - x_n)} \right )
\end{aligned}$$

Evaluating at $x_n$ and assuming the points $x_{n-1}, x_n, x_{n+1}$ are evenly spaced leads to

$$\begin{aligned}
    P_2'(x_n) &= \frac{f(x_{n+1}) - f(x_n)}{\Delta x} - \Delta x \left ( \frac{f(x_{n-1}) - f(x_{n+1})}{2\Delta x^2} + \frac{f(x_{n+1}) - f(x_n)}{\Delta x^2} \right ) \\
    &=\frac{f(x_{n+1}) - f(x_n)}{\Delta x} - \left ( \frac{f(x_{n+1}) - 2f(x_n) + f(x_{n-1})}{2\Delta x}\right ) \\
    &=\frac{2f(x_{n+1}) - 2f(x_n) - f(x_{n+1}) + 2f(x_n) - f(x_{n-1})}{2 \Delta x} \\
    &=\frac{f(x_{n+1}) - f(x_{n-1})}{2 \Delta x}
\end{aligned}$$

This finite-difference is second order accurate and is centered about the point it is meant to approximate ($x_n$).  We can show that it is second order by again considering the remainder term's derivative

$$\begin{aligned}
    R_2'(x) &= \left(\sum^{2}_{i=0} \left( \prod^{2}_{j=0,~j\neq i} (x - x_j) \right )\right ) \frac{f'''(c)}{3!} \\
    &= \left ( (x - x_{n+1}) (x - x_{n-1}) + (x-x_n) (x-x_{n-1}) + (x-x_n)(x-x_{n+1}) \right ) \frac{f'''(c)}{3!}
\end{aligned}$$

Again evaluating this expression at $x = x_n$ and assuming evenly space points we have

$$R_2'(x_n) = -\Delta x^2 \frac{f'''(c)}{3!}$$

showing that our error is $\mathcal{O}(\Delta x^2)$.

```python
f = lambda x: numpy.sin(x)
f_prime = lambda x: numpy.cos(x)

# Use uniform discretization
x = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, 1000)
N = 21
x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N)
delta_x = x_hat[1] - x_hat[0]

# Compute derivative
f_prime_hat = numpy.empty(x_hat.shape)
f_prime_hat[1:-1] = (f(x_hat[2:]) - f(x_hat[:-2])) / (2. * delta_x)

# Use first-order differences for points at edge of domain
f_prime_hat[0] = (f(x_hat[1]) - f(x_hat[0])) / delta_x     # Forward Difference at x_0
f_prime_hat[-1] = (f(x_hat[-1]) - f(x_hat[-2])) / delta_x  # Backward Difference at x_N
#f_prime_hat[0] = (-3.0 * f(x_hat[0]) + 4.0 * f(x_hat[1]) - f(x_hat[2])) / (2.0 * delta_x)
#f_prime_hat[-1] = (3.0 * f(x_hat[-1]) - 4.0 * f(x_hat[-2]) + f(x_hat[-3])) / (2.0 * delta_x)
```

```python
fig = plt.figure()
plt.rcdefaults()
axes = fig.add_subplot(1, 1, 1)

axes.plot(x, f_prime(x), 'k',label="$f'(x)$")
axes.plot(x_hat, f_prime_hat, 'ro')
axes.plot(x, f(x),'g--',label="$f(x)$")
axes.plot(x_hat, f(x_hat), 'go')
axes.set_xlim((x[0], x[-1]))
# axes.set_ylim((-1.1, 1.1))
axes.grid()
axes.legend()

plt.show()
```

```python
# Compute the error as a function of delta_x
delta_x = []
error = []
# for N in range(2, 101):
for N in range(50, 1000, 50):
    x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N + 1)
    delta_x.append(x_hat[1] - x_hat[0])

    # Compute derivative
    f_prime_hat = numpy.empty(x_hat.shape)
    f_prime_hat[1:-1] = (f(x_hat[2:]) - f(x_hat[:-2])) / (2 * delta_x[-1])

    # Use first-order differences for points at edge of domain
    f_prime_hat[0] = (f(x_hat[1]) - f(x_hat[0])) / delta_x[-1]
    f_prime_hat[-1] = (f(x_hat[-1]) - f(x_hat[-2])) / delta_x[-1]
    # Use second-order differences for points at edge of domain
    f_prime_hat[0] =  (-3.0 * f(x_hat[0])  +  4.0 * f(x_hat[1])  + - f(x_hat[2]))  / (2.0 * delta_x[-1])
    f_prime_hat[-1] = ( 3.0 * f(x_hat[-1]) + -4.0 * f(x_hat[-2]) +   f(x_hat[-3])) / (2.0 * delta_x[-1])

    # Compute Error
    error.append(numpy.linalg.norm(numpy.abs(f_prime(x_hat) - f_prime_hat), ord=numpy.infty))
#     error.append(numpy.linalg.norm(numpy.abs(f_prime(x_hat) - f_prime_hat), ord=2))

error = numpy.array(error)
delta_x = numpy.array(delta_x)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)

axes.loglog(delta_x, error, "ro", label="Approx. Derivative")

order_C = lambda delta_x, error, order: numpy.exp(numpy.log(error) - order * numpy.log(delta_x))
axes.loglog(delta_x, order_C(delta_x[0], error[0], 1.0) * delta_x**1.0, 'b--', label="1st Order")
axes.loglog(delta_x, order_C(delta_x[0], error[0], 2.0) * delta_x**2.0, 'r--', label="2nd Order")
axes.legend(loc=4)
axes.set_title("Convergence of 2nd Order Differences")
axes.set_xlabel("$\Delta x$")
axes.set_ylabel("$|f'(x) - \hat{f}'(x)|$")
axes.grid()

plt.show()
```

### Example 3: Alternative Derivations

An alternative method for finding finite-difference formulas is by using Taylor series expansions about the point we want to approximate.  The Taylor series about $x_n$ is

$$f(x) = f(x_n) + (x - x_n) f'(x_n) + \frac{(x - x_n)^2}{2!} f''(x_n) + \frac{(x - x_n)^3}{3!} f'''(x_n) + \mathcal{O}((x - x_n)^4)$$

Say we want to derive the second order accurate, first derivative approximation that we just did, this requires the values $(x_{n+1}, f(x_{n+1})$ and $(x_{n-1}, f(x_{n-1})$.  We can express these values via our Taylor series approximation above as

\begin{aligned}
    f(x_{n+1}) &= f(x_n) + (x_{n+1} - x_n) f'(x_n) + \frac{(x_{n+1} - x_n)^2}{2!} f''(x_n) + \frac{(x_{n+1} - x_n)^3}{3!} f'''(x_n) + \mathcal{O}((x_{n+1} - x_n)^4) \\
\end{aligned}

or
\begin{aligned}
&= f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)
\end{aligned}

and

\begin{align}
f(x_{n-1}) &= f(x_n) + (x_{n-1} - x_n) f'(x_n) + \frac{(x_{n-1} - x_n)^2}{2!} f''(x_n) + \frac{(x_{n-1} - x_n)^3}{3!} f'''(x_n) + \mathcal{O}((x_{n-1} - x_n)^4)
\end{align}

\begin{align}
&= f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)
\end{align}

Or all together (for regularly spaced points),
\begin{align}
f(x_{n+1}) &= f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)\\
f(x_{n-1})&= f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)
\end{align}

Now to find out how to combine these into an expression for the derivative we assume our approximation looks like

$$
    f'(x_n) + R(x_n) = A f(x_{n+1}) + B f(x_n) + C f(x_{n-1})
$$

where $R(x_n)$ is our error.

Plugging in the Taylor series approximations we find

$$\begin{aligned}
    f'(x_n) + R(x_n) &= A \left ( f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)\right ) \\
    & + B f(x_n) \\
    & + C \left ( f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4) \right )
\end{aligned}$$

Since we want $R(x_n) = \mathcal{O}(\Delta x^2)$ we want all terms lower than this to cancel except for those multiplying $f'(x_n)$ as those should sum to 1 to give us our approximation.  Collecting the terms with common evaluations of the derivatives on $f(x_n)$ we get a series of expressions for the coefficients $A$, $B$, and $C$ based on the fact we want an approximation to $f'(x_n)$.  The $n=0$ terms collected are $A + B + C$ and are set to 0 as we want the $f(x_n)$ term to also cancel.

$$\begin{aligned}
    f(x_n):&  &A + B + C &= 0 \\
    f'(x_n): & &A \Delta x - C \Delta x &= 1 \\
    f''(x_n): & &A \frac{\Delta x^2}{2} + C \frac{\Delta x^2}{2} &= 0
\end{aligned} $$

This last equation $\Rightarrow A = -C$, using this in the second equation gives $A = \frac{1}{2 \Delta x}$ and $C = -\frac{1}{2 \Delta x}$.  The first equation then leads to $B = 0$.  Putting this altogether then gives us our previous expression including an estimate for the error:

$$\begin{aligned}
    f'(x_n) + R(x_n) &= \quad \frac{1}{2 \Delta x} \left ( f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)\right ) \\
    & \quad + 0 \cdot f(x_n) \\
    & \quad - \frac{1}{2 \Delta x} \left ( f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4) \right ) \\
    &=  f'(x_n) + \frac{1}{2 \Delta x} \left ( \frac{2 \Delta x^3}{3!} f'''(x_n) + \mathcal{O}(\Delta x^4)\right )
\end{aligned}$$
so that we find
$$
    R(x_n) = \frac{\Delta x^2}{3!} f'''(x_n) + \mathcal{O}(\Delta x^3) = \mathcal{O}(\Delta x^2)
$$

#### Another way...

There is one more way to derive the second order accurate, first order finite-difference formula.  Consider the two first order forward and backward finite-differences averaged together:

$$\frac{D_1^+(f(x_n)) + D_1^-(f(x_n))}{2} = \frac{f(x_{n+1}) - f(x_n) + f(x_n) - f(x_{n-1})}{2 \Delta x} = \frac{f(x_{n+1}) - f(x_{n-1})}{2 \Delta x}$$

### Example 4: Higher Order Derivatives

Using our Taylor series approach lets derive the second order accurate second derivative formula.  Again we will use the same points and the Taylor series centered at $x = x_n$ so we end up with the same expression as before:

$$\begin{aligned}
    f''(x_n) + R(x_n) &= \quad A \left ( f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \frac{\Delta x^4}{4!} f^{(4)}(x_n) + \mathcal{O}(\Delta x^5)\right ) \\
    &+ \quad B \cdot f(x_n) \\
    &+ \quad C \left ( f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \frac{\Delta x^4}{4!} f^{(4)}(x_n) + \mathcal{O}(\Delta x^5) \right )
\end{aligned}$$

except this time we want to leave $f''(x_n)$ on the right hand side.

Try out the same trick as before and see if you can setup the equations that need to be solved.

Doing the same trick as before we have the following expressions:

$$\begin{aligned}
    f(x_n): & & A + B + C &= 0\\
    f'(x_n): & & A \Delta x - C \Delta x &= 0\\
    f''(x_n): & & A \frac{\Delta x^2}{2} + C \frac{\Delta x^2}{2} &= 1
\end{aligned}$$

The second equation implies $A = C$ which combined with the third implies

$$A = C = \frac{1}{\Delta x^2}$$

Finally the first equation gives

$$B = -\frac{2}{\Delta x^2}$$

leading to the final expression

$$\begin{aligned}
    f''(x_n) + R(x_n) &= \quad \frac{1}{\Delta x^2} \left ( f(x_n) + \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) + \frac{\Delta x^3}{3!} f'''(x_n) + \frac{\Delta x^4}{4!} f^{(4)}(x_n) + \mathcal{O}(\Delta x^5)\right ) \\
    &+ \quad -\frac{2}{\Delta x^2} \cdot f(x_n) \\
    &+ \quad \frac{1}{\Delta x^2} \left ( f(x_n) - \Delta x f'(x_n) + \frac{\Delta x^2}{2!} f''(x_n) - \frac{\Delta x^3}{3!} f'''(x_n) + \frac{\Delta x^4}{4!} f^{(4)}(x_n) + \mathcal{O}(\Delta x^5) \right ) \\
    &= f''(x_n) + \frac{1}{\Delta x^2} \left(\frac{2 \Delta x^4}{4!} f^{(4)}(x_n) + \mathcal{O}(\Delta x^5) \right )
\end{aligned}
$$
so that

$$
    R(x_n) = \frac{\Delta x^2}{12} f^{(4)}(x_n) + \mathcal{O}(\Delta x^3)
$$

```python
f = lambda x: numpy.sin(x)
f_dubl_prime = lambda x: -numpy.sin(x)

# Use uniform discretization
x = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, 1000)
N = 40
x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N)
delta_x = x_hat[1] - x_hat[0]

# Compute derivative
f_dubl_prime_hat = numpy.empty(x_hat.shape)
f_dubl_prime_hat[1:-1] = (f(x_hat[2:]) -2.0 * f(x_hat[1:-1]) + f(x_hat[:-2])) / (delta_x**2)

# Use first-order differences for points at edge of domain
f_dubl_prime_hat[0] = (2.0 * f(x_hat[0]) - 5.0 * f(x_hat[1]) + 4.0 * f(x_hat[2]) - f(x_hat[3])) / delta_x**2
f_dubl_prime_hat[-1] = (2.0 * f(x_hat[-1]) - 5.0 * f(x_hat[-2]) + 4.0 * f(x_hat[-3]) - f(x_hat[-4])) / delta_x**2
```

```python
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)

axes.plot(x, f_dubl_prime(x), 'k')
axes.plot(x_hat, f_dubl_prime_hat, 'ro')
axes.set_xlim((x[0], x[-1]))
axes.set_ylim((-1.1, 1.1))
axes.grid()

plt.show()
```

```python
f = lambda x: numpy.sin(x)
f_dubl_prime = lambda x: -numpy.sin(x)

# Compute the error as a function of delta_x
delta_x = []
error = []
# for N in xrange(2, 101):
for N in range(50, 1000, 50):
    x_hat = numpy.linspace(-2 * numpy.pi, 2 * numpy.pi, N)
    delta_x.append(x_hat[1] - x_hat[0])

    # Compute derivative
    f_dubl_prime_hat = numpy.empty(x_hat.shape)
    f_dubl_prime_hat[1:-1] = (f(x_hat[2:]) -2.0 * f(x_hat[1:-1]) + f(x_hat[:-2])) / (delta_x[-1]**2)

    # Use second-order differences for points at edge of domain
    f_dubl_prime_hat[0] = (2.0 * f(x_hat[0]) - 5.0 * f(x_hat[1]) + 4.0 * f(x_hat[2]) - f(x_hat[3])) / delta_x[-1]**2
    f_dubl_prime_hat[-1] = (2.0 * f(x_hat[-1]) - 5.0 * f(x_hat[-2]) + 4.0 * f(x_hat[-3]) - f(x_hat[-4])) / delta_x[-1]**2

    error.append(numpy.linalg.norm(numpy.abs(f_dubl_prime(x_hat) - f_dubl_prime_hat), ord=numpy.infty))

error = numpy.array(error)
delta_x = numpy.array(delta_x)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)

# axes.plot(delta_x, error)
axes.loglog(delta_x, error, "ko", label="Approx. Derivative")
order_C = lambda delta_x, error, order: numpy.exp(numpy.log(error) - order * numpy.log(delta_x))
axes.loglog(delta_x, order_C(delta_x[2], error[2], 1.0) * delta_x**1.0, 'b--', label="1st Order")
axes.loglog(delta_x, order_C(delta_x[2], error[2], 2.0) * delta_x**2.0, 'r--', label="2nd Order")
axes.legend(loc=4)
axes.grid()

plt.show()
```


```python
import numpy as np
import numpy.polynomial.polynomial as n_poly
import numpy.polynomial.legendre as leg
from scipy import integrate
from scipy import optimize
from scipy import interpolate
import sympy as sym
%pylab inline

```

```python
# Set the interval for L2 and the number of basis functions
A=-1.0
B=1.0
N=15
```

```python
# Define the scalar product in L2([A,B])
t = sym.symbols('t')

def scalar_prod(p0,p1):
    assert len(p0.free_symbols) <= 1, "I can only do this for single variable functions..."
    tt = p0.free_symbols.pop() if len(p0.free_symbols) == 1 else sym.symbols('t')
    foo1 = sym.lambdify(tt, p0, 'numpy')
    foo2 = sym.lambdify(tt, p1, 'numpy')
    foo3 = lambda x : foo1(x) * foo2(x)
    foo = integrate.quad(foo3,A,B)
    return foo[0]
```

```python

PLk = [1+0.*t] # Force it to be a sympy expression
PMk = [1+0.*t]

for k in range(1,N):
    s = 0
    for j in range(0,k):
        s+= scalar_prod(t**k,PLk[j])/scalar_prod(PLk[j],PLk[j])*PLk[j]
    pk = t**k-s
    # pk = pk/sym.sqrt(scalar_prod(pk,pk))
    pk = pk/pk.subs(t,1.)
    PLk.append(pk)
    PMk.append(t**k)

xx = np.linspace(A,B,100)
figure()
fs = sym.lambdify(t, PMk[0], 'numpy')
plot(xx, np.ones(xx.shape)*fs(xx))
for k in range(1,N):
    fs = sym.lambdify(t, PMk[k], 'numpy')
    plot(xx, fs(xx))
figure()
fs = sym.lambdify(t, PLk[0], 'numpy')
plot(xx, np.ones(xx.shape)*fs(xx))
for k in range(1,N):
    fs = sym.lambdify(t, PLk[k], 'numpy')
    plot(xx, fs(xx))

figure()
for k in range(N):
    c = np.zeros(k)
    if k == 0:
        c = 1
    else:
        c[-1] = 1
    plot(xx, leg.legval(xx,c))

```

```python
#Best Approximation
f = sym.cos(8.*np.pi/(B-A)*t)

M_LP = np.zeros([N,N])
M_MP = np.zeros([N,N])

f_M = np.zeros([N,1])

f_L = np.zeros([N,1])

for i in range(N):
    f_M[i] = scalar_prod(PMk[i],f)
    f_L[i] = scalar_prod(PLk[i],f)
    for j in range(N):
        M_LP[i,j] = scalar_prod(PLk[i],PLk[j])
        M_MP[i,j] = scalar_prod(PMk[i],PMk[j])

p_L = np.linalg.solve(M_LP, f_L)

p_M = np.linalg.solve(M_MP, f_M)
f_p_L = 0.
f_p_M = 0.
for i in range(N):
    f_p_L += p_L[i]*PLk[i]
    f_p_M += p_M[i]*PMk[i]

fpL = sym.lambdify(t, f_p_L[0], 'numpy')
fpM = sym.lambdify(t, f_p_M[0], 'numpy')
ff = sym.lambdify(t, f, 'numpy')

plot(xx, ff(xx), 'r*')
plot(xx, fpL(xx), 'b')
plot(xx, fpL(xx), 'g')

# print M_LP
print(np.linalg.cond(M_LP))
print(np.linalg.cond(M_MP))
```

```python
N_lagrange = N
x_interp = np.linspace(A,B,N_lagrange)
x_cheby = np.array([np.cos((2.*j-1)/(2.*N_lagrange)*np.pi) for j in range(1,N_lagrange+1)])
print x_cheby
lagrange_interp = interpolate.lagrange(x_interp, ff(x_interp))
cheb_interp = interpolate.lagrange(x_cheby, ff(x_cheby))
tck = interpolate.splrep(x_interp, ff(x_interp), s=0)
yspline = interpolate.splev(xx, tck, der=0)
figure()
plot(xx, lagrange_interp(xx))
figure()
plot(xx, cheb_interp(xx))
figure()
plot(xx, yspline)

print integrate.quad(lambda x: (ff(x) - lagrange_interp(x))**2,A,B)

print integrate.quad(lambda x: (ff(x) - cheb_interp(x))**2,A,B)

print integrate.quad(lambda x: (ff(x) - interpolate.splev(x, tck, der=0))**2,A,B)

print integrate.quad(lambda x: (ff(x) - fpL(x))**2,A,B)

print integrate.quad(lambda x: (ff(x) - fpM(x))**2,A,B)
```

```python
# Root of Legendre polynomial
fs = sym.lambdify(t, PLk[8], 'numpy')
x0 = linspace(A,B,8)
# roots = optimize.brentq(fs, A, B)
roots = optimize.fsolve(fs, x0)
# roots = optimize.root(fs, x0)
print roots
plot(xx,fs(xx))
x0[1] = -0.85
x0[-2] = 0.85
roots = optimize.fsolve(fs, x0)
print roots

c = np.zeros(8+1)
c[-1] = 1
print leg.legroots(c)

xg,wg = np.polynomial.legendre.leggauss(8)

print xg
```

```python
# Gauss Integration

f_gauss = lambda x : 8*np.sin(x)+3*np.cos(15.*x)

I = integrate.quad(f_gauss,A,B)

n_gauss = 10

xg,wg = np.polynomial.legendre.leggauss(n_gauss)

In = np.dot(wg,f_gauss(xg))

error = list()
Ngmax = 40
order = np.arange(1,Ngmax)
for i in order:
    xg,wg = np.polynomial.legendre.leggauss(i)
#     In = np.dot(wg,f_gauss(xg))
    In = np.einsum('j,j',wg,f_gauss(xg))
    error.append(np.abs(I[0]-In))

# print error
plot(order, error)
figure()
loglog(order,error)
loglog(order,np.ones(Ngmax-1)*I[1])
figure()
semilogy(order, error)
semilogy(order,np.ones(Ngmax-1)*I[1])

## REMARK QUADRATURE ORDER ON (f, vi) FOR BEST APPROXIMATION
```

```python

```


```python
from sympy import *
from sympy import poly
x = symbols("x")

def reduction(expr, p) -> bool:
    p_exp = poly(expr)
    values = []
    for i in range(p):
        result = p_exp.subs('x', i) % p
        print(expr + ' with x:' + str(i) +' = ' + str(result))
        if values.__contains__(result): return False
        values.append(result)
    return True

#pass argument with format like '2*x^2+3*x+7'
def bijectivity(expr):
    is_reduction = reduction(expr, 4)
    if(is_reduction): print('polynomial is bijective')
    else: print('polynomial is not bijective')

#pass argument with format like '2*x^2+3*x+7'
def transitivity(expr) -> bool:
    is_reduction = reduction(expr, 8)
    if(is_reduction): print('polynomial is transitive')
    else: print('polynomial is not transitive')

```


# Non Linear Regression Analysis

Estaimted time needed: **20** minutes

## Objectives

After completing this lab you will be able to:

-   Differentiate between Linear and non-linear regression
-   Use Non-linear regression model in Python

If the data shows a curvy trend, then linear regression will not produce very accurate results when compared to a non-linear regression because, as the name implies, linear regression presumes that the data is linear.
Let's learn about non linear regressions and apply an example on python. In this notebook, we fit a non-linear model to the datapoints corrensponding to China's GDP from 1960 to 2014.

Importing required libraries

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

Though Linear regression is very good to solve many problems, it cannot be used for all datasets. First recall how linear regression, could model a dataset. It models a linear relation between a dependent variable y and independent variable x. It had a simple equation, of degree 1, for example y = $2x$ + 3.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 2*(x) + 3
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
#plt.figure(figsize=(8,6))
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

Non-linear regressions are a relationship between independent variables $x$ and a dependent variable $y$ which result in a non-linear function modeled data. Essentially any relationship that is not linear can be termed as non-linear, and is usually represented by the polynomial of $k$ degrees (maximum power of $x$).

$$ \\ y = a x^3 + b x^2 + c x + d \\ $$

Non-linear functions can have elements like exponentials, logarithms, fractions, and others. For example: $$ y = \\log(x)$$

Or even, more complicated such as :
$$ y = \\log(a x^3 + b x^2 + c x + d)$$

Let's take a look at a cubic function's graph.

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 1*(x**3) + 1*(x**2) + 1*x + 3
y_noise = 20 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

As you can see, this function has $x^3$ and $x^2$ as independent variables. Also, the graphic of this function is not a straight line over the 2D plane. So this is a non-linear function.

Some other types of non-linear functions are:

### Quadratic

$$ Y = X^2 $$

```python
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

y = np.power(x,2)
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Exponential

An exponential function with base c is defined by $$ Y = a + b c^X$$ where b ≠0, c > 0 , c ≠1, and x is any real number. The base, c, is constant and the exponent, x, is a variable.

```python
X = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

Y= np.exp(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Logarithmic

The response $y$ is a results of applying logarithmic map from input $x$'s to output variable $y$. It is one of the simplest form of **log()**: i.e. $$ y = \\log(x)$$

Please consider that instead of $x$, we can use $X$, which can be polynomial representation of the $x$'s. In general form it would be written as
\\begin{equation}
y = \\log(X)
\\end{equation}

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = np.log(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Sigmoidal/Logistic

$$ Y = a + \\frac{b}{1+ c^{(X-d)}}$$

```python
X = np.arange(-5.0, 5.0, 0.1)

Y = 1-4/(1+np.power(3, X-2))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

# Non-Linear Regression example

For an example, we're going to try and fit a non-linear model to the datapoints corresponding to China's GDP from 1960 to 2014. We download a dataset with two columns, the first, a year between 1960 and 2014, the second, China's corresponding annual gross domestic income in US dollars for that year.

```python
import numpy as np
import pandas as pd

#downloading dataset
!wget -nv -O china_gdp.csv https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv

df = pd.read_csv("china_gdp.csv")
df.head(10)
```

    2020-09-21 08:24:59 URL:https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv [1218/1218] -> "china_gdp.csv" [1]

      Year
      Value

      0
      1960
      5.918412e+10

      1
      1961
      4.955705e+10

      2
      1962
      4.668518e+10

      3
      1963
      5.009730e+10

      4
      1964
      5.906225e+10

      5
      1965
      6.970915e+10

      6
      1966
      7.587943e+10

      7
      1967
      7.205703e+10

      8
      1968
      6.999350e+10

      9
      1969
      7.871882e+10

**Did you know?** When it comes to Machine Learning, you will likely be working with large datasets. As a business, where can you host your data? IBM is offering a unique opportunity for businesses, with 10 Tb of IBM Cloud Object Storage: [Sign up now for free](http://cocl.us/ML0101EN-IBM-Offer-CC)

### Plotting the Dataset

This is what the datapoints look like. It kind of looks like an either logistic or exponential function. The growth starts off slow, then from 2005 on forward, the growth is very significant. And finally, it decelerate slightly in the 2010s.

```python
plt.figure(figsize=(8,5))
x_data, y_data = (df["Year"].values, df["Value"].values)
plt.plot(x_data, y_data, 'ro')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

### Choosing a model

From an initial look at the plot, we determine that the logistic function could be a good approximation,
since it has the property of starting with a slow growth, increasing growth in the middle, and then decreasing again at the end; as illustrated below:

```python
X = np.arange(-5.0, 5.0, 0.1)
Y = 1.0 / (1.0 + np.exp(-X))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

The formula for the logistic function is the following:

$$ \\hat{Y} = \\frac1{1+e^{\\beta_1(X-\\beta_2)}}$$

$\\beta_1$: Controls the curve's steepness,

$\\beta_2$: Slides the curve on the x-axis.

### Building The Model

Now, let's build our regression model and initialize its parameters.

```python
def sigmoid(x, Beta_1, Beta_2):
     y = 1 / (1 + np.exp(-Beta_1*(x-Beta_2)))
     return y
```

Lets look at a sample sigmoid line that might fit with the data:

```python
beta_1 = 0.10
beta_2 = 1990.0

#logistic function
Y_pred = sigmoid(x_data, beta_1 , beta_2)

#plot initial prediction against datapoints
plt.plot(x_data, Y_pred*15000000000000.)
plt.plot(x_data, y_data, 'ro')
```

Our task here is to find the best parameters for our model. Lets first normalize our x and y:

```python
# Lets normalize our data
xdata =x_data/max(x_data)
ydata =y_data/max(y_data)
```

#### How we find the best parameters for our fit line?

we can use **curve_fit** which uses non-linear least squares to fit our sigmoid function, to data. Optimal values for the parameters so that the sum of the squared residuals of sigmoid(xdata, \*popt) - ydata is minimized.

popt are our optimized parameters.

```python
from scipy.optimize import curve_fit
popt, pcov = curve_fit(sigmoid, xdata, ydata)
#print the final parameters
print(" beta_1 = %f, beta_2 = %f" % (popt[0], popt[1]))
```

     beta_1 = 690.447527, beta_2 = 0.997207

Now we plot our resulting regression model.

```python
x = np.linspace(1960, 2015, 55)
x = x/max(x)
plt.figure(figsize=(8,5))
y = sigmoid(x, *popt)
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(x,y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

## Practice

Can you calculate what is the accuracy of our model?

```python
# write your code here

```

Double-click **here** for the solution.

Want to learn more?

IBM SPSS Modeler is a comprehensive analytics platform that has many machine learning algorithms. It has been designed to bring predictive intelligence to decisions made by individuals, by groups, by systems – by your enterprise as a whole. A free trial is available through this course, available here: SPSS Modeler

Also, you can use Watson Studio to run these notebooks faster with bigger datasets. Watson Studio is IBM's leading cloud solution for data scientists, built by data scientists. With Jupyter notebooks, RStudio, Apache Spark and popular libraries pre-packaged in the cloud, Watson Studio enables data scientists to collaborate on their projects without having to install anything. Join the fast-growing community of Watson Studio users today with a free account at Watson Studio

### Thank you for completing this lab!

## Author

Saeed Aghabozorgi

### Other Contributors

Joseph Santarcangelo

## Change Log

| Date (YYYY-MM-DD) | Version | Changed By | Change Description                 |
| ----------------- | ------- | ---------- | ---------------------------------- |
| 2020-08-27        | 2.0     | Lavanya    | Moved lab to course repo in GitLab |
|                   |         |            |                                    |
|                   |         |            |                                    |

##
 © IBM Corporation 2020. All rights reserved.

Non Linear Regression/Introduction and working example/ML0101EN-Reg-NoneLinearRegression-py-v1(1).ipynb


## A Quick Tour of DifferentialEquations.jl

DifferentialEquations.jl is a metapackage for solving differential equations in Julia. The basic workflow is:

- Define a problem
- Solve a problem
- Plot the solution

The API between different types of differential equations is unified through multiple dispatch.

See the [DifferentialEquations.jl Documentation](http://docs.juliadiffeq.org/latest/index.html).

## Example: Lotka-Volterra ODE

$$\begin{align}
x' &= ax - bxy\\
y' &= -cy + dxy
\end{align}$$

```julia
using DifferentialEquations
# Define a problem
p = (1.0,2.0,1.5,1.25) # a,b,c,d
f = function (du,u,p,t) # Define f as an in-place update into du
    a,b,c,d = p
    du[1] = a*u[1] - b*u[1]*u[2]
    du[2] = -c*u[2]+ d*u[1]*u[2]
end
u0 = [1.0;1.0]; tspan = (0.0,10.0)
prob = ODEProblem(f,u0,tspan,p);
```

    INFO: Recompiling stale cache file C:\Users\Chris\.julia\lib\v0.6\DifferentialEquations.ji for module DifferentialEquations.

```julia
# Solve the problem
sol = solve(prob);
```

```julia
# Plot the solution using the plot recipe
using Plots; gr() # Using the Plotly Backend
plot(sol,title="All Plots.jl Attributes are Available")
```

The plot recipe [contains special fields](http://docs.juliadiffeq.org/latest/basics/plot.html) for plotting phase diagrams and other transformations:

```julia
plot(sol,title="Phase Diagram",vars=(1,2))
```

## Extra Features

The solution object acts both as an array and as an interpolation of the solution

```julia
@show sol.t[3] # Time at the 3rd timestep
@show sol[3] # Value at the third timestep
@show sol(5) # Value at t=5 using the interpolation
```

    sol.t[3] = 0.2927716363825574
    sol[3] = [0.768635, 0.887673]
    sol(5) = [1.45932, 0.99208]

    2-element Array{Float64,1}:
     1.45932
     0.99208

## Stochastic Differential Equations

Also included are problems for stochastic differential equations

```julia
g = function (du,u,p,t)
    du[1] = .5*u[1]
    du[2] = .1*u[2]
end
prob = SDEProblem(f,g,u0,tspan,p)
sol = solve(prob,dt=1/2^4)
plot(sol)
```

## Documentation and Extended Tutorials

For more information, see the documentation: https://github.com/JuliaDiffEq/DifferentialEquations.jl. The repository [DiffEqTutorials.jl](https://github.com/JuliaDiffEq/DiffEqTutorials.jl) has a large array of tutorials for using the package in depth.

## Problems

### Problem 1

The DifferentialEquations.jl algorithms choose the number type of their calculation given their input. Use this fact to solve the [Lorenz equation](https://en.wikipedia.org/wiki/Lorenz_system) using BigFloats. You may want to [check out the example notebooks](https://github.com/JuliaDiffEq/DiffEqTutorials.jl). Make a 3D plot of the Lorenz attractor using the plot recipe.

### Problem 2

Use the [event handling](http://docs.juliadiffeq.org/latest/features/callback_functions.html) the model a bouncing ball with friction, i.e. at every bounce the velocity flips but is decreased to 80%. Does the ball eventually stop bouncing?


### DEMDP02
# Asset Replacement Model

Profit-maximizing entrepreneur must decide when to replace an aging asset.

```python
import numpy as np
import matplotlib.pyplot as plt
from compecon import BasisSpline, DPmodel, qnwnorm, demo, NLP
```

### Model Parameters

The maximum age of the asset is $A=6$,  production function coefficients are $\alpha =[50.0, -2.5, -2.5]$, net replacement cost is $\kappa = 40$, long-run mean unit profit is $\bar{p} = 1$, the unit profit autoregression coefficient is $\gamma = 0.5$, the standard deviation of unit profit shock is $\sigma = 0.15$, and the discount factor is $\delta = 0.9$.

```python
A       = 6
alpha   = np.array([50, -2.5, -2.5])
kappa   = 40
pbar    = 1.0
gamma   = 0.5
sigma   = 0.15
delta   = 0.9
```

### State space
The state variables are the current unit profit $p$ (continuous) and the age of asset $a\in\{1,2,\dots,A\}$.

```python
dstates = [f'a={age+1:d}' for age in range(A)]
print('Discrete states are :', dstates)
```

    Discrete states are : ['a=1', 'a=2', 'a=3', 'a=4', 'a=5', 'a=6']

Here, we approximate the unit profit with a cubic spline basis of $n=200$ nodes, over the interval $p\in[0,2]$.

```python
n  = 200
pmin, pmax = 0.0, 2.0
basis = BasisSpline(n, pmin, pmax, labels=['unit profit'])
```

### Action space
There is only one choice variable $j$: whether to keep(0) or replace(1) the asset.

```python
options = ['keep', 'replace']
```

### Reward Function
The instant profit depends on the asset age and whether it is replaced. An asset of age $a=A$ must be replaced.
If the asset is replaced, the profit is $50p$ minus the cost of replacement $\kappa$; otherwise the profit depends on the age of the asset, $(\alpha_0 + \alpha_1 a + \alpha_2 a^2)p$.

```python
def profit(p, x, i, j):
    a = i + 1
    if j or a == A:
        return p * 50 - kappa
    else:
        return p * (alpha[0] + alpha[1] * a + alpha[2] * a ** 2 )
```

### State Transition Function

The unit profit $p$ follows a Markov Process
\begin{equation}p' = \bar{p} + \gamma(p-\bar{p}) + \epsilon \end{equation}
where $\epsilon \sim N(0,\sigma^2)$.

```python
def transition(p, x, i, j, in_, e):
    return pbar + gamma * (p - pbar) + e
```

The continuous shock must be discretized. Here we use Gauss-Legendre quadrature to obtain nodes and weights defining a discrete distribution that matches the first 10 moments of the Normal distribution (this is achieved with $m=5$ nodes and weights).

```python
m = 5
e, w = qnwnorm(m,0,sigma ** 2)
```

On the other hand, the age of the asset is deterministic: if it is replaced now it will be new ($a=0$) next period; otherwise its age will be $a+1$ next period if current age is $a$.

```python
h = np.zeros((2, A),int)
h[0, :-1] = np.arange(1, A)
```

## Model Structure

```python
model = DPmodel(basis, profit, transition,
                i=dstates,
                j=options,
                discount=delta, e=e, w=w, h=h)
```

### SOLUTION
The ```solve``` method returns a pandas ```DataFrame```.

```python
S = model.solve()
S.head()
```

    Solving infinite-horizon model collocation equation by Newton's method
    iter change       time
    ------------------------------
       0       3.7e+02    0.2599
       1       4.3e+01    0.5558
       2       1.2e+01    0.8437
       3       5.7e-01    1.1456
       4       3.7e-13    1.4555
    Elapsed Time =    1.46 Seconds

      unit profit
      i
      value
      resid
      j*
      value[keep]
      value[replace]

      unit profit

      a=1
      0.000000
      0.000000
      0
      234.982583
      2.842171e-14
      keep
      234.982583
      206.124123

      0.001001
      0.001001
      0
      235.058576
      -6.159590e-07
      keep
      235.058576
      206.209036

      0.002001
      0.002001
      0
      235.134563
      -6.366804e-07
      keep
      235.134563
      206.293949

      0.003002
      0.003002
      0
      235.210544
      -2.317537e-07
      keep
      235.210544
      206.378862

      0.004002
      0.004002
      0
      235.286520
      4.292312e-07
      keep
      235.286520
      206.463774

## Analysis

### Plot Action-Contingent Value Functions

```python
# Compute and Plot Critical Unit Profit Contributions
pcrit = [NLP(lambda s: model.Value_j(s)[i].dot([1,-1])).broyden(0.0)[0] for i in range(A)]
vcrit = [model.Value(s)[i] for i, s in enumerate(pcrit)]
```

```python
fig1 = demo.figure('Action-Contingent Value Functions', 'Net Unit Profit','Value', figsize=[10,5])

cc = np.linspace(0.3,0.9,model.dims.ni)

for a, i in enumerate(dstates):
    plt.plot(S.loc[i,'value[keep]'] ,color=plt.cm.Blues(cc[a]), label='Keep ' + i)
    if pmin < pcrit[a] < pmax:
        demo.annotate(pcrit[a], vcrit[a], f'$p^*_{a+1}$', 'wo',(0, 0), fs=11, ms=18)
        print(f'Age {a+1:d}  Profit {pcrit[a]:5.2f}')

plt.plot(S.loc['a=1','value[replace]'], color=plt.cm.Oranges(0.5),label='Replace')
plt.legend()
```

### Plot Residual

```python
S['resid2'] = 100*S['resid'] / S['value']

fig2 = demo.figure('Bellman Equation Residual','Net Unit Profit','Percent Residual')
demo.qplot('unit profit','resid2','i',S)
plt.legend(model.labels.i)
```

### SIMULATION

```python
T = 50
nrep = 10000
sinit = np.full(nrep, pbar)
iinit = 0
data = model.simulate(T,sinit,iinit, seed=945)
data['age'] = data.i.cat.codes + 1
```

### Print Ergodic Moments

```python
frm = '\t{:<10s} = {:5.2f}'

print('\nErgodic Means')
print(frm.format('Price', data['unit profit'].mean()))
print(frm.format('Age', data['age'].mean()))
print('\nErgodic Standard Deviations')
print(frm.format('Price', data['unit profit'].std()))
print(frm.format('Age', data['age'].std()))
```

    Ergodic Means
    	Price      =  1.00
    	Age        =  2.01

    Ergodic Standard Deviations
    	Price      =  0.17
    	Age        =  0.83

### Plot Simulated and Expected Continuous State Path

```python
data.head()
```

      time
      _rep
      i
      unit profit
      j*
      age

      0
      0
      0
      a=1
      1.0
      keep
      1

      1
      0
      1
      a=1
      1.0
      keep
      1

      2
      0
      2
      a=1
      1.0
      keep
      1

      3
      0
      3
      a=1
      1.0
      keep
      1

      4
      0
      4
      a=1
      1.0
      keep
      1

```python
subdata = data[data['_rep'].isin(range(3))]
```

```python
fig3 = demo.figure('Simulated and Expected Price', 'Period','Net Unit Profit')
demo.qplot('time','unit profit','_rep',subdata)
plt.plot(data[['time','unit profit']].groupby('time').mean(),'k--',label='mean')
plt.legend()
```

### Plot Expected Discrete State Path

```python
fig4 = demo.figure('Expected Machine Age', 'Period','Age')
demo.qplot('time','age','_rep',subdata)
plt.plot(data[['time','age']].groupby('time').mean(),'k--',label='mean')
plt.legend()
```

notebooks/dp/02 Asset Replacement Model.ipynb